diff options
132 files changed, 2628 insertions, 734 deletions
diff --git a/Documentation/infiniband/user_mad.rst b/Documentation/infiniband/user_mad.rst index d88abfc0e370..cd66e7623d66 100644 --- a/Documentation/infiniband/user_mad.rst +++ b/Documentation/infiniband/user_mad.rst @@ -62,7 +62,7 @@ Receiving MADs struct ib_user_mad *mad; mad = malloc(sizeof *mad + 256); ret = read(fd, mad, sizeof *mad + 256); - if (ret == -ENOSPC)) { + if (ret == -ENOSPC) { length = mad.length; free(mad); mad = malloc(sizeof *mad + length); diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index ab7a2197bc86..47ef6b0afd29 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -38,6 +38,7 @@ ib_umad-y := user_mad.o ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types.o uverbs_ioctl.o \ uverbs_std_types_cq.o \ + uverbs_std_types_comp_cntr.o \ uverbs_std_types_dmabuf.o \ uverbs_std_types_dmah.o \ uverbs_std_types_flow_action.o uverbs_std_types_dm.o \ diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index e9fb7ad4c377..bdaa3eb83bcc 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -769,7 +769,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr) /* * sync canceling the work after removing it from the req_list - * guarentees no work is running and none will be started. + * guarantees no work is running and none will be started. */ cancel_delayed_work_sync(&found->work); kfree(found); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index e88d3efb967b..cc24fddf9aaa 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -2104,7 +2104,7 @@ static void destroy_id_handler_unlock(struct rdma_id_private *id_priv) /* * Setting the state to destroyed under the handler mutex provides a * fence against calling handler callbacks. If this is invoked due to - * the failure of a handler callback then it guarentees that no future + * the failure of a handler callback then it guarantees that no future * handlers will be called. */ lockdep_assert_held(&id_priv->handler_mutex); diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index a9e189194c13..a2c85840c501 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -234,7 +234,6 @@ static void rdma_counter_free(struct rdma_counter *counter) mutex_unlock(&port_counter->lock); - rdma_restrack_del(&counter->res); rdma_free_hw_stats_struct(counter->stats); kfree(counter); } @@ -329,6 +328,7 @@ static void counter_release(struct kref *kref) counter = container_of(kref, struct rdma_counter, kref); counter_history_stat_update(counter); + rdma_restrack_del(&counter->res); counter->device->ops.counter_dealloc(counter); rdma_counter_free(counter); } @@ -490,7 +490,8 @@ static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, return NULL; counter = container_of(res, struct rdma_counter, res); - kref_get(&counter->kref); + if (!kref_get_unless_zero(&counter->kref)) + counter = NULL; rdma_restrack_put(res); return counter; diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index 3d7b6cddd131..12304c9a9403 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -327,6 +327,7 @@ void ib_free_cq(struct ib_cq *cq) if (WARN_ON_ONCE(cq->cqe_used)) return; + rdma_restrack_del(&cq->res); if (cq->device->ops.pre_destroy_cq) { ret = cq->device->ops.pre_destroy_cq(cq); WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail"); @@ -353,7 +354,6 @@ void ib_free_cq(struct ib_cq *cq) else ret = cq->device->ops.destroy_cq(cq, NULL); WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); - rdma_restrack_del(&cq->res); kfree(cq->wc); kfree(cq); } @@ -393,8 +393,7 @@ static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes, * a reasonable batch size so that we can share CQs between * multiple users instead of allocating a larger number of CQs. */ - nr_cqes = min_t(unsigned int, dev->attrs.max_cqe, - max(nr_cqes, IB_MAX_SHARED_CQ_SZ)); + nr_cqes = min(dev->attrs.max_cqe, max(nr_cqes, IB_MAX_SHARED_CQ_SZ)); nr_cqs = min_t(unsigned int, dev->num_comp_vectors, num_online_cpus()); for (i = 0; i < nr_cqs; i++) { cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx); diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index d954eda63134..950a001b1de7 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2742,6 +2742,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, create_ah); SET_DEVICE_OP(dev_ops, create_counters); SET_DEVICE_OP(dev_ops, create_cq); + SET_DEVICE_OP(dev_ops, create_comp_cntr); SET_DEVICE_OP(dev_ops, create_user_cq); SET_DEVICE_OP(dev_ops, create_flow); SET_DEVICE_OP(dev_ops, create_qp); @@ -2762,6 +2763,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, destroy_ah); SET_DEVICE_OP(dev_ops, destroy_counters); SET_DEVICE_OP(dev_ops, destroy_cq); + SET_DEVICE_OP(dev_ops, destroy_comp_cntr); SET_DEVICE_OP(dev_ops, destroy_flow); SET_DEVICE_OP(dev_ops, destroy_flow_action); SET_DEVICE_OP(dev_ops, destroy_qp); @@ -2813,6 +2815,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, modify_hw_stat); SET_DEVICE_OP(dev_ops, modify_port); SET_DEVICE_OP(dev_ops, modify_qp); + SET_DEVICE_OP(dev_ops, qp_attach_comp_cntr); SET_DEVICE_OP(dev_ops, modify_srq); SET_DEVICE_OP(dev_ops, modify_wq); SET_DEVICE_OP(dev_ops, peek_cq); @@ -2826,6 +2829,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, post_srq_recv); SET_DEVICE_OP(dev_ops, process_mad); SET_DEVICE_OP(dev_ops, query_ah); + SET_DEVICE_OP(dev_ops, query_comp_cntr_caps); SET_DEVICE_OP(dev_ops, query_device); SET_DEVICE_OP(dev_ops, query_gid); SET_DEVICE_OP(dev_ops, query_pkey); @@ -2836,12 +2840,14 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, query_ucontext); SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); SET_DEVICE_OP(dev_ops, read_counters); + SET_DEVICE_OP(dev_ops, read_comp_cntr); SET_DEVICE_OP(dev_ops, reg_dm_mr); SET_DEVICE_OP(dev_ops, reg_user_mr); SET_DEVICE_OP(dev_ops, reg_user_mr_dmabuf); SET_DEVICE_OP(dev_ops, req_notify_cq); SET_DEVICE_OP(dev_ops, rereg_user_mr); SET_DEVICE_OP(dev_ops, resize_user_cq); + SET_DEVICE_OP(dev_ops, modify_comp_cntr); SET_DEVICE_OP(dev_ops, set_vf_guid); SET_DEVICE_OP(dev_ops, set_vf_link_state); SET_DEVICE_OP(dev_ops, ufile_hw_cleanup); @@ -2850,6 +2856,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_OBJ_SIZE(dev_ops, ib_ah); SET_OBJ_SIZE(dev_ops, ib_counters); SET_OBJ_SIZE(dev_ops, ib_cq); + SET_OBJ_SIZE(dev_ops, ib_comp_cntr); SET_OBJ_SIZE(dev_ops, ib_dmah); SET_OBJ_SIZE(dev_ops, ib_mw); SET_OBJ_SIZE(dev_ops, ib_pd); diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c index 4625abd29ac0..1b10f2973ad9 100644 --- a/drivers/infiniband/core/iwpm_msg.c +++ b/drivers/infiniband/core/iwpm_msg.c @@ -268,7 +268,7 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client) if (ret) goto query_mapping_error; - /* If flags are required and we're not V4, then return a quite error */ + /* If flags are required and we're not V4, then return a quiet error */ if (pm_msg->flags && iwpm_ulib_version == IWPM_UABI_VERSION_MIN) { ret = -EINVAL; goto query_mapping_error_nowarn; diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 02a0a9c0a4a6..b4bae92c5c4e 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -188,6 +188,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY] = { .type = NLA_U64 }, + [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX] = { .type = NLA_U64 }, }; static int put_driver_name_print_type(struct sk_buff *msg, const char *name, @@ -413,7 +414,7 @@ out: } static int fill_res_info_entry(struct sk_buff *msg, - const char *name, u64 curr) + const char *name, u64 curr, u64 max) { struct nlattr *entry_attr; @@ -427,6 +428,9 @@ static int fill_res_info_entry(struct sk_buff *msg, if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, RDMA_NLDEV_ATTR_PAD)) goto err; + if (max && nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, + max, RDMA_NLDEV_ATTR_PAD)) + goto err; nla_nest_end(msg, entry_attr); return 0; @@ -447,10 +451,13 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device, [RDMA_RESTRACK_MR] = "mr", [RDMA_RESTRACK_CTX] = "ctx", [RDMA_RESTRACK_SRQ] = "srq", + [RDMA_RESTRACK_COMP_CNTR] = "comp_cntr", }; + struct ib_comp_cntr_caps comp_cntr_caps = {}; struct nlattr *table_attr; - int ret, i, curr; + u64 curr, max; + int ret, i; if (fill_nldev_handle(msg, device)) return -EMSGSIZE; @@ -459,11 +466,36 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device, if (!table_attr) return -EMSGSIZE; + if (device->ops.query_comp_cntr_caps) + device->ops.query_comp_cntr_caps(device, &comp_cntr_caps, NULL); + for (i = 0; i < RDMA_RESTRACK_MAX; i++) { if (!names[i]) continue; curr = rdma_restrack_count(device, i, show_details); - ret = fill_res_info_entry(msg, names[i], curr); + switch (i) { + case RDMA_RESTRACK_QP: + max = device->attrs.max_qp; + break; + case RDMA_RESTRACK_CQ: + max = device->attrs.max_cq; + break; + case RDMA_RESTRACK_MR: + max = device->attrs.max_mr; + break; + case RDMA_RESTRACK_PD: + max = device->attrs.max_pd; + break; + case RDMA_RESTRACK_SRQ: + max = device->attrs.max_srq; + break; + case RDMA_RESTRACK_COMP_CNTR: + max = comp_cntr_caps.max_counters; + break; + default: + max = 0; + } + ret = fill_res_info_entry(msg, names[i], curr, max); if (ret) goto err; } @@ -2133,6 +2165,11 @@ static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[], nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS], rem) { + if (nla_len(entry_attr) != sizeof(u32)) { + ret = -EINVAL; + goto out; + } + index = nla_get_u32(entry_attr); if ((index >= stats->num_counters) || !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) { diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 56121103e9f4..2b91e8527287 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -159,6 +159,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile); extern const struct uapi_definition uverbs_def_obj_async_fd[]; extern const struct uapi_definition uverbs_def_obj_counters[]; +extern const struct uapi_definition uverbs_def_obj_comp_cntr[]; extern const struct uapi_definition uverbs_def_obj_cq[]; extern const struct uapi_definition uverbs_def_obj_device[]; extern const struct uapi_definition uverbs_def_obj_dm[]; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index cfee2071586c..f89a81dad72f 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -61,7 +61,7 @@ void rdma_restrack_clean(struct ib_device *dev) * @type: actual type of object to operate * @show_details: count driver specific objects */ -int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, +u32 rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, bool show_details) { struct rdma_restrack_root *rt = &dev->res[type]; @@ -104,6 +104,8 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) return container_of(res, struct ib_srq, res)->device; case RDMA_RESTRACK_DMAH: return container_of(res, struct ib_dmah, res)->device; + case RDMA_RESTRACK_COMP_CNTR: + return container_of(res, struct ib_comp_cntr, res)->device; default: WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type); return NULL; @@ -129,6 +131,46 @@ static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, res->user = true; } +static struct rdma_restrack_root *res_to_rt(struct rdma_restrack_entry *res) +{ + struct ib_device *dev = res_to_dev(res); + + if (WARN_ON(!dev)) + return NULL; + + return &dev->res[res->type]; +} + +static void restrack_drain_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, + GFP_KERNEL); + WARN_ON(old != res); + } + + rdma_restrack_put(res); + wait_for_completion(&res->comp); +} + +static void restrack_restore_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + reinit_completion(&res->comp); + kref_init(&res->kref); + + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, + GFP_KERNEL); + WARN_ON(old); + } +} + /** * rdma_restrack_set_name() - set the task for this resource * @res: resource entry @@ -177,22 +219,23 @@ void rdma_restrack_new(struct rdma_restrack_entry *res, EXPORT_SYMBOL(rdma_restrack_new); /** - * rdma_restrack_add() - add object to the resource tracking database + * rdma_restrack_add() - add object to the resource tracking database. + * If this resource reuses an ID of a resource that was already destroyed + * after calling rdma_restrack_begin() but didn't yet call + * rdma_restrack_commit_del() it can result in an untracked QP. * @res: resource entry */ void rdma_restrack_add(struct rdma_restrack_entry *res) { - struct ib_device *dev = res_to_dev(res); struct rdma_restrack_root *rt; int ret = 0; - if (!dev) - return; - if (res->no_track) goto out; - rt = &dev->res[res->type]; + rt = res_to_rt(res); + if (!rt) + return; if (res->type == RDMA_RESTRACK_QP) { /* Special case to ensure that LQPN points to right QP */ @@ -229,6 +272,28 @@ out: } EXPORT_SYMBOL(rdma_restrack_add); +/** + * rdma_restrack_abort_del() - re-add object to the resource tracking database + * it can only be used after rdma_restrack_begin_del(). + * @res: resource entry + */ +void rdma_restrack_abort_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_restore_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_abort_del); + int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) { return kref_get_unless_zero(&res->kref); @@ -265,7 +330,7 @@ static void restrack_release(struct kref *kref) struct rdma_restrack_entry *res; res = container_of(kref, struct rdma_restrack_entry, kref); - if (res->task) { + if (res->task && !res->valid) { put_task_struct(res->task); res->task = NULL; } @@ -291,37 +356,20 @@ EXPORT_SYMBOL(rdma_restrack_put); */ void rdma_restrack_sync(struct rdma_restrack_entry *res) { - struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct task_struct *task; - struct ib_device *dev; if (!res->valid || res->no_track) return; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) return; - old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); - if (WARN_ON(old != res)) - return; - - task = res->task; - if (task) - get_task_struct(task); - rdma_restrack_put(res); - wait_for_completion(&res->comp); - reinit_completion(&res->comp); - if (task) - res->task = task; - kref_init(&res->kref); - - xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); + restrack_drain_res(rt, res); + restrack_restore_res(rt, res); } EXPORT_SYMBOL(rdma_restrack_sync); @@ -333,7 +381,6 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) { struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct ib_device *dev; if (!res->valid) { if (res->task) { @@ -346,12 +393,10 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) if (res->no_track) goto out; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; - old = xa_erase(&rt->xa, res->id); WARN_ON(old != res); @@ -359,5 +404,61 @@ out: res->valid = false; rdma_restrack_put(res); wait_for_completion(&res->comp); + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } } EXPORT_SYMBOL(rdma_restrack_del); + +/** + * rdma_restrack_begin_del() - invalidate the object from the resource tracking + * database but preserve its index in the array. + * Since this preserves the index in the array until rdma_restrack_commit_del() + * is called, if rdma_restrack_add() is called in between with an old QP ID it + * can result in an untracked QP. + * @res: resource entry + */ +void rdma_restrack_begin_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_drain_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_begin_del); + +/** + * rdma_restrack_commit_del() - delete object from the resource tracking + * database and free the task. + * @res: resource entry + */ +void rdma_restrack_commit_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt; + + if (!res->valid || res->no_track) + goto out; + + rt = res_to_rt(res); + if (!rt) + return; + + xa_erase(&rt->xa, res->id); + +out: + res->valid = false; + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } +} +EXPORT_SYMBOL(rdma_restrack_commit_del); diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 75b8d1005a98..2df78e084e10 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -26,8 +26,11 @@ struct rdma_restrack_root { int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); +void rdma_restrack_abort_del(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); void rdma_restrack_sync(struct rdma_restrack_entry *res); +void rdma_restrack_begin_del(struct rdma_restrack_entry *res); +void rdma_restrack_commit_del(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 73498723a5d5..81f44dadfa52 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -209,7 +209,8 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, mmgrab(mm); - page_list = (struct page **) __get_free_page(GFP_KERNEL); + /* TODO: switch to "fast and as large as possible" allocation helper */ + page_list = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page_list) { ret = -ENOMEM; goto umem_kfree; @@ -269,7 +270,7 @@ umem_release: __ib_umem_release(device, umem, 0); atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm); out: - free_page((unsigned long) page_list); + kfree(page_list); umem_kfree: if (ret) { mmdrop(umem->owning_mm); diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index ad023c2d84d8..39b5564a4c35 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -181,7 +181,7 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, } EXPORT_SYMBOL(ib_umem_dmabuf_get); -static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { +static const struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { .allow_peer2peer = true, }; @@ -205,7 +205,7 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach) umem_dmabuf->revoked = 1; } -static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = { +static const struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = { .allow_peer2peer = true, .invalidate_mappings = ib_umem_dmabuf_revoke_locked, }; diff --git a/drivers/infiniband/core/uverbs_std_types_comp_cntr.c b/drivers/infiniband/core/uverbs_std_types_comp_cntr.c new file mode 100644 index 000000000000..2e7de84d94a6 --- /dev/null +++ b/drivers/infiniband/core/uverbs_std_types_comp_cntr.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#include <rdma/uverbs_std_types.h> +#include "rdma_core.h" +#include "uverbs.h" +#include "restrack.h" + +static int uverbs_free_comp_cntr(struct ib_uobject *uobject, enum rdma_remove_reason why, + struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr *cc = uobject->object; + int ret; + + if (atomic_read(&cc->usecnt)) + return -EBUSY; + + rdma_restrack_begin_del(&cc->res); + ret = cc->device->ops.destroy_comp_cntr(cc); + if (ret) { + rdma_restrack_abort_del(&cc->res); + return ret; + } + + rdma_restrack_commit_del(&cc->res); + kfree(cc); + return 0; +} + +static int UVERBS_HANDLER(UVERBS_METHOD_COMP_CNTR_CREATE)(struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, + UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE); + struct ib_device *ib_dev = attrs->context->device; + struct ib_comp_cntr *cc; + int ret; + + if (!ib_dev->ops.create_comp_cntr || + !ib_dev->ops.destroy_comp_cntr || + !ib_dev->ops.qp_attach_comp_cntr) + return -EOPNOTSUPP; + + cc = rdma_zalloc_drv_obj(ib_dev, ib_comp_cntr); + if (!cc) + return -ENOMEM; + + cc->device = ib_dev; + cc->uobject = uobj; + + rdma_restrack_new(&cc->res, RDMA_RESTRACK_COMP_CNTR); + rdma_restrack_set_name(&cc->res, NULL); + + ret = ib_dev->ops.create_comp_cntr(cc, attrs); + if (ret) + goto err_free; + + uobj->object = cc; + rdma_restrack_add(&cc->res); + uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE); + return 0; + +err_free: + rdma_restrack_put(&cc->res); + kfree(cc); + return ret; +} + +static int UVERBS_HANDLER(UVERBS_METHOD_COMP_CNTR_MODIFY)(struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr *cc = uverbs_attr_get_obj(attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE); + enum ib_comp_cntr_modify_op op; + enum ib_comp_cntr_entry entry; + u64 value; + int ret; + + if (!cc->device->ops.modify_comp_cntr) + return -EOPNOTSUPP; + + ret = uverbs_get_const(&entry, attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY); + if (ret) + return ret; + + ret = uverbs_get_const(&op, attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_OP); + if (ret) + return ret; + + ret = uverbs_copy_from(&value, attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE); + if (ret) + return ret; + + return cc->device->ops.modify_comp_cntr(cc, entry, op, value); +} + +static int UVERBS_HANDLER(UVERBS_METHOD_COMP_CNTR_READ)(struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr *cc = uverbs_attr_get_obj(attrs, UVERBS_ATTR_READ_COMP_CNTR_HANDLE); + enum ib_comp_cntr_entry entry; + u64 value = 0; + int ret; + + if (!cc->device->ops.read_comp_cntr) + return -EOPNOTSUPP; + + ret = uverbs_get_const(&entry, attrs, UVERBS_ATTR_READ_COMP_CNTR_ENTRY); + if (ret) + return ret; + + ret = cc->device->ops.read_comp_cntr(cc, entry, &value); + if (ret) + return ret; + + return uverbs_copy_to(attrs, UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, &value, sizeof(value)); +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_COMP_CNTR_CREATE, + UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_NEW, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD_DESTROY( + UVERBS_METHOD_COMP_CNTR_DESTROY, + UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_DESTROY, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_COMP_CNTR_MODIFY, + UVERBS_ATTR_IDR(UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_WRITE, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, + enum ib_uverbs_comp_cntr_entry, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_MODIFY_COMP_CNTR_OP, + enum ib_uverbs_comp_cntr_modify_op, + UA_MANDATORY), + UVERBS_ATTR_PTR_IN(UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_COMP_CNTR_READ, + UVERBS_ATTR_IDR(UVERBS_ATTR_READ_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_READ_COMP_CNTR_ENTRY, + enum ib_uverbs_comp_cntr_entry, + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_OBJECT( + UVERBS_OBJECT_COMP_CNTR, + UVERBS_TYPE_ALLOC_IDR(uverbs_free_comp_cntr), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_CREATE), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_DESTROY), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_MODIFY), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_READ)); + +const struct uapi_definition uverbs_def_obj_comp_cntr[] = { + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_COMP_CNTR, + UAPI_DEF_OBJ_NEEDS_FN(destroy_comp_cntr)), + {} +}; diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index 12ca15739cd2..ce0a7de00405 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -472,6 +472,42 @@ out: return ret; } +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_COMP_CNTR_CAPS)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr_caps caps = {}; + struct ib_ucontext *ucontext; + struct ib_device *ib_dev; + int ret; + + ucontext = ib_uverbs_get_ucontext(attrs); + if (IS_ERR(ucontext)) + return PTR_ERR(ucontext); + ib_dev = ucontext->device; + + if (!ib_dev->ops.query_comp_cntr_caps) + return -EOPNOTSUPP; + + ret = ib_dev->ops.query_comp_cntr_caps(ib_dev, &caps, attrs); + if (ret) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + &caps.max_counters, sizeof(caps.max_counters)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + &caps.max_value, sizeof(caps.max_value)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, + &caps.supported_qp_attach_ops, + sizeof(caps.supported_qp_attach_ops)); + return IS_UVERBS_COPY_ERR(ret) ? ret : 0; +} + DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_GET_CONTEXT, UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, @@ -542,6 +578,18 @@ DECLARE_UVERBS_NAMED_METHOD( netdev_ifindex), UA_MANDATORY)); +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QUERY_COMP_CNTR_CAPS, + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + UVERBS_ATTR_TYPE(u32), + UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + UVERBS_ATTR_TYPE(u64), + UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, + UVERBS_ATTR_TYPE(u32), + UA_OPTIONAL)); + DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, &UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), @@ -550,7 +598,8 @@ DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT_SPEED), &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_TABLE), - &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY)); + &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY), + &UVERBS_METHOD(UVERBS_METHOD_QUERY_COMP_CNTR_CAPS)); const struct uapi_definition uverbs_def_obj_device[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE), diff --git a/drivers/infiniband/core/uverbs_std_types_dmah.c b/drivers/infiniband/core/uverbs_std_types_dmah.c index 97101e093826..9873ab49a601 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmah.c +++ b/drivers/infiniband/core/uverbs_std_types_dmah.c @@ -18,11 +18,14 @@ static int uverbs_free_dmah(struct ib_uobject *uobject, if (atomic_read(&dmah->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&dmah->res); ret = dmah->device->ops.dealloc_dmah(dmah, attrs); - if (ret) + if (ret) { + rdma_restrack_abort_del(&dmah->res); return ret; + } - rdma_restrack_del(&dmah->res); + rdma_restrack_commit_del(&dmah->res); kfree(dmah); return 0; } diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c index 570b9656801d..0c72f801e0d3 100644 --- a/drivers/infiniband/core/uverbs_std_types_mr.c +++ b/drivers/infiniband/core/uverbs_std_types_mr.c @@ -364,7 +364,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)( dmah, attrs); else mr = pd->device->ops.reg_user_mr(pd, addr, length, iova, - access_flags, dmah, NULL); + access_flags, dmah, + &attrs->driver_udata); if (IS_ERR(mr)) return PTR_ERR(mr); @@ -527,7 +528,8 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY), UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY, UVERBS_ATTR_TYPE(u32), - UA_MANDATORY)); + UA_MANDATORY), + UVERBS_ATTR_UHW()); DECLARE_UVERBS_NAMED_METHOD_DESTROY( UVERBS_METHOD_MR_DESTROY, diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index 5767607dd420..30fc20fb251f 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -372,11 +372,76 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_qp_resp), UA_MANDATORY)); +static int UVERBS_HANDLER(UVERBS_METHOD_QP_ATTACH_COMP_CNTR)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *qp_uobj = uverbs_attr_get_uobject( + attrs, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE); + struct ib_comp_cntr *cc = uverbs_attr_get_obj( + attrs, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE); + struct ib_qp_attach_comp_cntr_attr attr = {}; + struct ib_qp *qp = qp_uobj->object; + int ret; + + if (!cc->device->ops.qp_attach_comp_cntr) + return -EOPNOTSUPP; + + if (qp->real_qp != qp) + return -EINVAL; + + ret = uverbs_get_flags32(&attr.op_mask, attrs, + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE); + if (ret) + return ret; + + if (!attr.op_mask) + return -EINVAL; + + if (attr.op_mask & qp->comp_cntr_op_mask) + return -EBUSY; + + ret = xa_err(xa_store(&qp->comp_cntrs, attr.op_mask, cc, GFP_KERNEL)); + if (ret) + return ret; + + ret = qp->device->ops.qp_attach_comp_cntr(qp, cc, &attr); + if (ret) { + xa_erase(&qp->comp_cntrs, attr.op_mask); + return ret; + } + + atomic_inc(&cc->usecnt); + qp->comp_cntr_op_mask |= attr.op_mask; + + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QP_ATTACH_COMP_CNTR, + UVERBS_ATTR_IDR(UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, + UVERBS_OBJECT_QP, + UVERBS_ACCESS_WRITE, + UA_MANDATORY), + UVERBS_ATTR_IDR(UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, + enum ib_uverbs_qp_attach_comp_cntr_op, + UA_MANDATORY)); + DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_QP, UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp), &UVERBS_METHOD(UVERBS_METHOD_QP_CREATE), - &UVERBS_METHOD(UVERBS_METHOD_QP_DESTROY)); + &UVERBS_METHOD(UVERBS_METHOD_QP_DESTROY), + &UVERBS_METHOD(UVERBS_METHOD_QP_ATTACH_COMP_CNTR)); const struct uapi_definition uverbs_def_obj_qp[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP, diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 4e2e556c8119..d150099b99d2 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -628,6 +628,7 @@ void uverbs_destroy_api(struct uverbs_api *uapi) static const struct uapi_definition uverbs_core_api[] = { UAPI_DEF_CHAIN(uverbs_def_obj_async_fd), UAPI_DEF_CHAIN(uverbs_def_obj_counters), + UAPI_DEF_CHAIN(uverbs_def_obj_comp_cntr), UAPI_DEF_CHAIN(uverbs_def_obj_cq), UAPI_DEF_CHAIN(uverbs_def_obj_device), UAPI_DEF_CHAIN(uverbs_def_obj_dm), diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 86811d31092c..04abc80c1327 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -392,6 +392,7 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) { int ret; + rdma_restrack_begin_del(&pd->res); if (pd->__internal_mr) { ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL); WARN_ON(ret); @@ -399,10 +400,12 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) } ret = pd->device->ops.dealloc_pd(pd, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&pd->res); return ret; + } - rdma_restrack_del(&pd->res); + rdma_restrack_commit_del(&pd->res); kfree(pd); return ret; } @@ -1140,16 +1143,20 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) if (atomic_read(&srq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&srq->res); + ret = srq->device->ops.destroy_srq(srq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&srq->res); return ret; + } atomic_dec(&srq->pd->usecnt); if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); - rdma_restrack_del(&srq->res); + rdma_restrack_commit_del(&srq->res); kfree(srq); return ret; @@ -1293,6 +1300,7 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, qp->qp_context = attr->qp_context; spin_lock_init(&qp->mr_lock); + xa_init(&qp->comp_cntrs); INIT_LIST_HEAD(&qp->rdma_mrs); INIT_LIST_HEAD(&qp->sig_mrs); init_completion(&qp->srq_completion); @@ -1327,6 +1335,7 @@ err_security: qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL); err_create: rdma_restrack_put(&qp->res); + xa_destroy(&qp->comp_cntrs); kfree(qp); return ERR_PTR(ret); @@ -2144,6 +2153,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr; const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr; struct ib_qp_security *sec; + struct ib_comp_cntr *cc; + unsigned long index; int ret; WARN_ON_ONCE(qp->mrs_used > 0); @@ -2154,6 +2165,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (qp->real_qp != qp) return __ib_destroy_shared_qp(qp); + rdma_restrack_begin_del(&qp->res); + sec = qp->qp_sec; if (sec) ib_destroy_qp_security_begin(sec); @@ -2166,6 +2179,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (ret) { if (sec) ib_destroy_qp_security_abort(sec); + rdma_restrack_abort_del(&qp->res); return ret; } @@ -2174,11 +2188,15 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (av_sgid_attr) rdma_put_gid_attr(av_sgid_attr); + xa_for_each(&qp->comp_cntrs, index, cc) + atomic_dec(&cc->usecnt); + xa_destroy(&qp->comp_cntrs); + ib_qp_usecnt_dec(qp); if (sec) ib_destroy_qp_security_end(sec); - rdma_restrack_del(&qp->res); + rdma_restrack_commit_del(&qp->res); kfree(qp); return ret; } @@ -2244,11 +2262,15 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (atomic_read(&cq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&cq->res); + ret = cq->device->ops.destroy_cq(cq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&cq->res); return ret; + } - rdma_restrack_del(&cq->res); + rdma_restrack_commit_del(&cq->res); kfree(cq); return ret; } diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c index 50156c300b33..ab6a2d2e95b5 100644 --- a/drivers/infiniband/hw/bng_re/bng_fw.c +++ b/drivers/infiniband/hw/bng_re/bng_fw.c @@ -401,14 +401,15 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie) { struct bng_re_cmdq_ctx *cmdq; struct bng_re_crsqe *crsqe; + unsigned long time_left; cmdq = &rcfw->cmdq; crsqe = &rcfw->crsqe_tbl[cookie]; do { - wait_event_timeout(cmdq->waitq, - !crsqe->is_in_used, - secs_to_jiffies(rcfw->max_timeout)); + time_left = wait_event_timeout(cmdq->waitq, + !crsqe->is_in_used, + secs_to_jiffies(rcfw->max_timeout)); if (!crsqe->is_in_used) return 0; @@ -417,6 +418,9 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie) if (!crsqe->is_in_used) return 0; + + if (!time_left) + return -ENODEV; } while (true); }; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 565762529007..a2a354a0fbef 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -193,7 +193,6 @@ int bnxt_re_query_device(struct ib_device *ibdev, if (rc) return rc; - memset(ib_attr, 0, sizeof(*ib_attr)); memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, min(sizeof(dev_attr->fw_ver), sizeof(ib_attr->fw_ver))); @@ -695,7 +694,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) struct bnxt_re_dev *rdev = pd->rdev; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -712,7 +711,7 @@ int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) &pd->qplib_pd)) atomic_dec(&rdev->stats.res.pd_count); } - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) @@ -844,7 +843,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, u8 nw_type; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -901,7 +900,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, if (active_ahs > rdev->stats.res.ah_watermark) rdev->stats.res.ah_watermark = active_ahs; - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr) @@ -1015,7 +1014,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) unsigned int flags; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -1064,7 +1063,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) if (scq_nq != rcq_nq) bnxt_re_synchronize_nq(rcq_nq); - return ib_respond_empty_udata(udata); + return 0; } static u8 __from_ib_qp_type(enum ib_qp_type type) @@ -2148,7 +2147,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -2159,7 +2158,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) free_page((unsigned long)srq->uctx_srq_page); ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); - return ib_respond_empty_udata(udata); + return 0; } static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, @@ -2297,34 +2296,25 @@ int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, { struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq); - struct bnxt_re_dev *rdev = srq->rdev; int ret; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; - switch (srq_attr_mask) { - case IB_SRQ_MAX_WR: - /* SRQ resize is not supported */ + if (srq_attr_mask != IB_SRQ_LIMIT) return -EINVAL; - case IB_SRQ_LIMIT: - /* Change the SRQ threshold */ - if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) - return -EINVAL; - - srq->qplib_srq.threshold = srq_attr->srq_limit; - bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); - /* On success, update the shadow */ - srq->srq_limit = srq_attr->srq_limit; - /* No need to Build and send response back to udata */ - return ib_respond_empty_udata(udata); - default: - ibdev_err(&rdev->ibdev, - "Unsupported srq_attr_mask 0x%x", srq_attr_mask); + if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) return -EINVAL; - } + + srq->qplib_srq.threshold = srq_attr->srq_limit; + bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold); + + /* On success, update the shadow */ + srq->srq_limit = srq_attr->srq_limit; + /* No need to Build and send response back to udata */ + return 0; } int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) @@ -2437,7 +2427,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, unsigned int flags; u8 nw_type; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -2689,7 +2679,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, if (rc) return rc; } - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, @@ -3471,7 +3461,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) nq = cq->qplib_cq.nq; cctx = rdev->chip_ctx; - ret = ib_is_udata_in_empty(udata); + ret = ib_no_udata_io(udata); if (ret) return ret; @@ -3486,7 +3476,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) atomic_dec(&rdev->stats.res.cq_count); kfree(cq->cql); ib_umem_release(cq->umem); - return ib_respond_empty_udata(udata); + return 0; } int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, @@ -3688,6 +3678,10 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; + rc = ib_respond_empty_udata(udata); + if (rc) + goto fail; + cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, entries * sizeof(struct cq_base), IB_ACCESS_LOCAL_WRITE); @@ -3717,7 +3711,7 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, cq->ib_cq.cqe = cq->resize_cqe; atomic_inc(&rdev->stats.res.resize_count); - return ib_respond_empty_udata(udata); + return 0; fail: if (cq->resize_umem) { @@ -4449,7 +4443,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) struct bnxt_re_dev *rdev = mr->rdev; int rc; - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; @@ -4472,7 +4466,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) atomic_dec(&rdev->stats.res.mr_count); if (rc) return rc; - return ib_respond_empty_udata(udata); + return 0; } static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) @@ -4991,11 +4985,13 @@ int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) case BNXT_RE_MMAP_DBR_PAGE: case BNXT_RE_MMAP_TOGGLE_PAGE: /* Driver doesn't expect write access for user space */ - if (vma->vm_flags & VM_WRITE) + if (vma->vm_flags & VM_WRITE) { ret = -EFAULT; - else + } else { + vm_flags_clear(vma, VM_MAYWRITE); ret = vm_insert_page(vma, vma->vm_start, virt_to_page((void *)bnxt_entry->mem_offset)); + } break; default: ret = -EINVAL; diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index cd1b01014198..49498c75f38f 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -541,7 +541,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (err) goto err_umem_release; - pages = (__be64 *) __get_free_page(GFP_KERNEL); + pages = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!pages) { err = -ENOMEM; goto err_pbl_free; @@ -568,7 +568,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mhp->wr_waitp); pbl_done: - free_page((unsigned long) pages); + kfree(pages); if (err) goto err_pbl_free; diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index e1eec37ee822..ebe3170a641c 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -263,7 +263,7 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro pr_debug("ibdev %p\n", ibdev); - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; @@ -300,7 +300,7 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro props->max_fast_reg_page_list_len = t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl); - return ib_respond_empty_udata(uhw); + return 0; } static int c4iw_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/efa/Makefile b/drivers/infiniband/hw/efa/Makefile index 6e83083af0bc..a6a433b0ba2f 100644 --- a/drivers/infiniband/hw/efa/Makefile +++ b/drivers/infiniband/hw/efa/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. # # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver. # obj-$(CONFIG_INFINIBAND_EFA) += efa.o -efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o +efa-y := efa_com_cmd.o efa_ah_cache.o efa_com.o efa_main.o efa_verbs.o diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index f4586bb170c1..6f6686eefb1a 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_ @@ -110,6 +110,14 @@ struct efa_cq { struct ib_umem *umem; }; +struct efa_comp_cntr { + struct ib_comp_cntr ibcc; + struct ib_umem *comp_umem; + struct ib_umem *err_umem; + u32 comp_handle; + u32 err_handle; +}; + struct efa_qp { struct ib_qp ibqp; dma_addr_t rq_dma_addr; @@ -164,6 +172,16 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs); +int efa_query_comp_cntr_caps(struct ib_device *ibdev, + struct ib_comp_cntr_caps *caps, + struct uverbs_attr_bundle *attrs); +int efa_create_comp_cntr(struct ib_comp_cntr *ibcc, + struct uverbs_attr_bundle *attrs); +int efa_destroy_comp_cntr(struct ib_comp_cntr *ibcc); +int efa_modify_comp_cntr(struct ib_comp_cntr *ibcc, enum ib_comp_cntr_entry entry, + enum ib_comp_cntr_modify_op op, u64 value); +int efa_qp_attach_comp_cntr(struct ib_qp *ibqp, struct ib_comp_cntr *ibcc, + struct ib_qp_attach_comp_cntr_attr *attr); struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_dmah *dmah, diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index 826790ca9d83..fc07213e26af 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -28,7 +28,13 @@ enum efa_admin_aq_opcode { EFA_ADMIN_CREATE_EQ = 18, EFA_ADMIN_DESTROY_EQ = 19, EFA_ADMIN_ALLOC_MR = 20, - EFA_ADMIN_MAX_OPCODE = 20, + EFA_ADMIN_SERVICE = 21, + EFA_ADMIN_CREATE_EVENT_COUNTER = 25, + EFA_ADMIN_DESTROY_EVENT_COUNTER = 26, + EFA_ADMIN_ATTACH_EVENT_COUNTER = 27, + EFA_ADMIN_MODIFY_EVENT_COUNTER = 28, + EFA_ADMIN_DETACH_EVENT_COUNTER = 29, + EFA_ADMIN_MAX_OPCODE = 29, }; enum efa_admin_aq_feature_id { @@ -367,10 +373,10 @@ struct efa_admin_reg_mr_cmd { /* * flags and page size - * 4:0 : phys_page_size_shift - page size is (1 << + * 5:0 : phys_page_size_shift - page size is (1 << * phys_page_size_shift). Page size is used for * building the Virtual to Physical address mapping - * 6:5 : reserved - MBZ + * 6 : reserved - MBZ * 7 : mem_addr_phy_mode_en - Enable bit for physical * memory registration (no translation), can be used * only by privileged clients. If set, PBL must @@ -722,7 +728,9 @@ struct efa_admin_feature_device_attr_desc { * on TX queues * 4 : unsolicited_write_recv - If set, unsolicited * write with imm. receive is supported - * 31:5 : reserved - MBZ + * 5 : event_counters - If set, event counters are + * supported + * 31:6 : reserved - MBZ */ u32 device_caps; @@ -811,6 +819,34 @@ struct efa_admin_feature_queue_attr_desc_1 { struct efa_admin_feature_queue_attr_desc_2 { /* Maximum size of data that can be sent inline in a Send WQE */ u16 inline_buf_size_ex; + + /* MBZ */ + u8 reserved[6]; + + /* + * Supported counter QP events + * 0 : send_comp + * 1 : send_comp_err + * 2 : recv_comp + * 3 : recv_comp_err + * 4 : read_comp + * 5 : read_comp_err + * 6 : write_comp + * 7 : write_comp_err + * 8 : remote_read_comp + * 9 : remote_write_comp + * 31:10 : reserved - MBZ + */ + u32 supported_event_counter_qp_events; + + /* Maximum number of counters */ + u32 max_event_counters; + + /* + * Maximum counter value, counter wraps around to 0 after reaching + * this value + */ + u64 event_counter_max_val; }; struct efa_admin_event_queue_attr_desc { @@ -1089,6 +1125,127 @@ struct efa_admin_host_info { u32 flags; }; +struct efa_admin_service_cmd { + struct efa_admin_aq_common_desc aq_common_descriptor; + + u8 buffer[60]; +}; + +struct efa_admin_service_resp { + struct efa_admin_acq_common_desc acq_common_desc; + + u8 buffer[56]; +}; + +/* Create Counter command */ +struct efa_admin_create_event_counter_cmd { + struct efa_admin_aq_common_desc aq_common_descriptor; + + /* UAR number */ + u16 uar; + + /* MBZ */ + u16 reserved; + + /* Counter physical address */ + u64 paddr; +}; + +struct efa_admin_create_event_counter_resp { + struct efa_admin_acq_common_desc acq_common_desc; + + /* Counter handle */ + u32 cntr_handle; + + /* MBZ */ + u32 reserved; +}; + +struct efa_admin_destroy_event_counter_cmd { + struct efa_admin_aq_common_desc aq_common_descriptor; + + /* Counter handle */ + u32 cntr_handle; +}; + +struct efa_admin_destroy_event_counter_resp { + struct efa_admin_acq_common_desc acq_common_desc; +}; + +enum efa_admin_event_counter_attach_type { + EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS = 0, +}; + +struct efa_admin_event_counter_attach_qp_events { + /* QP handle */ + u32 qp_handle; + + /* + * Bitmask of counter QP events + * 0 : send_comp + * 1 : send_comp_err + * 2 : recv_comp + * 3 : recv_comp_err + * 4 : read_comp + * 5 : read_comp_err + * 6 : write_comp + * 7 : write_comp_err + * 8 : remote_read_comp + * 9 : remote_write_comp + * 31:10 : reserved - MBZ + */ + u32 events; +}; + +struct efa_admin_attach_detach_event_counter_cmd { + struct efa_admin_aq_common_desc aq_common_descriptor; + + /* Counter handle */ + u32 cntr_handle; + + /* efa_admin_event_counter_attach_type */ + u8 attach_type; + + /* MBZ */ + u8 reserved[3]; + + union { + struct efa_admin_event_counter_attach_qp_events qp_events; + } u; +}; + +struct efa_admin_attach_detach_event_counter_resp { + struct efa_admin_acq_common_desc acq_common_desc; +}; + +/* Counter modify operations */ +enum efa_admin_event_counter_modify_ops { + /* Set counter value */ + EFA_ADMIN_EVENT_COUNTER_MODIFY_SET = 0, + /* Add to counter value */ + EFA_ADMIN_EVENT_COUNTER_MODIFY_ADD = 1, +}; + +struct efa_admin_modify_event_counter_cmd { + struct efa_admin_aq_common_desc aq_common_descriptor; + + /* Counter handle */ + u32 cntr_handle; + + /* Counter operation type (efa_admin_event_counter_modify_ops) */ + u8 operation; + + /* MBZ */ + u8 reserved[7]; + + /* Value for SET or ADD */ + u64 value; +}; + +struct efa_admin_modify_event_counter_resp { + struct efa_admin_acq_common_desc acq_common_desc; +}; + /* create_qp_cmd */ #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK BIT(0) #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK BIT(1) @@ -1103,7 +1260,7 @@ struct efa_admin_host_info { #define EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY_MASK BIT(5) /* reg_mr_cmd */ -#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(4, 0) +#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(5, 0) #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7) #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0) #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1) @@ -1129,6 +1286,19 @@ struct efa_admin_host_info { #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_DATA_POLLING_128_MASK BIT(2) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4) +#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5) + +/* feature_queue_attr_desc_2 */ +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_SEND_COMP_MASK BIT(0) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_SEND_COMP_ERR_MASK BIT(1) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_RECV_COMP_MASK BIT(2) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_RECV_COMP_ERR_MASK BIT(3) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_READ_COMP_MASK BIT(4) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_READ_COMP_ERR_MASK BIT(5) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_WRITE_COMP_MASK BIT(6) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_WRITE_COMP_ERR_MASK BIT(7) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_REMOTE_READ_COMP_MASK BIT(8) +#define EFA_ADMIN_FEATURE_QUEUE_ATTR_DESC_2_REMOTE_WRITE_COMP_MASK BIT(9) /* create_eq_cmd */ #define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) @@ -1147,4 +1317,16 @@ struct efa_admin_host_info { #define EFA_ADMIN_HOST_INFO_INTREE_MASK BIT(0) #define EFA_ADMIN_HOST_INFO_GDR_MASK BIT(1) +/* counter_attach_qp_events */ +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP_MASK BIT(0) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP_ERR_MASK BIT(1) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP_MASK BIT(2) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP_ERR_MASK BIT(3) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP_MASK BIT(4) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP_ERR_MASK BIT(5) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP_MASK BIT(6) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP_ERR_MASK BIT(7) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_READ_COMP_MASK BIT(8) +#define EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_WRITE_COMP_MASK BIT(9) + #endif /* _EFA_ADMIN_CMDS_H_ */ diff --git a/drivers/infiniband/hw/efa/efa_ah_cache.c b/drivers/infiniband/hw/efa/efa_ah_cache.c new file mode 100644 index 000000000000..6219529fa889 --- /dev/null +++ b/drivers/infiniband/hw/efa/efa_ah_cache.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Copyright 2026 Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#include <linux/slab.h> + +#include "efa_ah_cache.h" + +static const struct rhashtable_params ah_cache_params = { + .key_len = sizeof(struct efa_ah_cache_key), + .key_offset = offsetof(struct efa_ah_cache_entry, key), + .head_offset = offsetof(struct efa_ah_cache_entry, linkage), +}; + +int efa_ah_cache_init(struct efa_ah_cache *ah_cache) +{ + int err; + + mutex_init(&ah_cache->lock); + err = rhashtable_init(&ah_cache->hashtable, &ah_cache_params); + if (err) + mutex_destroy(&ah_cache->lock); + + return err; +} + +static void efa_ah_cache_entry_free(void *ptr, void *arg) +{ + struct efa_ah_cache_entry *entry = ptr; + + WARN_ON(entry->usecnt); + mutex_destroy(&entry->lock); + kfree(entry); +} + +void efa_ah_cache_destroy(struct efa_ah_cache *ah_cache) +{ + rhashtable_free_and_destroy(&ah_cache->hashtable, efa_ah_cache_entry_free, NULL); + mutex_destroy(&ah_cache->lock); +} + +static struct efa_ah_cache_entry *efa_ah_cache_lookup_locked(struct efa_ah_cache *ah_cache, u16 pd, + u8 *gid) + __must_hold(&ah_cache->lock) +{ + struct efa_ah_cache_key key = {}; + + memcpy(key.gid, gid, sizeof(key.gid)); + key.pd = pd; + + return rhashtable_lookup_fast(&ah_cache->hashtable, &key, ah_cache_params); +} + +struct efa_ah_cache_entry *efa_ah_cache_lookup(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid) +{ + struct efa_ah_cache_entry *entry; + + mutex_lock(&ah_cache->lock); + entry = efa_ah_cache_lookup_locked(ah_cache, pd, gid); + mutex_unlock(&ah_cache->lock); + + return entry; +} + +/** + * efa_ah_cache_get - Get or create an AH cache entry + * @ah_cache: AH cache + * @pd: Protection domain number + * @gid: GID address + * + * Look up an AH cache entry by PD and GID. If found, take a reference and + * return it. If not found, allocate a new entry and insert it. The caller must lock + * the entry mutex and check usecnt to determine whether a device create + * command is needed. + * + * Return: Pointer to the entry on success, ERR_PTR on failure. + */ +struct efa_ah_cache_entry *efa_ah_cache_get(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid) +{ + struct efa_ah_cache_entry *entry; + int err; + + mutex_lock(&ah_cache->lock); + + entry = efa_ah_cache_lookup_locked(ah_cache, pd, gid); + if (entry) { + refcount_inc(&entry->refcount); + mutex_unlock(&ah_cache->lock); + return entry; + } + + entry = kzalloc_obj(*entry); + if (!entry) { + mutex_unlock(&ah_cache->lock); + return ERR_PTR(-ENOMEM); + } + + memcpy(entry->key.gid, gid, sizeof(entry->key.gid)); + entry->key.pd = pd; + refcount_set(&entry->refcount, 1); + mutex_init(&entry->lock); + + err = rhashtable_insert_fast(&ah_cache->hashtable, &entry->linkage, ah_cache_params); + if (err) { + mutex_destroy(&entry->lock); + kfree(entry); + mutex_unlock(&ah_cache->lock); + return ERR_PTR(err); + } + + mutex_unlock(&ah_cache->lock); + return entry; +} + +/** + * efa_ah_cache_put - Put a refcount of an AH cache entry + * @ah_cache: AH cache + * @entry: AH cache entry + * + * Drop the refcount. If it reaches zero, remove the entry from the hashtable + * and free it. + */ +void efa_ah_cache_put(struct efa_ah_cache *ah_cache, struct efa_ah_cache_entry *entry) +{ + if (!refcount_dec_and_mutex_lock(&entry->refcount, &ah_cache->lock)) + return; + + /* AH cache lock is held here */ + rhashtable_remove_fast(&ah_cache->hashtable, &entry->linkage, ah_cache_params); + mutex_unlock(&ah_cache->lock); + + mutex_destroy(&entry->lock); + kfree(entry); +} diff --git a/drivers/infiniband/hw/efa/efa_ah_cache.h b/drivers/infiniband/hw/efa/efa_ah_cache.h new file mode 100644 index 000000000000..e7cdcbb64070 --- /dev/null +++ b/drivers/infiniband/hw/efa/efa_ah_cache.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright 2026 Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#ifndef _EFA_AH_CACHE_H_ +#define _EFA_AH_CACHE_H_ + +#include <linux/refcount.h> +#include <linux/rhashtable.h> + +#define EFA_AH_GID_SIZE 16 + +struct efa_ah_cache_key { + u8 gid[EFA_AH_GID_SIZE]; + u16 pd; +}; + +struct efa_ah_cache_entry { + struct efa_ah_cache_key key; + u16 ah; + unsigned int usecnt; + refcount_t refcount; + struct rhash_head linkage; + struct mutex lock; /* Serializes device commands per cache entry */ +}; + +struct efa_ah_cache { + struct rhashtable hashtable; + struct mutex lock; /* Protects AH cache hashtable */ +}; + +int efa_ah_cache_init(struct efa_ah_cache *ah_cache); +void efa_ah_cache_destroy(struct efa_ah_cache *ah_cache); +struct efa_ah_cache_entry *efa_ah_cache_get(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid); +struct efa_ah_cache_entry *efa_ah_cache_lookup(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid); +void efa_ah_cache_put(struct efa_ah_cache *ah_cache, struct efa_ah_cache_entry *entry); + +#endif /* _EFA_AH_CACHE_H_ */ diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index 7cc3f4af0bb9..7097d1c2f23d 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -724,6 +724,8 @@ void efa_com_admin_destroy(struct efa_com_dev *edev) size = aenq->depth * sizeof(*aenq->entries); dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr); + + efa_ah_cache_destroy(&edev->ah_cache); } /** @@ -782,6 +784,12 @@ int efa_com_admin_init(struct efa_com_dev *edev, return -ENODEV; } + err = efa_ah_cache_init(&edev->ah_cache); + if (err) { + ibdev_err(edev->efa_dev, "Failed to init AH cache\n"); + return err; + } + aq->depth = EFA_ADMIN_QUEUE_DEPTH; aq->dmadev = edev->dmadev; @@ -794,7 +802,7 @@ int efa_com_admin_init(struct efa_com_dev *edev, err = efa_com_init_comp_ctxt(aq); if (err) - return err; + goto err_destroy_ah_cache; err = efa_com_admin_init_sq(edev); if (err) @@ -832,6 +840,8 @@ err_destroy_sq: aq->sq.entries, aq->sq.dma_addr); err_destroy_comp_ctxt: devm_kfree(edev->dmadev, aq->comp_ctx); +err_destroy_ah_cache: + efa_ah_cache_destroy(&edev->ah_cache); return err; } diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h index f8c692b0e092..599db9d583bf 100644 --- a/drivers/infiniband/hw/efa/efa_com.h +++ b/drivers/infiniband/hw/efa/efa_com.h @@ -14,6 +14,7 @@ #include <rdma/ib_verbs.h> +#include "efa_ah_cache.h" #include "efa_common_defs.h" #include "efa_admin_defs.h" #include "efa_admin_cmds_defs.h" @@ -113,6 +114,8 @@ struct efa_com_dev { u32 supported_features; u32 dma_addr_bits; + struct efa_ah_cache ah_cache; + u32 dev_api_ver; struct efa_com_mmio_read mmio_read; }; diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 5db4f5805b59..a9fd44b4debf 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -322,8 +322,21 @@ int efa_com_create_ah(struct efa_com_dev *edev, struct efa_admin_create_ah_resp cmd_completion; struct efa_com_admin_queue *aq = &edev->aq; struct efa_admin_create_ah_cmd ah_cmd = {}; + struct efa_ah_cache_entry *entry; int err; + entry = efa_ah_cache_get(&edev->ah_cache, params->pdn, params->dest_addr); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + mutex_lock(&entry->lock); + if (entry->usecnt) { + result->ah = entry->ah; + entry->usecnt++; + mutex_unlock(&entry->lock); + return 0; + } + ah_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_AH; memcpy(ah_cmd.dest_addr, params->dest_addr, sizeof(ah_cmd.dest_addr)); @@ -335,13 +348,18 @@ int efa_com_create_ah(struct efa_com_dev *edev, (struct efa_admin_acq_entry *)&cmd_completion, sizeof(cmd_completion)); if (err) { + mutex_unlock(&entry->lock); + efa_ah_cache_put(&edev->ah_cache, entry); ibdev_err_ratelimited(edev->efa_dev, "Failed to create ah for %pI6 [%d]\n", ah_cmd.dest_addr, err); return err; } + entry->ah = cmd_completion.ah; result->ah = cmd_completion.ah; + entry->usecnt++; + mutex_unlock(&entry->lock); return 0; } @@ -352,11 +370,20 @@ int efa_com_destroy_ah(struct efa_com_dev *edev, struct efa_admin_destroy_ah_resp cmd_completion; struct efa_admin_destroy_ah_cmd ah_cmd = {}; struct efa_com_admin_queue *aq = &edev->aq; - int err; + struct efa_ah_cache_entry *entry; + int err = 0; + + entry = efa_ah_cache_lookup(&edev->ah_cache, params->pdn, params->gid); + if (!entry) + return -EINVAL; + + mutex_lock(&entry->lock); + if (entry->usecnt > 1) + goto out_put; ah_cmd.aq_common_desc.opcode = EFA_ADMIN_DESTROY_AH; - ah_cmd.ah = params->ah; - ah_cmd.pd = params->pdn; + ah_cmd.ah = entry->ah; + ah_cmd.pd = entry->key.pd; err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&ah_cmd, @@ -364,13 +391,19 @@ int efa_com_destroy_ah(struct efa_com_dev *edev, (struct efa_admin_acq_entry *)&cmd_completion, sizeof(cmd_completion)); if (err) { + mutex_unlock(&entry->lock); ibdev_err_ratelimited(edev->efa_dev, "Failed to destroy ah-%d pd-%d [%d]\n", ah_cmd.ah, ah_cmd.pd, err); return err; } - return 0; +out_put: + entry->usecnt--; + mutex_unlock(&entry->lock); + efa_ah_cache_put(&edev->ah_cache, entry); + + return err; } bool @@ -520,6 +553,10 @@ int efa_com_get_device_attr(struct efa_com_dev *edev, } result->inline_buf_size_ex = resp.u.queue_attr_2.inline_buf_size_ex; + result->max_event_counters = resp.u.queue_attr_2.max_event_counters; + result->event_counter_max_val = resp.u.queue_attr_2.event_counter_max_val; + result->supported_event_counter_qp_events = + resp.u.queue_attr_2.supported_event_counter_qp_events; } else { result->inline_buf_size_ex = result->inline_buf_size; } @@ -855,3 +892,135 @@ int efa_com_get_stats(struct efa_com_dev *edev, return 0; } + +int efa_com_create_event_counter(struct efa_com_dev *edev, + struct efa_com_create_event_counter_params *params, + struct efa_com_create_event_counter_result *result) +{ + struct efa_admin_create_event_counter_cmd cmd = {}; + struct efa_admin_create_event_counter_resp resp; + struct efa_com_admin_queue *aq = &edev->aq; + int err; + + cmd.aq_common_descriptor.opcode = EFA_ADMIN_CREATE_EVENT_COUNTER; + cmd.uar = params->uarn; + cmd.paddr = params->dma_addr; + + err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&cmd, + sizeof(cmd), + (struct efa_admin_acq_entry *)&resp, + sizeof(resp)); + if (err) { + ibdev_err_ratelimited(edev->efa_dev, + "Failed to create event counter [%d]\n", + err); + return err; + } + + result->cntr_handle = resp.cntr_handle; + return 0; +} + +int efa_com_destroy_event_counter(struct efa_com_dev *edev, + struct efa_com_destroy_event_counter_params *params) +{ + struct efa_admin_destroy_event_counter_cmd cmd = {}; + struct efa_admin_destroy_event_counter_resp resp; + struct efa_com_admin_queue *aq = &edev->aq; + int err; + + cmd.aq_common_descriptor.opcode = EFA_ADMIN_DESTROY_EVENT_COUNTER; + cmd.cntr_handle = params->cntr_handle; + + err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&cmd, + sizeof(cmd), + (struct efa_admin_acq_entry *)&resp, + sizeof(resp)); + if (err) { + ibdev_err_ratelimited(edev->efa_dev, + "Failed to destroy event counter [%d]\n", + err); + return err; + } + + return 0; +} + +static int efa_com_attach_detach_event_counter(struct efa_com_dev *edev, u8 opcode, + u32 cntr_handle, u32 qp_handle, + u32 events) +{ + struct efa_admin_attach_detach_event_counter_cmd cmd = {}; + struct efa_admin_attach_detach_event_counter_resp resp; + struct efa_com_admin_queue *aq = &edev->aq; + int err; + + cmd.aq_common_descriptor.opcode = opcode; + cmd.cntr_handle = cntr_handle; + cmd.attach_type = EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS; + cmd.u.qp_events.qp_handle = qp_handle; + cmd.u.qp_events.events = events; + + err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&cmd, + sizeof(cmd), + (struct efa_admin_acq_entry *)&resp, + sizeof(resp)); + if (err) { + ibdev_err_ratelimited(edev->efa_dev, + "Failed to %s event counter [%d]\n", + opcode == EFA_ADMIN_ATTACH_EVENT_COUNTER + ? "attach" + : "detach", + err); + return err; + } + + return 0; +} + +int efa_com_attach_event_counter(struct efa_com_dev *edev, + struct efa_com_attach_event_counter_params *params) +{ + return efa_com_attach_detach_event_counter(edev, + EFA_ADMIN_ATTACH_EVENT_COUNTER, + params->cntr_handle, + params->qp_handle, + params->events); +} + +int efa_com_detach_event_counter(struct efa_com_dev *edev, + struct efa_com_detach_event_counter_params *params) +{ + return efa_com_attach_detach_event_counter(edev, + EFA_ADMIN_DETACH_EVENT_COUNTER, + params->cntr_handle, + params->qp_handle, + params->events); +} + +int efa_com_modify_event_counter(struct efa_com_dev *edev, + struct efa_com_modify_event_counter_params *params) +{ + struct efa_admin_modify_event_counter_cmd cmd = {}; + struct efa_admin_modify_event_counter_resp resp; + struct efa_com_admin_queue *aq = &edev->aq; + int err; + + cmd.aq_common_descriptor.opcode = EFA_ADMIN_MODIFY_EVENT_COUNTER; + cmd.cntr_handle = params->cntr_handle; + cmd.operation = params->operation; + cmd.value = params->value; + + err = efa_com_cmd_exec(aq, (struct efa_admin_aq_entry *)&cmd, + sizeof(cmd), + (struct efa_admin_acq_entry *)&resp, + sizeof(resp)); + if (err) { + ibdev_err_ratelimited(edev->efa_dev, + "Failed to modify event counter [%d]\n", + err); + return err; + } + + return 0; +} diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index ef15b3c38429..3e7380fa0abb 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -106,6 +106,7 @@ struct efa_com_create_ah_result { struct efa_com_destroy_ah_params { u16 ah; + u8 gid[EFA_GID_SIZE]; u16 pdn; }; @@ -145,6 +146,9 @@ struct efa_com_get_device_attr_result { u16 min_sq_depth; u16 max_link_speed_gbps; u8 db_bar; + u32 max_event_counters; + u64 event_counter_max_val; + u32 supported_event_counter_qp_events; }; struct efa_com_get_hw_hints_result { @@ -300,6 +304,37 @@ union efa_com_get_stats_result { struct efa_com_network_stats network_stats; }; +struct efa_com_create_event_counter_params { + dma_addr_t dma_addr; + u16 uarn; +}; + +struct efa_com_create_event_counter_result { + u32 cntr_handle; +}; + +struct efa_com_destroy_event_counter_params { + u32 cntr_handle; +}; + +struct efa_com_attach_event_counter_params { + u32 cntr_handle; + u32 qp_handle; + u32 events; +}; + +struct efa_com_detach_event_counter_params { + u32 cntr_handle; + u32 qp_handle; + u32 events; +}; + +struct efa_com_modify_event_counter_params { + u32 cntr_handle; + u8 operation; + u64 value; +}; + int efa_com_create_qp(struct efa_com_dev *edev, struct efa_com_create_qp_params *params, struct efa_com_create_qp_result *res); @@ -350,5 +385,16 @@ int efa_com_dealloc_uar(struct efa_com_dev *edev, int efa_com_get_stats(struct efa_com_dev *edev, struct efa_com_get_stats_params *params, union efa_com_get_stats_result *result); +int efa_com_create_event_counter(struct efa_com_dev *edev, + struct efa_com_create_event_counter_params *params, + struct efa_com_create_event_counter_result *result); +int efa_com_destroy_event_counter(struct efa_com_dev *edev, + struct efa_com_destroy_event_counter_params *params); +int efa_com_attach_event_counter(struct efa_com_dev *edev, + struct efa_com_attach_event_counter_params *params); +int efa_com_detach_event_counter(struct efa_com_dev *edev, + struct efa_com_detach_event_counter_params *params); +int efa_com_modify_event_counter(struct efa_com_dev *edev, + struct efa_com_modify_event_counter_params *params); #endif /* _EFA_COM_CMD_H_ */ diff --git a/drivers/infiniband/hw/efa/efa_io_defs.h b/drivers/infiniband/hw/efa/efa_io_defs.h index a4c9fd33da38..a849d92a6eb6 100644 --- a/drivers/infiniband/hw/efa/efa_io_defs.h +++ b/drivers/infiniband/hw/efa/efa_io_defs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* - * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_IO_H_ @@ -65,6 +65,8 @@ enum efa_io_comp_status { EFA_IO_COMP_STATUS_REMOTE_ERROR_UNKNOWN_PEER = 14, /* Unreachable remote - never received a response */ EFA_IO_COMP_STATUS_LOCAL_ERROR_UNREACH_REMOTE = 15, + /* Remote feature mismatch */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_FEATURE_MISMATCH = 18, }; enum efa_io_frwr_pbl_mode { @@ -72,6 +74,11 @@ enum efa_io_frwr_pbl_mode { EFA_IO_FRWR_DIRECT_PBL = 1, }; +enum efa_io_processing_hint { + /* Optimize for throughput */ + EFA_IO_PROCESSING_HINT_BURST_PPS_SENSITIVE = 1 << 0, +}; + struct efa_io_tx_meta_desc { /* Verbs-generated Request ID */ u16 req_id; @@ -121,7 +128,15 @@ struct efa_io_tx_meta_desc { u16 ah; - u16 reserved; + /* + * control flags + * 1:0 : processing_hints - Bitmask of enum + * efa_io_processing_hint + * 7:2 : reserved - MBZ + */ + u8 ctrl3; + + u8 reserved; /* Queue key */ u32 qkey; @@ -365,6 +380,7 @@ struct efa_io_rx_cdesc_ex { #define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2) #define EFA_IO_TX_META_DESC_LAST_MASK BIT(3) #define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4) +#define EFA_IO_TX_META_DESC_PROCESSING_HINTS_MASK GENMASK(1, 0) /* tx_buf_desc */ #define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0) diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 97da8e828e34..4cd80727dcd2 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* - * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include <linux/module.h> @@ -17,12 +17,14 @@ #define PCI_DEV_ID_EFA1_VF 0xefa1 #define PCI_DEV_ID_EFA2_VF 0xefa2 #define PCI_DEV_ID_EFA3_VF 0xefa3 +#define PCI_DEV_ID_EFA4_VF 0xefa4 static const struct pci_device_id efa_pci_tbl[] = { { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) }, { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) }, { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA2_VF) }, { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA3_VF) }, + { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA4_VF) }, { } }; @@ -372,20 +374,25 @@ static const struct ib_device_ops efa_dev_ops = { .alloc_pd = efa_alloc_pd, .alloc_ucontext = efa_alloc_ucontext, .create_user_cq = efa_create_user_cq, + .create_comp_cntr = efa_create_comp_cntr, .create_qp = efa_create_qp, .create_user_ah = efa_create_ah, .dealloc_pd = efa_dealloc_pd, .dealloc_ucontext = efa_dealloc_ucontext, .dereg_mr = efa_dereg_mr, .destroy_ah = efa_destroy_ah, + .destroy_comp_cntr = efa_destroy_comp_cntr, .destroy_cq = efa_destroy_cq, .destroy_qp = efa_destroy_qp, .get_hw_stats = efa_get_hw_stats, .get_link_layer = efa_port_link_layer, .get_port_immutable = efa_get_port_immutable, + .modify_comp_cntr = efa_modify_comp_cntr, .mmap = efa_mmap, .mmap_free = efa_mmap_free, .modify_qp = efa_modify_qp, + .qp_attach_comp_cntr = efa_qp_attach_comp_cntr, + .query_comp_cntr_caps = efa_query_comp_cntr_caps, .query_device = efa_query_device, .query_gid = efa_query_gid, .query_pkey = efa_query_pkey, @@ -397,6 +404,7 @@ static const struct ib_device_ops efa_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_comp_cntr, efa_comp_cntr, ibcc), INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp), INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext), diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 06d3365aeb56..9bc8199b60bf 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -167,6 +167,11 @@ static inline struct efa_ah *to_eah(struct ib_ah *ibah) return container_of(ibah, struct efa_ah, ibah); } +static inline struct efa_comp_cntr *to_ecc(struct ib_comp_cntr *ibcc) +{ + return container_of(ibcc, struct efa_comp_cntr, ibcc); +} + static inline struct efa_user_mmap_entry * to_emmap(struct rdma_user_mmap_entry *rdma_entry) { @@ -222,7 +227,6 @@ int efa_query_device(struct ib_device *ibdev, dev_attr = &dev->dev_attr; - memset(props, 0, sizeof(*props)); props->max_mr_size = dev_attr->max_mr_pages * PAGE_SIZE; props->page_size_cap = dev_attr->page_size_cap; props->vendor_id = dev->pdev->vendor; @@ -265,6 +269,9 @@ int efa_query_device(struct ib_device *ibdev, if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV; + if (EFA_DEV_CAP(dev, EVENT_COUNTERS)) + resp.device_caps |= EFA_QUERY_DEVICE_CAPS_COMP_CNTR; + if (dev->neqs) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; @@ -2054,10 +2061,11 @@ int efa_mmap(struct ib_ucontext *ibucontext, static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah) { - struct efa_com_destroy_ah_params params = { - .ah = ah->ah, - .pdn = to_epd(ah->ibah.pd)->pdn, - }; + struct efa_com_destroy_ah_params params = {}; + + params.ah = ah->ah; + memcpy(params.gid, ah->id, sizeof(params.gid)); + params.pdn = to_epd(ah->ibah.pd)->pdn; return efa_com_destroy_ah(&dev->edev, ¶ms); } @@ -2260,6 +2268,220 @@ enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, return IB_LINK_LAYER_UNSPECIFIED; } +int efa_query_comp_cntr_caps(struct ib_device *ibdev, + struct ib_comp_cntr_caps *caps, + struct uverbs_attr_bundle *attrs) +{ + struct efa_dev *dev = to_edev(ibdev); + u32 dev_ops = dev->dev_attr.supported_event_counter_qp_events; + + caps->max_counters = dev->dev_attr.max_event_counters / 2; + caps->max_value = dev->dev_attr.event_counter_max_val; + + caps->supported_qp_attach_ops = 0; + if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP) && + EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP_ERR)) + caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_SEND; + if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP) && + EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP_ERR)) + caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_RECV; + if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP) && + EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP_ERR)) + caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ; + if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP) && + EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP_ERR)) + caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE; + if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_READ_COMP)) + caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ; + if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_WRITE_COMP)) + caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE; + + return 0; +} + +static int efa_create_event_counter(struct efa_dev *dev, u16 uarn, dma_addr_t addr, u32 *handle) +{ + struct efa_com_create_event_counter_params params = {}; + struct efa_com_create_event_counter_result result; + int err; + + params.uarn = uarn; + params.dma_addr = addr; + + err = efa_com_create_event_counter(&dev->edev, ¶ms, &result); + if (err) + return err; + + *handle = result.cntr_handle; + return 0; +} + +static int efa_destroy_event_counter(struct efa_dev *dev, u32 handle) +{ + struct efa_com_destroy_event_counter_params params = { + .cntr_handle = handle, + }; + + return efa_com_destroy_event_counter(&dev->edev, ¶ms); +} + +int efa_create_comp_cntr(struct ib_comp_cntr *ibcc, struct uverbs_attr_bundle *attrs) +{ + struct efa_dev *dev = to_edev(ibcc->device); + struct efa_comp_cntr *cc = to_ecc(ibcc); + struct efa_ucontext *ucontext; + struct ib_umem *comp_umem; + struct ib_umem *err_umem; + dma_addr_t comp_addr; + dma_addr_t err_addr; + int err; + + ucontext = rdma_udata_to_drv_context(&attrs->driver_udata, struct efa_ucontext, + ibucontext); + + comp_umem = ib_umem_get_attr(ibcc->device, attrs, EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, + sizeof(u64), IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(comp_umem)) + return PTR_ERR(comp_umem); + + err_umem = ib_umem_get_attr(ibcc->device, attrs, EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, + sizeof(u64), IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(err_umem)) { + err = PTR_ERR(err_umem); + goto err_comp_umem; + } + + comp_addr = ib_umem_start_dma_addr(comp_umem); + err_addr = ib_umem_start_dma_addr(err_umem); + + if (!IS_ALIGNED(comp_addr, sizeof(u64)) || !IS_ALIGNED(err_addr, sizeof(u64))) { + ibdev_dbg(&dev->ibdev, "Completion Counter memory is unaligned\n"); + err = -EINVAL; + goto err_err_umem; + } + + err = efa_create_event_counter(dev, ucontext->uarn, comp_addr, &cc->comp_handle); + if (err) { + ibdev_dbg(&dev->ibdev, "Failed to create comp event counter [%d]\n", err); + goto err_err_umem; + } + + err = efa_create_event_counter(dev, ucontext->uarn, err_addr, &cc->err_handle); + if (err) { + ibdev_dbg(&dev->ibdev, "Failed to create err event counter [%d]\n", err); + goto err_destroy_comp_event_cntr; + } + + cc->comp_umem = comp_umem; + cc->err_umem = err_umem; + + return 0; + +err_destroy_comp_event_cntr: + efa_destroy_event_counter(dev, cc->comp_handle); +err_err_umem: + ib_umem_release(err_umem); +err_comp_umem: + ib_umem_release(comp_umem); + return err; +} + +int efa_destroy_comp_cntr(struct ib_comp_cntr *ibcc) +{ + struct efa_dev *dev = to_edev(ibcc->device); + struct efa_comp_cntr *cc = to_ecc(ibcc); + + efa_destroy_event_counter(dev, cc->comp_handle); + efa_destroy_event_counter(dev, cc->err_handle); + + ib_umem_release(cc->comp_umem); + ib_umem_release(cc->err_umem); + return 0; +} + +int efa_modify_comp_cntr(struct ib_comp_cntr *ibcc, enum ib_comp_cntr_entry entry, + enum ib_comp_cntr_modify_op op, u64 value) +{ + struct efa_com_modify_event_counter_params params = {}; + struct efa_comp_cntr *cc = to_ecc(ibcc); + + params.cntr_handle = entry == IB_COMP_CNTR_ENTRY_ERR ? cc->err_handle : cc->comp_handle; + params.operation = op == IB_COMP_CNTR_MODIFY_OP_SET ? + EFA_ADMIN_EVENT_COUNTER_MODIFY_SET : EFA_ADMIN_EVENT_COUNTER_MODIFY_ADD; + params.value = value; + + return efa_com_modify_event_counter(&to_edev(ibcc->device)->edev, ¶ms); +} + +static u32 efa_comp_cntr_op_to_comp_events(u32 op_mask) +{ + u32 events = 0; + + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_SEND) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RECV) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_READ_COMP, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_WRITE_COMP, 1); + + return events; +} + +static u32 efa_comp_cntr_op_to_err_events(u32 op_mask) +{ + u32 events = 0; + + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_SEND) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP_ERR, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RECV) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP_ERR, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP_ERR, 1); + if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) + EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP_ERR, 1); + + return events; +} + +int efa_qp_attach_comp_cntr(struct ib_qp *ibqp, struct ib_comp_cntr *ibcc, + struct ib_qp_attach_comp_cntr_attr *attr) +{ + struct efa_com_detach_event_counter_params detach_params = {}; + struct efa_com_attach_event_counter_params params = {}; + struct efa_dev *dev = to_edev(ibqp->device); + struct efa_comp_cntr *cc = to_ecc(ibcc); + struct efa_qp *qp = to_eqp(ibqp); + int err; + + params.cntr_handle = cc->comp_handle; + params.qp_handle = qp->qp_handle; + params.events = efa_comp_cntr_op_to_comp_events(attr->op_mask); + + err = efa_com_attach_event_counter(&dev->edev, ¶ms); + if (err) + return err; + + params.cntr_handle = cc->err_handle; + params.events = efa_comp_cntr_op_to_err_events(attr->op_mask); + + err = efa_com_attach_event_counter(&dev->edev, ¶ms); + if (err) { + detach_params.cntr_handle = cc->comp_handle; + detach_params.qp_handle = qp->qp_handle; + detach_params.events = efa_comp_cntr_op_to_comp_events(attr->op_mask); + efa_com_detach_event_counter(&dev->edev, &detach_params); + return err; + } + + return 0; +} + DECLARE_UVERBS_NAMED_METHOD(EFA_IB_METHOD_MR_QUERY, UVERBS_ATTR_IDR(EFA_IB_ATTR_QUERY_MR_HANDLE, UVERBS_OBJECT_MR, @@ -2282,8 +2504,23 @@ ADD_UVERBS_METHODS(efa_mr, UVERBS_OBJECT_MR, &UVERBS_METHOD(EFA_IB_METHOD_MR_QUERY)); +ADD_UVERBS_ATTRIBUTES_SIMPLE( + efa_comp_cntr_create, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_METHOD_COMP_CNTR_CREATE, + UVERBS_ATTR_PTR_IN( + EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, + UVERBS_ATTR_STRUCT(struct ib_uverbs_buffer_desc, length), + UA_MANDATORY), + UVERBS_ATTR_PTR_IN( + EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, + UVERBS_ATTR_STRUCT(struct ib_uverbs_buffer_desc, length), + UA_MANDATORY)); + const struct uapi_definition efa_uapi_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &efa_mr), + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_COMP_CNTR, + &efa_comp_cntr_create), {}, }; diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index d5b9d19882b2..a8784e07acd6 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -220,6 +220,7 @@ static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) irq_set_affinity_hint(eqc->irq.msix_vector, NULL); free_irq(eqc->irq.msix_vector, eqc); + tasklet_kill(&eqc->tasklet); } static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 74afe6eb18b0..71e3e8618a61 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -320,12 +320,10 @@ int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct erdma_dev *dev = to_edev(ibdev); int err; - err = ib_is_udata_in_empty(udata); + err = ib_no_udata_io(udata); if (err) return err; - memset(attr, 0, sizeof(*attr)); - attr->max_mr_size = dev->attrs.max_mr_size; attr->vendor_id = PCI_VENDOR_ID_ALIBABA; attr->vendor_part_id = dev->pdev->device; @@ -363,7 +361,7 @@ int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, addrconf_addr_eui48((u8 *)&attr->sys_image_guid, dev->netdev->dev_addr); - return ib_respond_empty_udata(udata); + return 0; } int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 44c524e45396..592e330e74bf 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -11929,26 +11929,27 @@ u8 encode_rcv_header_entry_size(u8 size) /** * hfi1_validate_rcvhdrcnt - validate hdrcnt - * @dd: the device data + * @pdev: the PCI device * @thecnt: the header count */ -int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt) +int hfi1_validate_rcvhdrcnt(struct pci_dev *pdev, uint thecnt) { if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) { - dd_dev_err(dd, "Receive header queue count too small\n"); + dev_err(&pdev->dev, "Receive header queue count too small\n"); return -EINVAL; } if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) { - dd_dev_err(dd, - "Receive header queue count cannot be greater than %u\n", - HFI1_MAX_HDRQ_EGRBUF_CNT); + dev_err(&pdev->dev, + "Receive header queue count cannot be greater than %u\n", + HFI1_MAX_HDRQ_EGRBUF_CNT); return -EINVAL; } if (thecnt % HDRQ_INCREMENT) { - dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n", - thecnt, HDRQ_INCREMENT); + dev_err(&pdev->dev, + "Receive header queue count %u must be divisible by %lu\n", + thecnt, HDRQ_INCREMENT); return -EINVAL; } @@ -15007,7 +15008,7 @@ int hfi1_init_dd(struct hfi1_devdata *dd) */ ret = hfi1_pcie_ddinit(dd, pdev); if (ret < 0) - goto bail_free; + goto bail; /* Save PCI space registers to rewrite after device reset */ ret = save_pci_variables(dd); @@ -15262,8 +15263,6 @@ bail_clear_intr: bail_cleanup: hfi1_free_rx(dd); hfi1_pcie_ddcleanup(dd); -bail_free: - hfi1_free_devdata(dd); bail: return ret; } diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h index 56e03d486ace..bc1c9e1c3172 100644 --- a/drivers/infiniband/hw/hfi1/chip.h +++ b/drivers/infiniband/hw/hfi1/chip.h @@ -660,7 +660,7 @@ static inline u32 chip_rcv_array_count(struct hfi1_devdata *dd) } u8 encode_rcv_header_entry_size(u8 size); -int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt); +int hfi1_validate_rcvhdrcnt(struct pci_dev *pdev, uint thecnt); void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt); u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, diff --git a/drivers/infiniband/hw/hfi1/device.c b/drivers/infiniband/hw/hfi1/device.c index a98a4175e53b..adcfb80d52d4 100644 --- a/drivers/infiniband/hw/hfi1/device.c +++ b/drivers/infiniband/hw/hfi1/device.c @@ -10,18 +10,6 @@ #include "hfi.h" #include "device.h" -static char *hfi1_devnode(const struct device *dev, umode_t *mode) -{ - if (mode) - *mode = 0600; - return kasprintf(GFP_KERNEL, "%s", dev_name(dev)); -} - -static const struct class class = { - .name = "hfi1", - .devnode = hfi1_devnode, -}; - static char *hfi1_user_devnode(const struct device *dev, umode_t *mode) { if (mode) @@ -38,7 +26,6 @@ static dev_t hfi1_dev; int hfi1_cdev_init(int minor, const char *name, const struct file_operations *fops, struct cdev *cdev, struct device **devp, - bool user_accessible, struct kobject *parent) { const dev_t dev = MKDEV(MAJOR(hfi1_dev), minor); @@ -57,10 +44,7 @@ int hfi1_cdev_init(int minor, const char *name, goto done; } - if (user_accessible) - device = device_create(&user_class, NULL, dev, NULL, "%s", name); - else - device = device_create(&class, NULL, dev, NULL, "%s", name); + device = device_create(&user_class, NULL, dev, NULL, "%s", name); if (IS_ERR(device)) { ret = PTR_ERR(device); @@ -100,33 +84,21 @@ int __init dev_init(void) ret = alloc_chrdev_region(&hfi1_dev, 0, HFI1_NMINORS, DRIVER_NAME); if (ret < 0) { pr_err("Could not allocate chrdev region (err %d)\n", -ret); - goto done; - } - - ret = class_register(&class); - if (ret) { - pr_err("Could not create device class (err %d)\n", -ret); - unregister_chrdev_region(hfi1_dev, HFI1_NMINORS); - goto done; + return ret; } ret = class_register(&user_class); if (ret) { pr_err("Could not create device class for user accessible files (err %d)\n", -ret); - class_unregister(&class); unregister_chrdev_region(hfi1_dev, HFI1_NMINORS); - goto done; } -done: return ret; } void dev_cleanup(void) { - class_unregister(&class); class_unregister(&user_class); - unregister_chrdev_region(hfi1_dev, HFI1_NMINORS); } diff --git a/drivers/infiniband/hw/hfi1/device.h b/drivers/infiniband/hw/hfi1/device.h index a91bea426ba5..3e2d21770e6c 100644 --- a/drivers/infiniband/hw/hfi1/device.h +++ b/drivers/infiniband/hw/hfi1/device.h @@ -9,7 +9,6 @@ int hfi1_cdev_init(int minor, const char *name, const struct file_operations *fops, struct cdev *cdev, struct device **devp, - bool user_accessible, struct kobject *parent); void hfi1_cdev_cleanup(struct cdev *cdev, struct device **devp); const char *class_name(void); diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 56031becb273..dc548e6802e2 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -1689,7 +1689,7 @@ static int user_add(struct hfi1_devdata *dd) snprintf(name, sizeof(name), "%s_%d", class_name(), dd->unit); ret = hfi1_cdev_init(dd->unit, name, &hfi1_file_ops, &dd->user_cdev, &dd->user_device, - true, &dd->verbs_dev.rdi.ibdev.dev.kobj); + &dd->verbs_dev.rdi.ibdev.dev.kobj); if (ret) user_remove(dd); diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index 5a0310f758dc..80d480c4fc6a 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -2024,9 +2024,7 @@ struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd) /* waiting for an urgent packet to arrive */ #define HFI1_CTXT_WAITING_URG 4 -/* free up any allocated data at closes */ int hfi1_init_dd(struct hfi1_devdata *dd); -void hfi1_free_devdata(struct hfi1_devdata *dd); /* LED beaconing functions */ void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, @@ -2132,7 +2130,7 @@ void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd); /* Hook for sysfs read of QSFP */ int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len); -int hfi1_pcie_init(struct hfi1_devdata *dd); +int hfi1_pcie_init(struct pci_dev *pdev); void hfi1_pcie_cleanup(struct pci_dev *pdev); int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev); void hfi1_pcie_ddcleanup(struct hfi1_devdata *); diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index b7fd8b1fbbbd..f94d896e7212 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -629,8 +629,6 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd, ppd->sm_trap_qp = 0x0; ppd->sa_qp = 0x1; - ppd->hfi1_wq = NULL; - spin_lock_init(&ppd->cca_timer_lock); for (i = 0; i < OPA_MAX_SLS; i++) { @@ -740,31 +738,27 @@ static int create_workqueues(struct hfi1_devdata *dd) for (pidx = 0; pidx < dd->num_pports; ++pidx) { ppd = dd->pport + pidx; - if (!ppd->hfi1_wq) { - ppd->hfi1_wq = - alloc_workqueue( - "hfi%d_%d", - WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | - WQ_PERCPU, - HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES, - dd->unit, pidx); - if (!ppd->hfi1_wq) - goto wq_error; - } - if (!ppd->link_wq) { - /* - * Make the link workqueue single-threaded to enforce - * serialization. - */ - ppd->link_wq = - alloc_workqueue( - "hfi_link_%d_%d", - WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND, - 1, /* max_active */ - dd->unit, pidx); - if (!ppd->link_wq) - goto wq_error; - } + ppd->hfi1_wq = + alloc_workqueue( + "hfi%d_%d", + WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | + WQ_PERCPU, + HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES, + dd->unit, pidx); + if (!ppd->hfi1_wq) + goto wq_error; + /* + * Make the link workqueue single-threaded to enforce + * serialization. + */ + ppd->link_wq = + alloc_workqueue( + "hfi_link_%d_%d", + WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND, + 1, /* max_active */ + dd->unit, pidx); + if (!ppd->link_wq) + goto wq_error; } return 0; wq_error: @@ -1161,7 +1155,7 @@ static void finalize_asic_data(struct hfi1_devdata *dd, * It cleans up and frees all data structures set up by * by hfi1_alloc_devdata(). */ -void hfi1_free_devdata(struct hfi1_devdata *dd) +static void hfi1_free_devdata(struct hfi1_devdata *dd) { struct hfi1_asic_data *ad; unsigned long flags; @@ -1225,8 +1219,9 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, GFP_KERNEL); if (ret < 0) { dev_err(&pdev->dev, - "Could not allocate unit ID: error %d\n", -ret); - goto bail; + "Could not allocate unit ID: error %pe\n", ERR_PTR(ret)); + rvt_dealloc_device(&dd->verbs_dev.rdi); + return ERR_PTR(ret); } /* @@ -1554,50 +1549,28 @@ static void postinit_cleanup(struct hfi1_devdata *dd) hfi1_dev_affinity_clean_up(dd); hfi1_pcie_ddcleanup(dd); - hfi1_pcie_cleanup(dd->pcidev); cleanup_device_data(dd); - - hfi1_free_devdata(dd); } static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { - int ret = 0, j, pidx, initfail; + int ret; struct hfi1_devdata *dd; - struct hfi1_pportdata *ppd; /* First, lock the non-writable module parameters */ HFI1_CAP_LOCK(); - /* Validate dev ids */ - if (!(ent->device == PCI_DEVICE_ID_INTEL0 || - ent->device == PCI_DEVICE_ID_INTEL1)) { - dev_err(&pdev->dev, "Failing on unknown Intel deviceid 0x%x\n", - ent->device); - ret = -ENODEV; - goto bail; - } - - /* Allocate the dd so we can get to work */ - dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS * - sizeof(struct hfi1_pportdata)); - if (IS_ERR(dd)) { - ret = PTR_ERR(dd); - goto bail; - } - /* Validate some global module parameters */ - ret = hfi1_validate_rcvhdrcnt(dd, rcvhdrcnt); + ret = hfi1_validate_rcvhdrcnt(pdev, rcvhdrcnt); if (ret) - goto bail; + return ret; /* use the encoding function as a sanitization check */ if (!encode_rcv_header_entry_size(hfi1_hdrq_entsize)) { - dd_dev_err(dd, "Invalid HdrQ Entry size %u\n", - hfi1_hdrq_entsize); - ret = -EINVAL; - goto bail; + dev_err(&pdev->dev, "Invalid HdrQ Entry size %u\n", + hfi1_hdrq_entsize); + return -EINVAL; } /* The receive eager buffer size must be set before the receive @@ -1617,87 +1590,75 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) clamp_val(eager_buffer_size, MIN_EAGER_BUFFER * 8, MAX_EAGER_BUFFER_TOTAL); - dd_dev_info(dd, "Eager buffer size %u\n", - eager_buffer_size); + pci_info(pdev, "Eager buffer size %u\n", eager_buffer_size); } else { - dd_dev_err(dd, "Invalid Eager buffer size of 0\n"); - ret = -EINVAL; - goto bail; + dev_err(&pdev->dev, "Invalid Eager buffer size of 0\n"); + return -EINVAL; } /* restrict value of hfi1_rcvarr_split */ hfi1_rcvarr_split = clamp_val(hfi1_rcvarr_split, 0, 100); - ret = hfi1_pcie_init(dd); + ret = hfi1_pcie_init(pdev); if (ret) - goto bail; + return ret; + + /* Allocate the dd so we can get to work */ + dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS * + sizeof(struct hfi1_pportdata)); + if (IS_ERR(dd)) { + ret = PTR_ERR(dd); + goto clean_pcie; + } + + ret = create_workqueues(dd); + if (ret) + goto free_devdata; /* - * Do device-specific initialization, function table setup, dd - * allocation, etc. + * Do device-specific initialization, function table setup, etc. */ ret = hfi1_init_dd(dd); if (ret) - goto clean_bail; /* error already printed */ - - ret = create_workqueues(dd); - if (ret) - goto clean_bail; + goto destroy_workqueues; /* error already printed */ /* do the generic initialization */ - initfail = hfi1_init(dd, 0); + ret = hfi1_init(dd, 0); + if (ret) + goto free_rx; ret = hfi1_register_ib_device(dd); + if (ret) + goto free_rx; /* * Now ready for use. this should be cleared whenever we - * detect a reset, or initiate one. If earlier failure, - * we still create devices, so diags, etc. can be used - * to determine cause of problem. + * detect a reset, or initiate one. */ - if (!initfail && !ret) { - dd->flags |= HFI1_INITTED; - /* create debufs files after init and ib register */ - hfi1_dbg_ibdev_init(&dd->verbs_dev); - } + dd->flags |= HFI1_INITTED; - j = hfi1_device_create(dd); - if (j) - dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j); - - if (initfail || ret) { - msix_clean_up_interrupts(dd); - stop_timers(dd); - flush_workqueue(ib_wq); - for (pidx = 0; pidx < dd->num_pports; ++pidx) { - hfi1_quiet_serdes(dd->pport + pidx); - ppd = dd->pport + pidx; - if (ppd->hfi1_wq) { - destroy_workqueue(ppd->hfi1_wq); - ppd->hfi1_wq = NULL; - } - if (ppd->link_wq) { - destroy_workqueue(ppd->link_wq); - ppd->link_wq = NULL; - } - } - if (!j) - hfi1_device_remove(dd); - if (!ret) - hfi1_unregister_ib_device(dd); - postinit_cleanup(dd); - if (initfail) - ret = initfail; - goto bail; /* everything already cleaned */ - } + ret = hfi1_device_create(dd); + if (ret) + dd_dev_err(dd, "Failed to create /dev devices: %pe\n", + ERR_PTR(ret)); sdma_start(dd); + hfi1_dbg_ibdev_init(&dd->verbs_dev); return 0; -clean_bail: +free_rx: + hfi1_free_rx(dd); + shutdown_device(dd); + stop_timers(dd); + postinit_cleanup(dd); + +destroy_workqueues: + destroy_workqueues(dd); +free_devdata: + hfi1_free_devdata(dd); +clean_pcie: hfi1_pcie_cleanup(pdev); -bail: return ret; } @@ -1737,14 +1698,11 @@ static void remove_one(struct pci_dev *pdev) * clear dma engines, etc. */ shutdown_device(dd); - destroy_workqueues(dd); - stop_timers(dd); - - /* wait until all of our (qsfp) queue_work() calls complete */ - flush_workqueue(ib_wq); - postinit_cleanup(dd); + destroy_workqueues(dd); + hfi1_free_devdata(dd); + hfi1_pcie_cleanup(pdev); } static void shutdown_one(struct pci_dev *pdev) diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 7133964749f8..1154b8cc713c 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -21,10 +21,9 @@ /* * Do all the common PCIe setup and initialization. */ -int hfi1_pcie_init(struct hfi1_devdata *dd) +int hfi1_pcie_init(struct pci_dev *pdev) { int ret; - struct pci_dev *pdev = dd->pcidev; ret = pci_enable_device(pdev); if (ret) { @@ -40,13 +39,15 @@ int hfi1_pcie_init(struct hfi1_devdata *dd) * about that, it appears. If the original BAR was retained * in the kernel data structures, this may be OK. */ - dd_dev_err(dd, "pci enable failed: error %d\n", -ret); + dev_err(&pdev->dev, "pci enable failed: error %pe\n", + ERR_PTR(ret)); return ret; } ret = pci_request_regions(pdev, DRIVER_NAME); if (ret) { - dd_dev_err(dd, "pci_request_regions fails: err %d\n", -ret); + dev_err(&pdev->dev, "pci_request_regions fails: err %pe\n", + ERR_PTR(ret)); goto bail; } @@ -59,7 +60,8 @@ int hfi1_pcie_init(struct hfi1_devdata *dd) */ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { - dd_dev_err(dd, "Unable to set DMA mask: %d\n", ret); + dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n", + ERR_PTR(ret)); goto bail; } } @@ -512,29 +514,28 @@ pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) switch (state) { case pci_channel_io_normal: - dd_dev_info(dd, "State Normal, ignoring\n"); + dev_info(&pdev->dev, "State Normal, ignoring\n"); break; case pci_channel_io_frozen: - dd_dev_info(dd, "State Frozen, requesting reset\n"); + dev_info(&pdev->dev, "State Frozen, requesting reset\n"); pci_disable_device(pdev); ret = PCI_ERS_RESULT_NEED_RESET; break; case pci_channel_io_perm_failure: + dev_info(&pdev->dev, "State Permanent Failure, disabling\n"); if (dd) { - dd_dev_info(dd, "State Permanent Failure, disabling\n"); /* no more register accesses! */ dd->flags &= ~HFI1_PRESENT; hfi1_disable_after_error(dd); } - /* else early, or other problem */ ret = PCI_ERS_RESULT_DISCONNECT; break; default: /* shouldn't happen */ - dd_dev_info(dd, "HFI1 PCI errors detected (state %d)\n", - state); + dev_info(&pdev->dev, "HFI1 PCI errors detected (state %d)\n", + state); break; } return ret; @@ -561,9 +562,7 @@ pci_mmio_enabled(struct pci_dev *pdev) static pci_ers_result_t pci_slot_reset(struct pci_dev *pdev) { - struct hfi1_devdata *dd = pci_get_drvdata(pdev); - - dd_dev_info(dd, "HFI1 slot_reset function called, ignored\n"); + dev_info(&pdev->dev, "HFI1 slot_reset function called, ignored\n"); return PCI_ERS_RESULT_CAN_RECOVER; } @@ -572,7 +571,10 @@ pci_resume(struct pci_dev *pdev) { struct hfi1_devdata *dd = pci_get_drvdata(pdev); - dd_dev_info(dd, "HFI1 resume function called\n"); + dev_info(&pdev->dev, "HFI1 resume function called\n"); + if (!dd) + return; + /* * Running jobs will fail, since it's asynchronous * unlike sysfs-requested reset. Better than diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c index 8ea5ed918a02..be6b82ba93af 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.c +++ b/drivers/infiniband/hw/hfi1/user_sdma.c @@ -1026,6 +1026,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, struct user_sdma_txreq *tx, u32 datalen) { u32 ahg[AHG_KDETH_ARRAY_SIZE]; + int ret; int idx = 0; u8 omfactor; /* KDETH.OM */ struct hfi1_user_sdma_pkt_q *pq = req->pq; @@ -1130,11 +1131,13 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt, req->info.comp_idx, req->sde->this_idx, req->ahg_idx, ahg, idx, tidval); - sdma_txinit_ahg(&tx->txreq, - SDMA_TXREQ_F_USE_AHG, - datalen, req->ahg_idx, idx, - ahg, sizeof(req->hdr), - user_sdma_txreq_cb); + ret = sdma_txinit_ahg(&tx->txreq, + SDMA_TXREQ_F_USE_AHG, + datalen, req->ahg_idx, idx, + ahg, sizeof(req->hdr), + user_sdma_txreq_cb); + if (ret) + return ret; return idx; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 2b3a1cafd1b2..27cc7df55ee7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4393,7 +4393,7 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_mhop mhop; struct hns_roce_hem *hem; unsigned long mhop_obj = obj; - int i, j, k; + u64 i, j, k; int ret = 0; u64 hem_idx = 0; u64 l1_idx = 0; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index c6f633bd5a34..662959efcf31 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -223,12 +223,10 @@ static int hns_roce_query_device(struct ib_device *ib_dev, struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); int ret; - ret = ib_is_udata_in_empty(uhw); + ret = ib_no_udata_io(uhw); if (ret) return ret; - memset(props, 0, sizeof(*props)); - props->fw_ver = hr_dev->caps.fw_ver; props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid); props->max_mr_size = (u64)(~(0ULL)); @@ -279,7 +277,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) props->device_cap_flags |= IB_DEVICE_XRC; - return ib_respond_empty_udata(uhw); + return 0; } static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num, diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 9d91f7667d4f..a70ef59a8064 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -391,6 +391,16 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) goto err_mmap_dbell; } + if (dev->lif_cfg.phc_state) { + ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0, + IONIC_MMAP_PHC, + &resp.phc_offset); + if (!ctx->mmap_phc) { + rc = -ENOMEM; + goto err_mmap_phc; + } + } + resp.page_shift = PAGE_SHIFT; resp.dbell_offset = db_phys & ~PAGE_MASK; @@ -421,6 +431,8 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) return 0; err_resp: + rdma_user_mmap_entry_remove(ctx->mmap_phc); +err_mmap_phc: rdma_user_mmap_entry_remove(ctx->mmap_dbell); err_mmap_dbell: ionic_put_dbid(dev, ctx->dbid); @@ -433,10 +445,26 @@ void ionic_dealloc_ucontext(struct ib_ucontext *ibctx) struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); struct ionic_ctx *ctx = to_ionic_ctx(ibctx); + rdma_user_mmap_entry_remove(ctx->mmap_phc); rdma_user_mmap_entry_remove(ctx->mmap_dbell); ionic_put_dbid(dev, ctx->dbid); } +static int ionic_mmap_phc_state(struct ionic_ibdev *dev, + struct vm_area_struct *vma) +{ + if (!(vma->vm_flags & VM_SHARED)) + return -EINVAL; + + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) + return -EPERM; + + vm_flags_clear(vma, VM_MAYWRITE); + + return vm_insert_page(vma, vma->vm_start, + virt_to_page(dev->lif_cfg.phc_state)); +} + int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma) { struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device); @@ -455,6 +483,12 @@ int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma) ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry, rdma_entry); + if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) { + rc = ionic_mmap_phc_state(dev, vma); + rdma_user_mmap_entry_put(rdma_entry); + return rc; + } + ibdev_dbg(&dev->ibdev, "writecombine? %d\n", ionic_entry->mmap_flags & IONIC_MMAP_WC); if (ionic_entry->mmap_flags & IONIC_MMAP_WC) @@ -487,6 +521,11 @@ int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); + int rc; + + rc = ib_no_udata_io(udata); + if (rc) + return rc; return ionic_get_pdid(dev, &pd->pdid); } @@ -495,6 +534,11 @@ int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device); struct ionic_pd *pd = to_ionic_pd(ibpd); + int rc; + + rc = ib_no_udata_io(udata); + if (rc) + return rc; ionic_put_pdid(dev, pd->pdid); @@ -741,6 +785,10 @@ int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, u32 flags = init_attr->flags; int rc; + rc = ib_is_udata_in_empty(udata); + if (rc) + return rc; + rc = ionic_get_ahid(dev, &ah->ahid); if (rc) return rc; @@ -877,6 +925,10 @@ struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, unsigned long pg_sz; int rc; + rc = ib_no_udata_io(udata); + if (rc) + return ERR_PTR(rc); + if (dmah) return ERR_PTR(-EOPNOTSUPP); @@ -1008,6 +1060,10 @@ int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) struct ionic_mr *mr = to_ionic_mr(ibmr); int rc; + rc = ib_no_udata_io(udata); + if (rc) + return rc; + if (!mr->ibmr.lkey) goto out; @@ -1120,6 +1176,10 @@ int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) struct ionic_mr *mr = to_ionic_mw(ibmw); int rc; + rc = ib_no_udata_io(udata); + if (rc) + return rc; + rc = ionic_get_mrid(dev, &mr->mrid); if (rc) return rc; @@ -1292,6 +1352,10 @@ int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct ionic_vcq *vcq = to_ionic_vcq(ibcq); int udma_idx, rc_tmp, rc = 0; + rc = ib_no_udata_io(udata); + if (rc) + return rc; + for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) { --udma_idx; @@ -2585,6 +2649,10 @@ int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ionic_qp *qp = to_ionic_qp(ibqp); int rc; + rc = ib_no_udata_io(udata); + if (rc) + return rc; + rc = ionic_check_modify_qp(qp, attr, mask); if (rc) return rc; @@ -2658,6 +2726,10 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) struct ionic_cq *cq; int rc; + rc = ib_no_udata_io(udata); + if (rc) + return rc; + rc = ionic_destroy_qp_cmd(dev, qp->qpid); if (rc) return rc; diff --git a/drivers/infiniband/hw/ionic/ionic_datapath.c b/drivers/infiniband/hw/ionic/ionic_datapath.c index aa2944887f23..3e2300f7ea10 100644 --- a/drivers/infiniband/hw/ionic/ionic_datapath.c +++ b/drivers/infiniband/hw/ionic/ionic_datapath.c @@ -32,6 +32,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) { struct ionic_rq_meta *meta; struct ionic_v1_wqe *wqe; + u64 wqe_idx; if (!qp->rq_flush) return 0; @@ -40,21 +41,22 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) return 0; wqe = ionic_queue_at_cons(&qp->rq); + wqe_idx = le64_to_cpu(wqe->base.wqe_idx); - /* wqe_id must be a valid queue index */ - if (unlikely(wqe->base.wqe_id >> qp->rq.depth_log2)) { + /* wqe_idx must be a valid queue index */ + if (unlikely(wqe_idx >> qp->rq.depth_log2)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu invalid\n", - qp->qpid, (unsigned long long)wqe->base.wqe_id); + qp->qpid, (unsigned long long)wqe_idx); return -EIO; } - /* wqe_id must indicate a request that is outstanding */ - meta = &qp->rq_meta[wqe->base.wqe_id]; + /* wqe_idx must indicate a request that is outstanding */ + meta = &qp->rq_meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu not posted\n", - qp->qpid, (unsigned long long)wqe->base.wqe_id); + qp->qpid, (unsigned long long)wqe_idx); return -EIO; } @@ -133,8 +135,8 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, { struct ionic_qp *qp = NULL; struct ionic_rq_meta *meta; + u16 vlan_tag, wqe_idx; u32 src_qpn, st_len; - u16 vlan_tag; u8 op; if (cqe_qp->rq_flush) @@ -144,7 +146,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, st_len = be32_to_cpu(cqe->status_length); - /* ignore wqe_id in case of flush error */ + /* ignore wqe_idx in case of flush error */ if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { cqe_qp->rq_flush = true; cq->flush = true; @@ -160,20 +162,19 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, return -EIO; } - /* wqe_id must be a valid queue index */ - if (unlikely(cqe->recv.wqe_id >> qp->rq.depth_log2)) { + wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK; + /* wqe_idx must be a valid queue index */ + if (unlikely(wqe_idx >> qp->rq.depth_log2)) { ibdev_warn(&dev->ibdev, - "qp %u recv index %llu invalid\n", - qp->qpid, (unsigned long long)cqe->recv.wqe_id); + "qp %u recv index %u invalid\n", qp->qpid, wqe_idx); return -EIO; } - /* wqe_id must indicate a request that is outstanding */ - meta = &qp->rq_meta[cqe->recv.wqe_id]; + /* wqe_idx must indicate a request that is outstanding */ + meta = &qp->rq_meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(&dev->ibdev, - "qp %u recv index %llu not posted\n", - qp->qpid, (unsigned long long)cqe->recv.wqe_id); + "qp %u recv index %u not posted\n", qp->qpid, wqe_idx); return -EIO; } @@ -408,7 +409,7 @@ static int ionic_comp_msn(struct ionic_qp *qp, struct ionic_v1_cqe *cqe) static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe) { struct ionic_sq_meta *meta; - u16 cqe_idx; + u16 wqe_idx; u32 st_len; if (qp->sq_flush) @@ -430,8 +431,8 @@ static int ionic_comp_npg(struct ionic_qp *qp, struct ionic_v1_cqe *cqe) return 0; } - cqe_idx = cqe->send.npg_wqe_id & qp->sq.mask; - meta = &qp->sq_meta[cqe_idx]; + wqe_idx = le64_to_cpu(cqe->send.npg_wqe_idx_timestamp) & qp->sq.mask; + meta = &qp->sq_meta[wqe_idx]; meta->local_comp = true; if (ionic_v1_cqe_error(cqe)) { @@ -811,7 +812,7 @@ static void ionic_prep_base(struct ionic_qp *qp, meta->signal = false; meta->local_comp = false; - wqe->base.wqe_id = qp->sq.prod; + wqe->base.wqe_idx = cpu_to_le64(qp->sq.prod); if (wr->send_flags & IB_SEND_FENCE) wqe->base.flags |= cpu_to_be16(IONIC_V1_FLAG_FENCE); @@ -1205,7 +1206,7 @@ static int ionic_prep_recv(struct ionic_qp *qp, meta->wrid = wr->wr_id; - wqe->base.wqe_id = meta - qp->rq_meta; + wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq_meta); wqe->base.num_sge_key = wr->num_sge; /* total length for recv goes in base imm_data_key */ diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h index adfbb89d856c..ee23062a1762 100644 --- a/drivers/infiniband/hw/ionic/ionic_fw.h +++ b/drivers/infiniband/hw/ionic/ionic_fw.h @@ -332,7 +332,7 @@ struct ionic_v1_cqe { __le16 old_rq_cq_cindex; } admin; struct { - __u64 wqe_id; + __le64 wqe_idx_timestamp; __be32 src_qpn_op; __u8 src_mac[6]; __be16 vlan_tag; @@ -342,13 +342,19 @@ struct ionic_v1_cqe { __u8 rsvd[4]; __be32 msg_msn; __u8 rsvd2[8]; - __u64 npg_wqe_id; + __le64 npg_wqe_idx_timestamp; } send; }; __be32 status_length; __be32 qid_type_flags; }; +/* bits for cqe wqe_idx and timestamp */ +enum ionic_v1_cqe_wqe_idx_timestamp_bits { + IONIC_V1_CQE_WQE_IDX_MASK = 0xffff, + IONIC_V1_CQE_TIMESTAMP_SHIFT = 16, +}; + /* bits for cqe recv */ enum ionic_v1_cqe_src_qpn_bits { IONIC_V1_CQE_RECV_QPN_MASK = 0xffffff, @@ -423,7 +429,7 @@ static inline u32 ionic_v1_cqe_qtf_qid(u32 qtf) /* v1 base wqe header */ struct ionic_v1_base_hdr { - __u64 wqe_id; + __le64 wqe_idx; __u8 op; __u8 num_sge_key; __be16 flags; diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index b0449c75f893..cba7809ec3d9 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -27,7 +27,7 @@ static int ionic_query_device(struct ib_device *ibdev, struct net_device *ndev; int err; - err = ib_is_udata_in_empty(udata); + err = ib_no_udata_io(udata); if (err) return err; @@ -74,7 +74,7 @@ static int ionic_query_device(struct ib_device *ibdev, attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; attr->max_pkeys = IONIC_PKEY_TBL_LEN; - return ib_respond_empty_udata(udata); + return 0; } static int ionic_query_port(struct ib_device *ibdev, u32 port, @@ -216,6 +216,7 @@ static const struct ib_device_ops ionic_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_IONIC, .uverbs_abi_ver = IONIC_ABI_VERSION, + .uverbs_robust_udata = true, .alloc_ucontext = ionic_alloc_ucontext, .dealloc_ucontext = ionic_dealloc_ucontext, diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 63828240d659..d7adaa161e6f 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -33,7 +33,6 @@ #define IONIC_MAX_QPID 0xffffff #define IONIC_SPEC_HIGH 8 #define IONIC_MAX_PD 1024 -#define IONIC_SPEC_HIGH 8 #define IONIC_SQCMB_ORDER 5 #define IONIC_RQCMB_ORDER 0 @@ -72,6 +71,7 @@ enum ionic_admin_flags { enum ionic_mmap_flag { IONIC_MMAP_WC = BIT(0), + IONIC_MMAP_PHC = BIT(1), }; struct ionic_mmap_entry { @@ -173,6 +173,7 @@ struct ionic_ctx { struct ib_ucontext ibctx; u32 dbid; struct rdma_user_mmap_entry *mmap_dbell; + struct rdma_user_mmap_entry *mmap_phc; }; struct ionic_tbl_buf { diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index f3cd281c3a2f..f827dce59973 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -40,6 +40,8 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); cfg->dbpage = lif->kern_dbpage; cfg->intr_ctrl = lif->ionic->idev.intr_ctrl; + if (lif->phc) + cfg->phc_state = lif->phc->state_page; cfg->db_phys = lif->ionic->bars[IONIC_PCI_BAR_DBELL].bus_addr; diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h index 20853429f623..2b29e646c193 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h @@ -23,6 +23,7 @@ struct ionic_lif_cfg { u64 __iomem *dbpage; struct ionic_intr __iomem *intr_ctrl; phys_addr_t db_phys; + void *phc_state; u64 page_size_supported; u32 npts_per_lif; diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.c b/drivers/infiniband/hw/irdma/i40iw_hw.c index 60c1f2b1811d..8301938b4543 100644 --- a/drivers/infiniband/hw/irdma/i40iw_hw.c +++ b/drivers/infiniband/hw/irdma/i40iw_hw.c @@ -29,7 +29,9 @@ static u32 i40iw_regs[IRDMA_MAX_REGS] = { I40E_PFHMC_PDINV, I40E_GLHMC_VFPDINV(0), I40E_GLPE_CRITERR, - 0xffffffff /* PFINT_RATEN not used in FPK */ + 0xffffffff, /* PFINT_RATEN not used in FPK */ + 0xffffffff, /* PFHMC_ERRORINFO not used in FPK */ + 0xffffffff /* PFHMC_ERRORDATA not used in FPK */ }; static u32 i40iw_stat_offsets[] = { diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.c b/drivers/infiniband/hw/irdma/icrdma_hw.c index 32f26284a788..b1f1b5485762 100644 --- a/drivers/infiniband/hw/irdma/icrdma_hw.c +++ b/drivers/infiniband/hw/irdma/icrdma_hw.c @@ -29,6 +29,8 @@ static u32 icrdma_regs[IRDMA_MAX_REGS] = { GLHMC_VFPDINV(0), GLPE_CRITERR, GLINT_RATE(0), + PFHMC_ERRORINFO, + PFHMC_ERRORDATA, }; static u64 icrdma_masks[IRDMA_MAX_MASKS] = { diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.h b/drivers/infiniband/hw/irdma/icrdma_hw.h index d97944ab45da..0acdeda1236d 100644 --- a/drivers/infiniband/hw/irdma/icrdma_hw.h +++ b/drivers/infiniband/hw/irdma/icrdma_hw.h @@ -40,6 +40,8 @@ #define GLHMC_VFPDINV(_i) (0x00528300 + ((_i) * 4)) /* _i=0...31 */ #define GLPE_CRITERR 0x00534000 #define GLINT_RATE(_INT) (0x0015A000 + ((_INT) * 4)) /* _i=0...2047 */ /* Reset Source: CORER */ +#define PFHMC_ERRORINFO 0x00520400 +#define PFHMC_ERRORDATA 0x00520500 #define ICRDMA_DB_ADDR_OFFSET (8 * 1024 * 1024 - 64 * 1024) diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c index 2172a2092e3f..4b451d8482a4 100644 --- a/drivers/infiniband/hw/irdma/icrdma_if.c +++ b/drivers/infiniband/hw/irdma/icrdma_if.c @@ -91,8 +91,12 @@ static void icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info, } } if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) { - ibdev_err(&iwdev->ibdev, "HMC Error\n"); - iwdev->rf->reset = true; + u32 hmc_errinfo = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_PFHMC_ERRORINFO]); + u32 hmc_errdata = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_PFHMC_ERRORDATA]); + + /* Log diagnostics; do not reset here. */ + ibdev_warn(&iwdev->ibdev, "HMC Error: errinfo=0x%08x errdata=0x%08x\n", + hmc_errinfo, hmc_errdata); } if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) { ibdev_err(&iwdev->ibdev, "PE Push Error\n"); diff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h index b5ce515f4ee8..d4530520e39f 100644 --- a/drivers/infiniband/hw/irdma/irdma.h +++ b/drivers/infiniband/hw/irdma/irdma.h @@ -66,6 +66,8 @@ enum irdma_registers { IRDMA_GLHMC_VFPDINV, IRDMA_GLPE_CRITERR, IRDMA_GLINT_RATE, + IRDMA_PFHMC_ERRORINFO, + IRDMA_PFHMC_ERRORDATA, IRDMA_MAX_REGS, /* Must be last entry */ }; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index e4037d5ef899..290ad02ed657 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -1168,6 +1168,12 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) iwqp->kqp.dma_mem.va = NULL; kfree(iwqp->kqp.sq_wrid_mem); kfree(iwqp->kqp.rq_wrid_mem); + + if (iwqp->user_mode && iwqp->iwpbl) { + struct irdma_mr *iwmr = iwqp->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } /** diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index b7388b41ed95..8e094e6568b3 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -18,11 +18,10 @@ static int irdma_query_device(struct ib_device *ibdev, struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; int err; - err = ib_is_udata_in_empty(udata); + err = ib_no_udata_io(udata); if (err) return err; - memset(props, 0, sizeof(*props)); addrconf_addr_eui48((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | @@ -76,7 +75,7 @@ static int irdma_query_device(struct ib_device *ibdev, if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_3) props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B; - return ib_respond_empty_udata(udata); + return 0; } /** @@ -407,6 +406,10 @@ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) u32 pd_id = 0; int err; + err = ib_is_udata_in_empty(udata); + if (err) + return err; + if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) return -EINVAL; @@ -445,6 +448,11 @@ static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); + int ret; + + ret = ib_no_udata_io(udata); + if (ret) + return ret; irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); @@ -464,6 +472,9 @@ static struct irdma_pbl *irdma_get_pbl(unsigned long va, list_for_each_entry (iwpbl, pbl_list, list) { if (iwpbl->user_base == va) { + struct irdma_mr *iwmr = iwpbl->iwmr; + + refcount_inc(&iwmr->user_ring_refs); list_del(&iwpbl->list); iwpbl->on_list = false; return iwpbl; @@ -534,11 +545,10 @@ static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, } /** - * irdma_destroy_qp - destroy qp + * _irdma_destroy_qp - destroy qp * @ibqp: qp's ib pointer also to get to device's qp address - * @udata: user data */ -static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +static void _irdma_destroy_qp(struct ib_qp *ibqp) { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; @@ -570,6 +580,22 @@ static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (iwqp->sc_qp.qp_uk.qp_id == 1) iwdev->rf->hwqp1_rsvd = false; irdma_free_qp_rsrc(iwqp); +} + +/** + * irdma_destroy_qp - destroy qp + * @ibqp: qp's ib pointer also to get to device's qp address + * @udata: user data + */ +static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) +{ + int ret; + + ret = ib_no_udata_io(udata); + if (ret) + return ret; + + _irdma_destroy_qp(ibqp); return 0; } @@ -605,37 +631,29 @@ static void irdma_setup_virt_qp(struct irdma_device *iwdev, /** * irdma_setup_umode_qp - setup sq and rq size in user mode qp - * @udata: udata + * @ucontext: user context + * @req: user request pointer * @iwdev: iwarp device * @iwqp: qp ptr (user or kernel) * @info: initialize info to return * @init_attr: Initial QP create attributes */ -static int irdma_setup_umode_qp(struct ib_udata *udata, +static int irdma_setup_umode_qp(struct irdma_ucontext *ucontext, + struct irdma_create_qp_req *req, struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *info, struct ib_qp_init_attr *init_attr) { - struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, - struct irdma_ucontext, ibucontext); struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; - struct irdma_create_qp_req req; unsigned long flags; int ret; - ret = ib_copy_from_udata(&req, udata, - min(sizeof(req), udata->inlen)); - if (ret) { - ibdev_dbg(&iwdev->ibdev, "VERBS: ib_copy_from_data fail\n"); - return ret; - } - - iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; + iwqp->ctx_info.qp_compl_ctx = req->user_compl_ctx; iwqp->user_mode = 1; spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, + iwqp->iwpbl = irdma_get_pbl((unsigned long)req->user_wqe_bufs, &ucontext->qp_reg_mem_list); spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); @@ -962,6 +980,7 @@ static int irdma_create_qp(struct ib_qp *ibqp, struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; struct irdma_qp_init_info init_info = {}; struct irdma_qp_host_ctx_info *ctx_info; + struct irdma_create_qp_req ureq = {}; struct irdma_srq *iwsrq; bool srq_valid = false; u32 srq_id = 0; @@ -979,9 +998,14 @@ static int irdma_create_qp(struct ib_qp *ibqp, if (err_code) return err_code; - if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || - udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) - return -EINVAL; + if (udata) { + if (udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN) + return -EINVAL; + + err_code = ib_copy_validate_udata_in(udata, ureq, user_compl_ctx); + if (err_code) + return err_code; + } init_info.vsi = &iwdev->vsi; init_info.qp_uk_init_info.uk_attrs = uk_attrs; @@ -1040,9 +1064,14 @@ static int irdma_create_qp(struct ib_qp *ibqp, init_waitqueue_head(&iwqp->mod_qp_waitq); if (udata) { + struct irdma_ucontext *ucontext = + rdma_udata_to_drv_context(udata, + struct irdma_ucontext, + ibucontext); + init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; - err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, - init_attr); + err_code = irdma_setup_umode_qp(ucontext, &ureq, iwdev, iwqp, + &init_info, init_attr); } else { INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; @@ -1109,8 +1138,12 @@ static int irdma_create_qp(struct ib_qp *ibqp, init_completion(&iwqp->free_qp); if (udata) { - /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ - if (udata->outlen < sizeof(uresp)) { + /* GEN_1 legacy support with libi40iw does not have expanded + * uresp struct. Check for the exact legacy size (20 bytes) to + * ensure that newer expanded uresp structs don't accidentally + * trigger the legacy fallback. + */ + if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) { uresp.lsmm = 1; uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; } else { @@ -1124,7 +1157,7 @@ static int irdma_create_qp(struct ib_qp *ibqp, err_code = ib_respond_udata(udata, uresp); if (err_code) { - irdma_destroy_qp(&iwqp->ibqp, udata); + _irdma_destroy_qp(&iwqp->ibqp); return err_code; } } @@ -1273,7 +1306,6 @@ static int irdma_wait_for_suspend(struct irdma_qp *iwqp) int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { -#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) struct irdma_pd *iwpd = to_iwpd(ibqp->pd); struct irdma_qp *iwqp = to_iwqp(ibqp); @@ -1289,15 +1321,26 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, u8 issue_modify_qp = 0; int ret = 0; + /* Clear the response buffer (if any). It may be updated again later. */ + ret = ib_respond_empty_udata(udata); + if (ret) + return ret; + ctx_info = &iwqp->ctx_info; roce_info = &iwqp->roce_info; udp_info = &iwqp->udp_info; if (udata) { /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ - if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || - (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) + if (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN) return -EINVAL; + + /* For current irdma, validate against ABI def. */ + if (udata->inlen) { + ret = ib_copy_validate_udata_in(udata, ureq, rsvd); + if (ret) + return ret; + } } if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) @@ -1540,10 +1583,6 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { - if (ib_copy_from_udata(&ureq, udata, - min(sizeof(ureq), udata->inlen))) - return -EINVAL; - irdma_flush_wqes(iwqp, (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | @@ -1633,7 +1672,6 @@ exit: int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { -#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; @@ -1649,11 +1687,20 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, int err; unsigned long flags; + err = ib_respond_empty_udata(udata); + if (err) + return err; + if (udata) { /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ - if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || - (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) + if (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN) return -EINVAL; + + if (udata->inlen) { + err = ib_copy_validate_udata_in(udata, ureq, rsvd); + if (err) + return err; + } } if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) @@ -1743,10 +1790,6 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { - if (ib_copy_from_udata(&ureq, udata, - min(sizeof(ureq), udata->inlen))) - return -EINVAL; - irdma_flush_wqes(iwqp, (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | @@ -1880,6 +1923,11 @@ static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); iwsrq->kmem.va = NULL; + } else { + /* Not called in any failure path, so iwpbl is valid. */ + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); } irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); @@ -1902,6 +1950,21 @@ static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) iwcq->kmem_shadow.size, iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); iwcq->kmem_shadow.va = NULL; + } else { + struct irdma_mr *iwmr; + + /* May be called in a failure path before iwpbl is valid. */ + if (iwcq->iwpbl) { + iwmr = iwcq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } + + if (iwcq->iwpbl_shadow) { + iwmr = iwcq->iwpbl_shadow->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); @@ -1958,6 +2021,11 @@ static int irdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) struct irdma_device *iwdev = to_iwdev(ibsrq->device); struct irdma_srq *iwsrq = to_iwsrq(ibsrq); struct irdma_sc_srq *srq = &iwsrq->sc_srq; + int ret; + + ret = ib_no_udata_io(udata); + if (ret) + return ret; irdma_srq_wq_destroy(iwdev->rf, srq); irdma_srq_free_rsrc(iwdev->rf, iwsrq); @@ -1978,6 +2046,11 @@ static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); unsigned long flags; + int ret; + + ret = ib_no_udata_io(udata); + if (ret) + return ret; spin_lock_irqsave(&iwcq->lock, flags); if (!list_empty(&iwcq->cmpl_generated)) @@ -2008,7 +2081,6 @@ static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, struct ib_udata *udata) { -#define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer) struct irdma_cq *iwcq = to_iwcq(ibcq); struct irdma_sc_dev *dev = iwcq->sc_cq.dev; struct irdma_cqp_request *cqp_request; @@ -2017,7 +2089,7 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, struct irdma_modify_cq_info info = {}; struct irdma_dma_mem kmem_buf; struct irdma_cq_mr *cqmr_buf; - struct irdma_pbl *iwpbl_buf; + struct irdma_pbl *iwpbl_buf = NULL; struct irdma_device *iwdev; struct irdma_pci_f *rf; struct irdma_cq_buf *cq_buf = NULL; @@ -2032,12 +2104,13 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, IRDMA_FEATURE_CQ_RESIZE)) return -EOPNOTSUPP; - if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN) - return -EINVAL; - if (entries > rf->max_cqe) return -EINVAL; + ret = ib_respond_empty_udata(udata); + if (ret) + return ret; + if (!iwcq->user_mode) { entries += 2; @@ -2064,9 +2137,9 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - if (ib_copy_from_udata(&req, udata, - min(sizeof(req), udata->inlen))) - return -EINVAL; + ret = ib_copy_validate_udata_in(udata, req, user_cq_buffer); + if (ret) + return ret; spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, @@ -2128,6 +2201,19 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, goto error; spin_lock_irqsave(&iwcq->lock, flags); + if (udata) { + struct irdma_pbl *old_iwpbl = iwcq->iwpbl; + + /* Only update if the resize was successful. Otherwise, HW is + * still pointing to the old PBL. + */ + iwcq->iwpbl = iwpbl_buf; + if (old_iwpbl) { + struct irdma_mr *old_iwmr = old_iwpbl->iwmr; + + refcount_dec(&old_iwmr->user_ring_refs); + } + } if (cq_buf) { cq_buf->kmem_buf = iwcq->kmem; cq_buf->hw = dev->hw; @@ -2143,6 +2229,11 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries, return 0; error: + if (iwpbl_buf) { + struct irdma_mr *iwmr = iwpbl_buf->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } if (!udata) { dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, kmem_buf.pa); @@ -2194,6 +2285,10 @@ static int irdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, struct cqp_cmds_info *cqp_info; int status; + status = ib_no_udata_io(udata); + if (status) + return status; + if (attr_mask & IB_SRQ_MAX_WR) return -EINVAL; @@ -2236,24 +2331,20 @@ static int irdma_setup_umode_srq(struct irdma_device *iwdev, struct irdma_srq_init_info *info, struct ib_udata *udata) { -#define IRDMA_CREATE_SRQ_MIN_REQ_LEN \ - offsetofend(struct irdma_create_srq_req, user_shadow_area) struct irdma_create_srq_req req = {}; struct irdma_ucontext *ucontext; struct irdma_srq_mr *srqmr; struct irdma_pbl *iwpbl; unsigned long flags; + int ret; iwsrq->user_mode = true; ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - if (udata->inlen < IRDMA_CREATE_SRQ_MIN_REQ_LEN) - return -EINVAL; - - if (ib_copy_from_udata(&req, udata, - min(sizeof(req), udata->inlen))) - return -EFAULT; + ret = ib_copy_validate_udata_in(udata, req, user_shadow_area); + if (ret) + return ret; spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); iwpbl = irdma_get_pbl((unsigned long)req.user_srq_buf, @@ -2316,6 +2407,7 @@ static int irdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *initattrs, struct ib_udata *udata) { +#define IRDMA_CREATE_SRQ_MIN_RESP_LEN offsetofend(struct irdma_create_srq_resp, srq_size) struct irdma_device *iwdev = to_iwdev(ibsrq->device); struct ib_srq_attr *attr = &initattrs->attr; struct irdma_pd *iwpd = to_iwpd(ibsrq->pd); @@ -2336,6 +2428,9 @@ static int irdma_create_srq(struct ib_srq *ibsrq, if (initattrs->srq_type != IB_SRQT_BASIC) return -EOPNOTSUPP; + if (udata && udata->outlen < IRDMA_CREATE_SRQ_MIN_RESP_LEN) + return -EINVAL; + if (!(uk_attrs->feature_flags & IRDMA_FEATURE_SRQ) || attr->max_sge > uk_attrs->max_hw_wq_frags) return -EINVAL; @@ -2419,6 +2514,11 @@ free_dmem: dma_free_coherent(rf->hw.device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); free_rsrc: + if (iwsrq->user_mode && iwsrq->iwpbl) { + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); return err_code; } @@ -2458,7 +2558,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { -#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) struct ib_udata *udata = &attrs->driver_udata; struct ib_device *ibdev = ibcq->device; @@ -2482,8 +2581,7 @@ static int irdma_create_cq(struct ib_cq *ibcq, if (err_code) return err_code; - if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || - udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) + if (udata && udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN) return -EINVAL; err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, @@ -2498,6 +2596,8 @@ static int irdma_create_cq(struct ib_cq *ibcq, INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); iwcq->cq_num = cq_num; + iwcq->iwpbl = NULL; + iwcq->iwpbl_shadow = NULL; info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; @@ -2517,49 +2617,50 @@ static int irdma_create_cq(struct ib_cq *ibcq, struct irdma_ucontext *ucontext; struct irdma_create_cq_req req = {}; struct irdma_cq_mr *cqmr; - struct irdma_pbl *iwpbl; - struct irdma_pbl *iwpbl_shadow; struct irdma_cq_mr *cqmr_shadow; iwcq->user_mode = true; ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - if (ib_copy_from_udata(&req, udata, - min(sizeof(req), udata->inlen))) { - err_code = -EFAULT; + /* Even though the last member of struct irdma_create_cq_req + * was always user_shadow_area, we need backwards compat with + * the legacy i40iw struct i40iw_ucreate_cq which stopped + * at user_cq_buffer. + */ + err_code = ib_copy_validate_udata_in(udata, req, user_cq_buf); + if (err_code) goto cq_free_rsrc; - } spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, - &ucontext->cq_reg_mem_list); + iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, + &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl) { + if (!iwcq->iwpbl) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr = &iwpbl->cq_mr; + cqmr = &iwcq->iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl_shadow = irdma_get_pbl( + iwcq->iwpbl_shadow = irdma_get_pbl( (unsigned long)req.user_shadow_area, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl_shadow) { + if (!iwcq->iwpbl_shadow) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr_shadow = &iwpbl_shadow->cq_mr; + cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; } else { info.shadow_area_pa = cqmr->shadow; } - if (iwpbl->pbl_allocated) { + if (iwcq->iwpbl->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr->cq_pbl.idx; @@ -3034,6 +3135,10 @@ static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) int err_code; u32 stag; + err_code = ib_no_udata_io(udata); + if (err_code) + return err_code; + stag = irdma_create_stag(iwdev); if (!stag) return -ENOMEM; @@ -3362,6 +3467,7 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, if (!iwmr) return ERR_PTR(-ENOMEM); + refcount_set(&iwmr->user_ring_refs, 1); iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->region = region; @@ -3508,7 +3614,6 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, struct ib_dmah *dmah, struct ib_udata *udata) { -#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_mem_reg_req req = {}; struct ib_umem *region = NULL; @@ -3518,10 +3623,15 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, if (dmah) return ERR_PTR(-EOPNOTSUPP); - if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) - return ERR_PTR(-EINVAL); + err = ib_copy_validate_udata_in(udata, req, sq_pages); + if (err) + return ERR_PTR(err); + + err = ib_respond_empty_udata(udata); + if (err) + return ERR_PTR(err); - if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) + if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) return ERR_PTR(-EINVAL); region = ib_umem_get_va(pd->device, start, len, access); @@ -3532,11 +3642,6 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, return (struct ib_mr *)region; } - if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { - ib_umem_release(region); - return ERR_PTR(-EFAULT); - } - iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); if (IS_ERR(iwmr)) { ib_umem_release(region); @@ -3784,6 +3889,10 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, struct ib_umem_dmabuf *umem_dmabuf; int ret; + ret = ib_no_udata_io(udata); + if (ret) + return ERR_PTR(ret); + if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) return ERR_PTR(-EINVAL); @@ -3929,41 +4038,41 @@ static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) * irdma_del_memlist - Deleting pbl list entries for CQ/QP * @iwmr: iwmr for IB's user page addresses * @ucontext: ptr to user context + * + * Return: True if the MR is currently in-use by a QP/CQ/SRQ ring. */ -static void irdma_del_memlist(struct irdma_mr *iwmr, +static bool irdma_del_memlist(struct irdma_mr *iwmr, struct irdma_ucontext *ucontext) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; unsigned long flags; + spinlock_t *lock; + bool in_use = false; switch (iwmr->type) { case IRDMA_MEMREG_TYPE_CQ: - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + lock = &ucontext->cq_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_QP: - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + lock = &ucontext->qp_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_SRQ: - spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); + lock = &ucontext->srq_reg_mem_list_lock; break; default: - break; + return false; } + + spin_lock_irqsave(lock, flags); + if (!refcount_dec_if_one(&iwmr->user_ring_refs)) { + in_use = true; + } else if (iwpbl->on_list) { + iwpbl->on_list = false; + list_del(&iwpbl->list); + } + spin_unlock_irqrestore(lock, flags); + + return in_use; } /** @@ -3979,6 +4088,10 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) bool dmabuf_revocable = iwmr->region && iwmr->region->is_dmabuf; int ret; + ret = ib_no_udata_io(udata); + if (ret) + return ret; + if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { if (iwmr->region) { struct irdma_ucontext *ucontext; @@ -3986,7 +4099,12 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - irdma_del_memlist(iwmr, ucontext); + + /* Do not allow the MR to be unpinned if it is still + * backing a user ring. + */ + if (irdma_del_memlist(iwmr, ucontext)) + return -EBUSY; } goto done; } @@ -5295,6 +5413,10 @@ static int irdma_create_user_ah(struct ib_ah *ibah, struct irdma_ah *parent_ah; int err; + err = ib_is_udata_in_empty(udata); + if (err) + return err; + if (udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) return -EINVAL; @@ -5346,6 +5468,10 @@ static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, struct irdma_device *iwdev = to_iwdev(ibah->pd->device); int err; + err = ib_no_udata_io(udata); + if (err) + return err; + err = irdma_setup_ah(ibah, attr); if (err) return err; @@ -5428,6 +5554,7 @@ static const struct ib_device_ops irdma_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_IRDMA, .uverbs_abi_ver = IRDMA_ABI_VER, + .uverbs_robust_udata = true, .alloc_hw_port_stats = irdma_alloc_hw_port_stats, .alloc_mr = irdma_alloc_mr, diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index 289ebc9b23ca..a1651641eb71 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -120,6 +120,7 @@ struct irdma_mr { u64 len; u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR]; struct irdma_pbl iwpbl; + refcount_t user_ring_refs; }; struct irdma_srq { @@ -152,6 +153,8 @@ struct irdma_cq { struct list_head resize_list; struct irdma_cq_poll_info cur_cqe; struct list_head cmpl_generated; + struct irdma_pbl *iwpbl; + struct irdma_pbl *iwpbl_shadow; }; struct irdma_cmpl_gen { diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index f2547989f422..d4e5e3f91268 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -27,7 +27,8 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, is_rnic_cq = mana_ib_is_rnic(mdev); if (udata) { - err = ib_copy_validate_udata_in(udata, ucmd, buf_addr); + err = ib_copy_validate_udata_in_cm(udata, ucmd, buf_addr, + MANA_IB_CREATE_RNIC_CQ); if (err) return err; @@ -105,6 +106,11 @@ int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct ib_device *ibdev = ibcq->device; struct mana_ib_dev *mdev; + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 9811570ab8f8..a8d19586a78a 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -15,6 +15,7 @@ static const struct ib_device_ops mana_ib_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_MANA, .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, + .uverbs_robust_udata = true, .add_gid = mana_ib_gd_add_gid, .alloc_mw = mana_ib_alloc_mw, diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index efe2935bda29..0bfb1883a186 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -101,6 +101,10 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct gdma_context *gc; int err; + err = ib_no_udata_io(udata); + if (err) + return err; + dev = container_of(ibdev, struct mana_ib_dev, ib_dev); gc = mdev_to_gc(dev); @@ -133,6 +137,11 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct gdma_destroy_pd_req req = {}; struct mana_ib_dev *dev; struct gdma_context *gc; + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; dev = container_of(ibdev, struct mana_ib_dev, ib_dev); gc = mdev_to_gc(dev); @@ -142,7 +151,11 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) req.pd_handle = pd->pd_handle; - return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) + return err; + + return 0; } static int mana_gd_destroy_doorbell_page(struct gdma_context *gc, @@ -198,17 +211,17 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, int doorbell_page; int ret; + ret = ib_no_udata_io(udata); + if (ret) + return ret; + mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); gc = mdev_to_gc(mdev); /* Allocate a doorbell page index */ ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); - if (ret) { - ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret); + if (ret) return ret; - } - - ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page); ucontext->doorbell = doorbell_page; @@ -575,11 +588,10 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)->dev); int err; - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; - memset(props, 0, sizeof(*props)); props->vendor_id = pdev->vendor; props->vendor_part_id = dev->gdma_dev->dev_id.type; props->max_mr_size = MANA_IB_MAX_MR_SIZE; @@ -605,7 +617,7 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, if (!mana_ib_is_rnic(dev)) props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM; - return ib_respond_empty_udata(uhw); + return 0; } int mana_ib_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/mr.c index 030bfdcfff3c..1233685b4b89 100644 --- a/drivers/infiniband/hw/mana/mr.c +++ b/drivers/infiniband/hw/mana/mr.c @@ -113,6 +113,10 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length, if (dmah) return ERR_PTR(-EOPNOTSUPP); + err = ib_no_udata_io(udata); + if (err) + return ERR_PTR(err); + dev = container_of(ibdev, struct mana_ib_dev, ib_dev); ibdev_dbg(ibdev, @@ -327,6 +331,11 @@ int mana_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) { struct mana_ib_dev *mdev = container_of(ibmw->device, struct mana_ib_dev, ib_dev); struct mana_ib_pd *pd = container_of(ibmw->pd, struct mana_ib_pd, ibpd); + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; return mana_ib_gd_create_mw(mdev, pd, ibmw); } @@ -346,6 +355,10 @@ int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) struct mana_ib_dev *dev; int err; + err = ib_no_udata_io(udata); + if (err) + return err; + dev = container_of(ibdev, struct mana_ib_dev, ib_dev); err = mana_ib_gd_destroy_mr(dev, mr->mr_handle); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 60926f39ab9d..c52f3ec14032 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -472,6 +472,12 @@ static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); } +static void mana_table_drain_qp_ref(struct mana_ib_qp *qp) +{ + mana_put_qp_ref(qp); + wait_for_completion(&qp->free); +} + static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) { u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; @@ -490,6 +496,7 @@ static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *q remove_sq: xa_erase_irq(&mdev->qp_table_wq, qids); + mana_table_drain_qp_ref(qp); return err; } @@ -537,8 +544,7 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, qp->ibqp.qp_type); return; } - mana_put_qp_ref(qp); - wait_for_completion(&qp->free); + mana_table_drain_qp_ref(qp); } static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, @@ -651,10 +657,8 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, u32 doorbell, queue_size; int i, err; - if (udata) { - ibdev_dbg(&mdev->ib_dev, "User-level UD QPs are not supported\n"); + if (udata) return -EOPNOTSUPP; - } for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) { queue_size = mana_ib_queue_size(attr, i); @@ -745,6 +749,11 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct gdma_context *gc = mdev_to_gc(mdev); struct mana_port_context *mpc; struct net_device *ndev; + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp)); @@ -797,7 +806,11 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, req.ah_attr.flow_label = attr->ah_attr.grh.flow_label; } - return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) + return err; + + return 0; } int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, @@ -826,7 +839,11 @@ static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, struct mana_ib_pd *pd; struct mana_ib_wq *wq; struct ib_wq *ibwq; - int i; + int i, err; + + err = ib_no_udata_io(udata); + if (err) + return err; ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); mpc = netdev_priv(ndev); @@ -872,6 +889,11 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata) struct mana_port_context *mpc; struct net_device *ndev; struct mana_ib_pd *pd; + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); mpc = netdev_priv(ndev); @@ -890,7 +912,11 @@ static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata) { struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); - int i; + int i, err; + + err = ib_no_udata_io(udata); + if (err) + return err; mana_table_remove_qp(mdev, qp); @@ -908,7 +934,11 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata) { struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); - int i; + int i, err; + + err = ib_no_udata_io(udata); + if (err) + return err; mana_remove_qp_from_cqs(qp); mana_table_remove_qp(mdev, qp); diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c index 5c2134a0b1a1..6b066d605dcb 100644 --- a/drivers/infiniband/hw/mana/wq.c +++ b/drivers/infiniband/hw/mana/wq.c @@ -55,6 +55,11 @@ int mana_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) struct mana_ib_wq *wq = container_of(ibwq, struct mana_ib_wq, ibwq); struct ib_device *ib_dev = ibwq->device; struct mana_ib_dev *mdev; + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; mdev = container_of(ib_dev, struct mana_ib_dev, ib_dev); @@ -69,10 +74,17 @@ int mana_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table, struct ib_rwq_ind_table_init_attr *init_attr, struct ib_udata *udata) { + int err; + + err = ib_no_udata_io(udata); + if (err) + return err; + /* * There is no additional data in ind_table to be maintained by this * driver, do nothing */ + return 0; } diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c index 63a868a3822f..1e5e525fea73 100644 --- a/drivers/infiniband/hw/mlx4/cm.c +++ b/drivers/infiniband/hw/mlx4/cm.c @@ -40,6 +40,7 @@ #include "mlx4_ib.h" #define CM_CLEANUP_CACHE_TIMEOUT (30 * HZ) +#define CM_RTU_TIMEOUT (60 * HZ) struct id_map_entry { struct rb_node node; @@ -48,6 +49,7 @@ struct id_map_entry { u32 pv_cm_id; int slave_id; int scheduled_delete; + bool rtu_timeout; struct mlx4_ib_dev *dev; struct list_head list; @@ -184,6 +186,10 @@ static void id_map_ent_timeout(struct work_struct *work) struct rb_root *sl_id_map = &sriov->sl_id_map; spin_lock(&sriov->id_map_lock); + if (!ent->scheduled_delete) { + spin_unlock(&sriov->id_map_lock); + return; + } if (!xa_erase(&sriov->pv_id_table, ent->pv_cm_id)) goto out; found_ent = id_map_find_by_sl_id(&dev->ib_dev, ent->slave_id, ent->sl_cm_id); @@ -228,8 +234,12 @@ static void sl_id_map_add(struct ib_device *ibdev, struct id_map_entry *new) rb_insert_color(&new->node, sl_id_map); } +static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id, + unsigned long timeout, bool rtu_timeout); + static struct id_map_entry * -id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id) +id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id, + u32 *pv_cm_id) { int ret; struct id_map_entry *ent; @@ -242,6 +252,7 @@ id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id) ent->sl_cm_id = sl_cm_id; ent->slave_id = slave_id; ent->scheduled_delete = 0; + ent->rtu_timeout = false; ent->dev = to_mdev(ibdev); INIT_DELAYED_WORK(&ent->timeout, id_map_ent_timeout); @@ -251,6 +262,8 @@ id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id) spin_lock(&sriov->id_map_lock); sl_id_map_add(ibdev, ent); list_add_tail(&ent->list, &sriov->cm_list); + *pv_cm_id = ent->pv_cm_id; + schedule_delayed(ibdev, ent, CM_RTU_TIMEOUT, true); spin_unlock(&sriov->id_map_lock); return ent; } @@ -267,42 +280,41 @@ id_map_get(struct ib_device *ibdev, int *pv_cm_id, int slave_id, int sl_cm_id) struct id_map_entry *ent; struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; - spin_lock(&sriov->id_map_lock); + lockdep_assert_held(&sriov->id_map_lock); if (*pv_cm_id == -1) { ent = id_map_find_by_sl_id(ibdev, slave_id, sl_cm_id); if (ent) *pv_cm_id = (int) ent->pv_cm_id; } else ent = xa_load(&sriov->pv_id_table, *pv_cm_id); - spin_unlock(&sriov->id_map_lock); return ent; } -static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id) +static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id, + unsigned long timeout, bool rtu_timeout) { struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; unsigned long flags; - spin_lock(&sriov->id_map_lock); + lockdep_assert_held(&sriov->id_map_lock); spin_lock_irqsave(&sriov->going_down_lock, flags); /*make sure that there is no schedule inside the scheduled work.*/ - if (!sriov->is_going_down && !id->scheduled_delete) { + if (!sriov->is_going_down || id->scheduled_delete) { id->scheduled_delete = 1; - queue_delayed_work(cm_wq, &id->timeout, CM_CLEANUP_CACHE_TIMEOUT); - } else if (id->scheduled_delete) { - /* Adjust timeout if already scheduled */ - mod_delayed_work(cm_wq, &id->timeout, CM_CLEANUP_CACHE_TIMEOUT); + id->rtu_timeout = rtu_timeout; + mod_delayed_work(cm_wq, &id->timeout, timeout); } spin_unlock_irqrestore(&sriov->going_down_lock, flags); - spin_unlock(&sriov->id_map_lock); } #define REJ_REASON(m) be16_to_cpu(((struct cm_generic_msg *)(m))->rej_reason) int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id, struct ib_mad *mad) { + struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; struct id_map_entry *id; + u32 pv_cm_id_to_set = 0; u32 sl_cm_id; int pv_cm_id = -1; @@ -312,21 +324,62 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id mad->mad_hdr.attr_id == CM_SIDR_REQ_ATTR_ID || (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID && REJ_REASON(mad) == IB_CM_REJ_TIMEOUT)) { sl_cm_id = get_local_comm_id(mad); + spin_lock(&sriov->id_map_lock); id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); + if (id) { + pv_cm_id_to_set = id->pv_cm_id; + if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) + schedule_delayed(ibdev, id, + CM_CLEANUP_CACHE_TIMEOUT, + false); + } + spin_unlock(&sriov->id_map_lock); if (id) goto cont; - id = id_map_alloc(ibdev, slave_id, sl_cm_id); + if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) + return 0; + id = id_map_alloc(ibdev, slave_id, sl_cm_id, + &pv_cm_id_to_set); if (IS_ERR(id)) { mlx4_ib_warn(ibdev, "%s: id{slave: %d, sl_cm_id: 0x%x} Failed to id_map_alloc\n", __func__, slave_id, sl_cm_id); return PTR_ERR(id); } - } else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID || - mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { + } else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) { + sl_cm_id = get_local_comm_id(mad); + spin_lock(&sriov->id_map_lock); + id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); + if (id) { + pv_cm_id_to_set = id->pv_cm_id; + schedule_delayed(ibdev, id, CM_CLEANUP_CACHE_TIMEOUT, + false); + } + spin_unlock(&sriov->id_map_lock); + if (!id) + return 0; + } else if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) { return 0; } else { sl_cm_id = get_local_comm_id(mad); + spin_lock(&sriov->id_map_lock); id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id); + if (id) { + if (mad->mad_hdr.attr_id == CM_RTU_ATTR_ID && + id->rtu_timeout) { + id->rtu_timeout = false; + if (cancel_delayed_work(&id->timeout)) + id->scheduled_delete = 0; + else + id = NULL; + } + if (id) + pv_cm_id_to_set = id->pv_cm_id; + if (id && mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) + schedule_delayed(ibdev, id, + CM_CLEANUP_CACHE_TIMEOUT, + false); + } + spin_unlock(&sriov->id_map_lock); } if (!id) { @@ -336,10 +389,7 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id } cont: - set_local_comm_id(mad, id->pv_cm_id); - - if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID) - schedule_delayed(ibdev, id); + set_local_comm_id(mad, pv_cm_id_to_set); return 0; } @@ -429,7 +479,10 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov; u32 rem_pv_cm_id = get_local_comm_id(mad); u32 pv_cm_id; + u32 sl_cm_id = 0; struct id_map_entry *id; + int pv_cm_id_int; + int slave_id = 0; int sts; if (mad->mad_hdr.attr_id == CM_REQ_ATTR_ID || @@ -457,7 +510,28 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, } pv_cm_id = get_remote_comm_id(mad); - id = id_map_get(ibdev, (int *)&pv_cm_id, -1, -1); + pv_cm_id_int = pv_cm_id; + spin_lock(&sriov->id_map_lock); + id = id_map_get(ibdev, &pv_cm_id_int, -1, -1); + if (id) { + if (mad->mad_hdr.attr_id == CM_RTU_ATTR_ID && + id->rtu_timeout) { + id->rtu_timeout = false; + if (cancel_delayed_work(&id->timeout)) + id->scheduled_delete = 0; + else + id = NULL; + } + if (id && slave) + slave_id = id->slave_id; + if (id) + sl_cm_id = id->sl_cm_id; + if (id && (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID || + mad->mad_hdr.attr_id == CM_REJ_ATTR_ID)) + schedule_delayed(ibdev, id, + CM_CLEANUP_CACHE_TIMEOUT, false); + } + spin_unlock(&sriov->id_map_lock); if (!id) { if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID && @@ -472,12 +546,8 @@ int mlx4_ib_demux_cm_handler(struct ib_device *ibdev, int port, int *slave, } if (slave) - *slave = id->slave_id; - set_remote_comm_id(mad, id->sl_cm_id); - - if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID || - mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) - schedule_delayed(ibdev, id); + *slave = slave_id; + set_remote_comm_id(mad, sl_cm_id); return 0; } diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 17073e8f105a..7266a6141944 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -471,8 +471,6 @@ static int mlx4_ib_query_device(struct ib_device *ibdev, if (err) goto out; - memset(props, 0, sizeof *props); - have_ib_ports = num_ib_ports(dev->dev); props->fw_ver = dev->dev->caps.fw_ver; diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 761e2c05dd0f..a6b4abce1bfe 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -313,7 +313,7 @@ mlx4_alloc_priv_pages(struct ib_device *device, MLX4_MR_PAGES_ALIGN); /* Prevent cross page boundary allocation. */ - mr->pages = (__be64 *)get_zeroed_page(GFP_KERNEL); + mr->pages = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!mr->pages) return -ENOMEM; @@ -328,7 +328,7 @@ mlx4_alloc_priv_pages(struct ib_device *device, return 0; err: - free_page((unsigned long)mr->pages); + kfree(mr->pages); return ret; } @@ -340,7 +340,7 @@ mlx4_free_priv_pages(struct mlx4_ib_mr *mr) dma_unmap_single(device->dev.parent, mr->page_map, mr->page_map_size, DMA_TO_DEVICE); - free_page((unsigned long)mr->pages); + kfree(mr->pages); mr->pages = NULL; } } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 4ff6cca7e581..711abd650ecd 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -969,7 +969,6 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, if (err) return err; - memset(props, 0, sizeof(*props)); err = mlx5_query_system_image_guid(ibdev, &props->sys_image_guid); if (err) @@ -5501,13 +5500,13 @@ static int __init mlx5_ib_init(void) { int ret; - xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL); + xlt_emergency_page = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!xlt_emergency_page) return -ENOMEM; mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0); if (!mlx5_ib_event_wq) { - free_page((unsigned long)xlt_emergency_page); + kfree(xlt_emergency_page); return -ENOMEM; } @@ -5542,7 +5541,7 @@ rep_err: mlx5_ib_qp_event_cleanup(); qp_event_err: destroy_workqueue(mlx5_ib_event_wq); - free_page((unsigned long)xlt_emergency_page); + kfree(xlt_emergency_page); return ret; } @@ -5556,7 +5555,7 @@ static void __exit mlx5_ib_cleanup(void) mlx5_ib_qp_event_cleanup(); destroy_workqueue(mlx5_ib_event_wq); - free_page((unsigned long)xlt_emergency_page); + kfree(xlt_emergency_page); } module_init(mlx5_ib_init); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index e6b74955d95d..00e13028762a 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -339,7 +339,7 @@ static int mlx5r_build_frmr_key(struct ib_device *device, return 0; } -static struct ib_frmr_pool_ops mlx5r_frmr_pool_ops = { +static const struct ib_frmr_pool_ops mlx5r_frmr_pool_ops = { .create_frmrs = mlx5r_create_mkeys, .destroy_frmrs = mlx5r_destroy_mkeys, .build_key = mlx5r_build_frmr_key, @@ -898,8 +898,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) ib_umem_dmabuf_unmap_pages(umem_dmabuf); } -static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = { - .allow_peer2peer = 1, +static const struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = { + .allow_peer2peer = true, .invalidate_mappings = mlx5_ib_dmabuf_invalidate_cb, }; diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1badec9bf527..b8618610737a 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -37,6 +37,7 @@ #include <linux/hmm.h> #include <linux/hmm-dma.h> #include <linux/pci-p2pdma.h> +#include <linux/slab.h> #include "mlx5_ib.h" #include "cmd.h" @@ -1414,7 +1415,8 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, goto resolve_page_fault; } - wqe_start = (void *)__get_free_page(GFP_KERNEL); + /* TODO: switch to "fast and as large as possible" allocation helper */ + wqe_start = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!wqe_start) { mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); goto resolve_page_fault; @@ -1475,7 +1477,7 @@ resolve_page_fault: pfault->wqe.wq_num, resume_with_error, pfault->type); mlx5_core_res_put(res); - free_page((unsigned long)wqe_start); + kfree(wqe_start); } static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 48cae5cc1c1b..80d0d190b26c 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */ +#include <linux/slab.h> #include <rdma/ib_umem_odp.h> #include <rdma/iter.h> #include "mlx5_ib.h" @@ -517,22 +518,20 @@ static void *mlx5r_umr_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask) size = min_t(size_t, ent_size * ALIGN(*nents, xlt_chunk_align), MLX5_MAX_UMR_CHUNK); *nents = size / ent_size; - res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN, - get_order(size)); + res = kmalloc(size, gfp_mask | __GFP_NOWARN); if (res) return res; if (size > MLX5_SPARE_UMR_CHUNK) { size = MLX5_SPARE_UMR_CHUNK; *nents = size / ent_size; - res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN, - get_order(size)); + res = kmalloc(size, gfp_mask | __GFP_NOWARN); if (res) return res; } *nents = PAGE_SIZE / ent_size; - res = (void *)__get_free_page(gfp_mask); + res = kmalloc(PAGE_SIZE, gfp_mask); if (res) return res; @@ -548,7 +547,7 @@ static void mlx5r_umr_free_xlt(void *xlt, size_t length) return; } - free_pages((unsigned long)xlt, get_order(length)); + kfree(xlt); } static void mlx5r_umr_unmap_free_xlt(struct mlx5_ib_dev *dev, void *xlt, @@ -978,7 +977,7 @@ static inline int _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, start_block, nblocks); } -/** +/* * This function makes an mkey non-present by zapping the translation entries of * the mkey by zapping (zeroing out) the first N entries, where N is determined * by the largest page size supported by the device and the MR length. diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c index dedc301235a0..117a070e784e 100644 --- a/drivers/infiniband/hw/mthca/mthca_allocator.c +++ b/drivers/infiniband/hw/mthca/mthca_allocator.c @@ -126,7 +126,7 @@ int mthca_array_set(struct mthca_array *array, int index, void *value) /* Allocate with GFP_ATOMIC because we'll be called with locks held. */ if (!array->page_list[p].page) - array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC); + array->page_list[p].page = kzalloc(PAGE_SIZE, GFP_ATOMIC); if (!array->page_list[p].page) return -ENOMEM; @@ -142,7 +142,7 @@ void mthca_array_clear(struct mthca_array *array, int index) int p = (index * sizeof (void *)) >> PAGE_SHIFT; if (--array->page_list[p].used == 0) { - free_page((unsigned long) array->page_list[p].page); + kfree(array->page_list[p].page); array->page_list[p].page = NULL; } else array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL; @@ -174,7 +174,7 @@ void mthca_array_cleanup(struct mthca_array *array, int nent) int i; for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i) - free_page((unsigned long) array->page_list[i].page); + kfree(array->page_list[i].page); kfree(array->page_list); } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index f90f67afc8fa..8989e572e32c 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -58,7 +58,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr int err; struct mthca_dev *mdev = to_mdev(ibdev); - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; @@ -69,8 +69,6 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr goto out; } - memset(props, 0, sizeof *props); - props->fw_ver = mdev->fw_ver; ib_init_query_mad(in_mad); @@ -114,7 +112,6 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * props->max_mcast_grp; - err = ib_respond_empty_udata(uhw); out: kfree(in_mad); kfree(out_mad); @@ -895,7 +892,8 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, goto err_umem; } - pages = (u64 *) __get_free_page(GFP_KERNEL); + /* TODO: switch to "fast and as large as possible" allocation helper */ + pages = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!pages) { err = -ENOMEM; goto err_mtt; @@ -924,7 +922,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (i) err = mthca_write_mtt(dev, mr->mtt, n, pages, i); mtt_done: - free_page((unsigned long) pages); + kfree(pages); if (err) goto err_mtt; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 53ba32d168a1..cfe3d19b73b3 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -70,11 +70,10 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); int err; - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; - memset(attr, 0, sizeof *attr); memcpy(&attr->fw_ver, &dev->attr.fw_ver[0], min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver))); addrconf_addr_eui48((u8 *)&attr->sys_image_guid, @@ -112,7 +111,7 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr; attr->max_pkeys = 1; - return ib_respond_empty_udata(uhw); + return 0; } static inline void get_link_speed_and_width(struct ocrdma_dev *dev, diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index c90a1b5c8ee7..012a0ab98d6b 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -114,12 +114,10 @@ int qedr_query_device(struct ib_device *ibdev, return -EINVAL; } - rc = ib_is_udata_in_empty(udata); + rc = ib_no_udata_io(udata); if (rc) return rc; - memset(attr, 0, sizeof(*attr)); - attr->fw_ver = qattr->fw_ver; attr->sys_image_guid = qattr->sys_image_guid; attr->max_mr_size = qattr->max_mr_size; @@ -148,7 +146,7 @@ int qedr_query_device(struct ib_device *ibdev, attr->max_qp_init_rd_atom = 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); attr->max_qp_rd_atom = - min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), + min(1U << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), attr->max_qp_init_rd_atom); attr->max_srq = qattr->max_srq; @@ -160,7 +158,7 @@ int qedr_query_device(struct ib_device *ibdev, attr->max_pkeys = qattr->max_pkey; attr->max_ah = qattr->max_ah; - return ib_respond_empty_udata(udata); + return 0; } static inline void get_link_speed_and_width(int speed, u16 *ib_speed, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index dc355b00f61c..1a1647d0e345 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -278,13 +278,12 @@ int usnic_ib_query_device(struct ib_device *ibdev, int err; usnic_dbg("\n"); - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; mutex_lock(&us_ibdev->usdev_lock); us_ibdev->netdev->ethtool_ops->get_drvinfo(us_ibdev->netdev, &info); - memset(props, 0, sizeof(*props)); usnic_mac_ip_to_gid(us_ibdev->ufdev->mac, us_ibdev->ufdev->inaddr, &gid.raw[0]); memcpy(&props->sys_image_guid, &gid.global.interface_id, @@ -324,7 +323,7 @@ int usnic_ib_query_device(struct ib_device *ibdev, * max_qp_wr, max_sge, max_sge_rd, max_cqe */ mutex_unlock(&us_ibdev->usdev_lock); - return ib_respond_empty_udata(uhw); + return 0; } int usnic_ib_query_port(struct ib_device *ibdev, u32 port, diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c index 691c64a73516..201c35039a74 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom.c +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c @@ -114,7 +114,7 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, INIT_LIST_HEAD(chunk_list); - page_list = (struct page **) __get_free_page(GFP_KERNEL); + page_list = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page_list) return -ENOMEM; @@ -182,7 +182,7 @@ out: mmgrab(uiomr->owning_mm); mmap_read_unlock(mm); - free_page((unsigned long) page_list); + kfree(page_list); return ret; } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index 1d29a535f76a..59c2a88c158a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -69,7 +69,7 @@ int pvrdma_query_device(struct ib_device *ibdev, struct pvrdma_dev *dev = to_vdev(ibdev); int err; - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; @@ -116,7 +116,7 @@ int pvrdma_query_device(struct ib_device *ibdev, props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_RC_RNR_NAK_GEN; - return ib_respond_empty_udata(uhw); + return 0; } /** diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index 70e7d08fdce6..c40cce69e945 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -263,7 +263,7 @@ static inline bool wss_exceeds_threshold(struct rvt_wss *wss) static void get_map_page(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map) { - unsigned long page = get_zeroed_page(GFP_KERNEL); + void *page = kzalloc(PAGE_SIZE, GFP_KERNEL); /* * Free the page if someone raced with us installing it. @@ -271,9 +271,9 @@ static void get_map_page(struct rvt_qpn_table *qpt, spin_lock(&qpt->lock); if (map->page) - free_page(page); + kfree(page); else - map->page = (void *)page; + map->page = page; spin_unlock(&qpt->lock); } @@ -343,7 +343,7 @@ static void free_qpn_table(struct rvt_qpn_table *qpt) int i; for (i = 0; i < ARRAY_SIZE(qpt->map); i++) - free_page((unsigned long)qpt->map[i].page); + kfree(qpt->map[i].page); } /** diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index 5fa3a1f33326..1c112f4dc994 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -55,8 +55,10 @@ struct rvt_dev_info *rvt_alloc_device(size_t size, int nports) return rdi; rdi->ports = kzalloc_objs(*rdi->ports, nports); - if (!rdi->ports) + if (!rdi->ports) { ib_dealloc_device(&rdi->ibdev); + return NULL; + } return rdi; } @@ -82,14 +84,14 @@ static int rvt_query_device(struct ib_device *ibdev, struct rvt_dev_info *rdi = ib_to_rvt(ibdev); int err; - err = ib_is_udata_in_empty(uhw); + err = ib_no_udata_io(uhw); if (err) return err; /* * Return rvt_dev_info.dparms.props contents */ *props = rdi->dparms.props; - return ib_respond_empty_udata(uhw); + return 0; } static int rvt_get_numa_node(struct ib_device *ibdev) diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c index 379e65bfcd49..bddb7a257831 100644 --- a/drivers/infiniband/sw/rxe/rxe_mw.c +++ b/drivers/infiniband/sw/rxe/rxe_mw.c @@ -72,13 +72,6 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, return -EINVAL; } - /* C10-72 */ - if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) { - rxe_dbg_mw(mw, - "attempt to bind type 2 MW with qp with different PD\n"); - return -EINVAL; - } - /* o10-37.2.40 */ if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) { rxe_dbg_mw(mw, @@ -87,10 +80,21 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, } } - /* remaining checks only apply to a nonzero MR */ + /* C10-72 */ + if (unlikely(qp->pd != rxe_mw_pd(mw))) { + rxe_dbg_mw(mw, "attempt to bind MW with qp with different PD\n"); + return -EINVAL; + } + if (!mr) return 0; + /* remaining checks only apply to a nonzero MR */ + if (unlikely(qp->pd != mr_pd(mr))) { + rxe_dbg_mw(mw, "attempt to bind MW/QP to MR with different PD\n"); + return -EINVAL; + } + if (unlikely(mr->access & IB_ZERO_BASED)) { rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n"); return -EINVAL; diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index ff904d5e54a7..c189c033175d 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -87,14 +87,9 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, rxe_mr_init(access_flags, mr); - if (!start && length == U64_MAX) { - if (iova != 0) - return -EINVAL; - if (!(rxe->attr.odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) - return -EINVAL; - - /* Never reach here, for implicit ODP is not implemented. */ - } + /* Implicit ODP (start=0, length=U64_MAX) is not implemented. */ + if (!start && length == U64_MAX) + return -EOPNOTSUPP; umem_odp = ib_umem_odp_get(&rxe->ib_dev, start, length, access_flags, &rxe_mn_ops); diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index f3dff1aea96a..ec2cfb12559a 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -67,27 +67,27 @@ static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap, int has_srq) { if (cap->max_send_wr > rxe->attr.max_qp_wr) { - rxe_dbg_dev(rxe, "invalid send wr = %u > %d\n", - cap->max_send_wr, rxe->attr.max_qp_wr); + rxe_dbg_dev(rxe, "invalid send wr = %u > %u\n", + cap->max_send_wr, rxe->attr.max_qp_wr); goto err1; } if (cap->max_send_sge > rxe->attr.max_send_sge) { - rxe_dbg_dev(rxe, "invalid send sge = %u > %d\n", - cap->max_send_sge, rxe->attr.max_send_sge); + rxe_dbg_dev(rxe, "invalid send sge = %u > %u\n", + cap->max_send_sge, rxe->attr.max_send_sge); goto err1; } if (!has_srq) { if (cap->max_recv_wr > rxe->attr.max_qp_wr) { - rxe_dbg_dev(rxe, "invalid recv wr = %u > %d\n", - cap->max_recv_wr, rxe->attr.max_qp_wr); + rxe_dbg_dev(rxe, "invalid recv wr = %u > %u\n", + cap->max_recv_wr, rxe->attr.max_qp_wr); goto err1; } if (cap->max_recv_sge > rxe->attr.max_recv_sge) { - rxe_dbg_dev(rxe, "invalid recv sge = %u > %d\n", - cap->max_recv_sge, rxe->attr.max_recv_sge); + rxe_dbg_dev(rxe, "invalid recv sge = %u > %u\n", + cap->max_recv_sge, rxe->attr.max_recv_sge); goto err1; } } @@ -172,6 +172,7 @@ static void free_rd_atomic_resources(struct rxe_qp *qp) } kfree(qp->resp.resources); qp->resp.resources = NULL; + qp->resp.res = NULL; } } @@ -537,9 +538,9 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, if (mask & IB_QP_MAX_QP_RD_ATOMIC) { if (attr->max_rd_atomic > rxe->attr.max_qp_rd_atom) { - rxe_dbg_qp(qp, "invalid max_rd_atomic %d > %d\n", - attr->max_rd_atomic, - rxe->attr.max_qp_rd_atom); + rxe_dbg_qp(qp, "invalid max_rd_atomic %u > %u\n", + attr->max_rd_atomic, + rxe->attr.max_qp_rd_atom); goto err1; } } @@ -709,11 +710,23 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, qp->attr.max_dest_rd_atomic = max_dest_rd_atomic; + /* + * Not gated by IB_QP_STATE, so the responder task is live. + * Quiesce recv_task like rxe_qp_reset() before swapping the + * rd_atomic array, so rxe_receiver() cannot race the free/ + * realloc. + */ + rxe_disable_task(&qp->recv_task); free_rd_atomic_resources(qp); - err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic); + /* + * On ENOMEM leave recv_task quiesced: qp->resp.resources is + * NULL and rxe_prepare_res()/find_resource() would deref it. + * Re-enable only after a fresh array is installed. + */ if (err) return err; + rxe_enable_task(&qp->recv_task); } if (mask & IB_QP_EN_SQD_ASYNC_NOTIFY) diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 12d03f390b09..24f5c044363f 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -701,6 +701,21 @@ int rxe_requester(struct rxe_qp *qp) if (unlikely(!wqe)) goto exit; + /* + * Don't trust user space data: a user QP's WQE comes from an mmap'd + * ring, so num_sge/cur_sge are attacker-controlled. Bound num_sge like + * get_srq_wqe(); bound cur_sge only when payload exists (dma.resid), + * since copy_data() skips dma->sge[] on a zero-length copy (all a + * max_sge == 0 QP can post). + */ + if (unlikely(wqe->dma.num_sge > qp->sq.max_sge || + (wqe->dma.resid && + wqe->dma.cur_sge >= qp->sq.max_sge))) { + rxe_dbg_qp(qp, "invalid num_sge/cur_sge in send wqe\n"); + wqe->status = IB_WC_LOC_QP_OP_ERR; + goto err; + } + if (rxe_wqe_is_fenced(qp, wqe)) { qp->req.wait_fence = 1; goto exit; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index d8cbdfa70cdb..02b16e2b49b8 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -1217,7 +1217,14 @@ finish: spin_lock_irqsave(&qp->state_lock, flags); if (unlikely(qp_state(qp) == IB_QPS_ERR)) { spin_unlock_irqrestore(&qp->state_lock, flags); - return RESPST_CHK_RESOURCE; + /* The packet was executed and completed before the QP + * moved to ERROR; it must be consumed exactly once. + * Re-entering the request chain with the stale packet + * would copy it into every remaining recv WQE as a new + * completion. Remaining WQEs are flushed by the drain + * path at rxe_receiver() entry. + */ + return pkt ? RESPST_CLEANUP : RESPST_CHK_RESOURCE; } spin_unlock_irqrestore(&qp->state_lock, flags); diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c index c9a7cd38953d..74904a6fdf2b 100644 --- a/drivers/infiniband/sw/rxe/rxe_srq.c +++ b/drivers/infiniband/sw/rxe/rxe_srq.c @@ -13,8 +13,8 @@ int rxe_srq_chk_init(struct rxe_dev *rxe, struct ib_srq_init_attr *init) struct ib_srq_attr *attr = &init->attr; if (attr->max_wr > rxe->attr.max_srq_wr) { - rxe_dbg_dev(rxe, "max_wr(%d) > max_srq_wr(%d)\n", - attr->max_wr, rxe->attr.max_srq_wr); + rxe_dbg_dev(rxe, "max_wr(%u) > max_srq_wr(%u)\n", + attr->max_wr, rxe->attr.max_srq_wr); goto err1; } @@ -27,8 +27,8 @@ int rxe_srq_chk_init(struct rxe_dev *rxe, struct ib_srq_init_attr *init) attr->max_wr = RXE_MIN_SRQ_WR; if (attr->max_sge > rxe->attr.max_srq_sge) { - rxe_dbg_dev(rxe, "max_sge(%d) > max_srq_sge(%d)\n", - attr->max_sge, rxe->attr.max_srq_sge); + rxe_dbg_dev(rxe, "max_sge(%u) > max_srq_sge(%u)\n", + attr->max_sge, rxe->attr.max_srq_sge); goto err1; } @@ -107,8 +107,8 @@ int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq, if (mask & IB_SRQ_MAX_WR) { if (attr->max_wr > rxe->attr.max_srq_wr) { - rxe_dbg_srq(srq, "max_wr(%d) > max_srq_wr(%d)\n", - attr->max_wr, rxe->attr.max_srq_wr); + rxe_dbg_srq(srq, "max_wr(%u) > max_srq_wr(%u)\n", + attr->max_wr, rxe->attr.max_srq_wr); goto err1; } @@ -129,8 +129,8 @@ int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq, if (mask & IB_SRQ_LIMIT) { if (attr->srq_limit > rxe->attr.max_srq_wr) { - rxe_dbg_srq(srq, "srq_limit(%d) > max_srq_wr(%d)\n", - attr->srq_limit, rxe->attr.max_srq_wr); + rxe_dbg_srq(srq, "srq_limit(%u) > max_srq_wr(%u)\n", + attr->srq_limit, rxe->attr.max_srq_wr); goto err1; } diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 1ec130fee8ea..c8562866e21f 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -22,13 +22,13 @@ static int rxe_query_device(struct ib_device *ibdev, struct rxe_dev *rxe = to_rdev(ibdev); int err; - err = ib_is_udata_in_empty(udata); + err = ib_no_udata_io(udata); if (err) return err; memcpy(attr, &rxe->attr, sizeof(*attr)); - return ib_respond_empty_udata(udata); + return 0; } static int rxe_query_port(struct ib_device *ibdev, diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 2349c08c32ed..80ef4cb95fc3 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -132,12 +132,10 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, struct siw_device *sdev = to_siw_dev(base_dev); int rv; - rv = ib_is_udata_in_empty(udata); + rv = ib_no_udata_io(udata); if (rv) return rv; - memset(attr, 0, sizeof(*attr)); - /* Revisit atomic caps if RFC 7306 gets supported */ attr->atomic_cap = 0; attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; @@ -167,7 +165,7 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr, addrconf_addr_eui48((u8 *)&attr->sys_image_guid, sdev->raw_gid); - return ib_respond_empty_udata(udata); + return 0; } int siw_query_port(struct ib_device *base_dev, u32 port, diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 57fec88a1629..ed0592898384 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -1071,8 +1071,7 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_ struct ib_qp *tx_qp; if (dev->features & NETIF_F_SG) - attr.cap.max_send_sge = min_t(u32, priv->ca->attrs.max_send_sge, - MAX_SKB_FRAGS + 1); + attr.cap.max_send_sge = min(priv->ca->attrs.max_send_sge, MAX_SKB_FRAGS + 1); tx_qp = ib_create_qp(priv->pd, &attr); tx->max_send_sge = attr.cap.max_send_sge; @@ -1582,7 +1581,8 @@ static void ipoib_cm_create_srq(struct net_device *dev, int max_sge) int ipoib_cm_dev_init(struct net_device *dev) { struct ipoib_dev_priv *priv = ipoib_priv(dev); - int max_srq_sge, i; + u32 max_srq_sge; + int i; u8 addr; INIT_LIST_HEAD(&priv->cm.passive_ids); @@ -1600,9 +1600,9 @@ int ipoib_cm_dev_init(struct net_device *dev) skb_queue_head_init(&priv->cm.skb_queue); - ipoib_dbg(priv, "max_srq_sge=%d\n", priv->ca->attrs.max_srq_sge); + ipoib_dbg(priv, "max_srq_sge=%u\n", priv->ca->attrs.max_srq_sge); - max_srq_sge = min_t(int, IPOIB_CM_RX_SG, priv->ca->attrs.max_srq_sge); + max_srq_sge = min(priv->ca->attrs.max_srq_sge, IPOIB_CM_RX_SG); ipoib_cm_create_srq(dev, max_srq_sge); if (ipoib_cm_has_srq(dev)) { priv->cm.max_cm_mtu = max_srq_sge * PAGE_SIZE - 0x10; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c index 3ed1ea566690..2490696a1aab 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c @@ -147,8 +147,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca) .cap = { .max_send_wr = ipoib_sendq_size, .max_recv_wr = ipoib_recvq_size, - .max_send_sge = min_t(u32, priv->ca->attrs.max_send_sge, - MAX_SKB_FRAGS + 1), + .max_send_sge = min(priv->ca->attrs.max_send_sge, MAX_SKB_FRAGS + 1), .max_recv_sge = IPOIB_UD_RX_SG }, .sq_sig_type = IB_SIGNAL_ALL_WR, diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index f03b3bb3c0c4..55fe68e5b837 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -244,8 +244,7 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn) max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1; else max_send_wr = ISER_QP_MAX_REQ_DTOS + 1; - max_send_wr = min_t(unsigned int, max_send_wr, - (unsigned int)ib_dev->attrs.max_qp_wr); + max_send_wr = min(max_send_wr, ib_dev->attrs.max_qp_wr); cq_size = max_send_wr + ISER_QP_MAX_RECV_DTOS; ib_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_SOFTIRQ); @@ -589,7 +588,7 @@ static void iser_route_handler(struct rdma_cm_id *cma_id) goto failure; memset(&conn_param, 0, sizeof conn_param); - conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom; + conn_param.responder_resources = min(ib_dev->attrs.max_qp_rd_atom, U8_MAX); conn_param.initiator_depth = 1; conn_param.retry_count = 7; conn_param.rnr_retry_count = 6; diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 1015a51f750a..4691845bf815 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -214,9 +214,9 @@ isert_create_device_ib_res(struct isert_device *device) struct ib_device *ib_dev = device->ib_device; int ret; - isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", + isert_dbg("devattr->max_send_sge: %u devattr->max_recv_sge %u\n", ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); - isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); + isert_dbg("devattr->max_sge_rd: %u\n", ib_dev->attrs.max_sge_rd); device->pd = ib_alloc_pd(ib_dev, 0); if (IS_ERR(device->pd)) { @@ -381,8 +381,7 @@ isert_set_nego_params(struct isert_conn *isert_conn, struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; /* Set max inflight RDMA READ requests */ - isert_conn->initiator_depth = min_t(u8, param->initiator_depth, - attr->max_qp_init_rd_atom); + isert_conn->initiator_depth = min(param->initiator_depth, attr->max_qp_init_rd_atom); isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); if (param->private_data) { diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index d34d7e5f34d6..7b2c51ae614f 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1675,8 +1675,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) * + 2 for drain and heartbeat * in case qp gets into error state. */ - max_send_wr = - min_t(int, wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2); + max_send_wr = min(wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2); max_recv_wr = max_send_wr; } else { /* @@ -1692,11 +1691,9 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr; /* Shared between connections */ clt_path->s.dev_ref++; - max_send_wr = min_t(int, wr_limit, - /* QD * (REQ + RSP + FR REGS or INVS) + drain */ - clt_path->queue_depth * 4 + 1); - max_recv_wr = min_t(int, wr_limit, - clt_path->queue_depth * 3 + 1); + /* QD * (REQ + RSP + FR REGS or INVS) + drain */ + max_send_wr = min(wr_limit, clt_path->queue_depth * 4 + 1); + max_recv_wr = min(wr_limit, clt_path->queue_depth * 3 + 1); max_send_sge = 2; } atomic_set(&con->c.sq_wr_avail, max_send_wr); diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 7d8e4422cc57..5314599d29fe 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -1740,21 +1740,16 @@ static int create_con(struct rtrs_srv_path *srv_path, * All receive and all send (each requiring invalidate) * + 2 for drain and heartbeat */ - max_send_wr = min_t(int, wr_limit, - SERVICE_CON_QUEUE_DEPTH * 2 + 2); + max_send_wr = min(wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2); max_recv_wr = max_send_wr; s->signal_interval = min_not_zero(srv->queue_depth, (size_t)SERVICE_CON_QUEUE_DEPTH); } else { /* when always_invlaidate enalbed, we need linv+rinv+mr+imm */ if (always_invalidate) - max_send_wr = - min_t(int, wr_limit, - srv->queue_depth * (1 + 4) + 1); + max_send_wr = min(wr_limit, srv->queue_depth * (1 + 4) + 1); else - max_send_wr = - min_t(int, wr_limit, - srv->queue_depth * (1 + 2) + 1); + max_send_wr = min(wr_limit, srv->queue_depth * (1 + 2) + 1); max_recv_wr = srv->queue_depth + 1; } diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index acbd787de265..0caebbc2810f 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -557,7 +557,7 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch) init_attr->cap.max_send_wr = m * target->queue_size; init_attr->cap.max_recv_wr = target->queue_size + 1; init_attr->cap.max_recv_sge = 1; - init_attr->cap.max_send_sge = min(SRP_MAX_SGE, attr->max_send_sge); + init_attr->cap.max_send_sge = min(attr->max_send_sge, SRP_MAX_SGE); init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; init_attr->qp_type = IB_QPT_RC; init_attr->send_cq = send_cq; diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index f66cfd70c263..7197d95f2216 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -77,8 +77,8 @@ module_param(srp_max_req_size, int, 0444); MODULE_PARM_DESC(srp_max_req_size, "Maximum size of SRP request messages in bytes."); -static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE; -module_param(srpt_srq_size, int, 0444); +static unsigned int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE; +module_param(srpt_srq_size, uint, 0444); MODULE_PARM_DESC(srpt_srq_size, "Shared receive queue (SRQ) size."); @@ -405,8 +405,7 @@ static void srpt_get_ioc(struct srpt_port *sport, u32 slot, if (sdev->use_srq) send_queue_depth = sdev->srq_size; else - send_queue_depth = min(MAX_SRPT_RQ_SIZE, - sdev->device->attrs.max_qp_wr); + send_queue_depth = min(sdev->device->attrs.max_qp_wr, MAX_SRPT_RQ_SIZE); memset(iocp, 0, sizeof(*iocp)); strcpy(iocp->id_string, SRPT_ID_STRING); @@ -960,6 +959,7 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, struct srpt_rdma_ch *ch = ioctx->ch; struct scatterlist *prev = NULL; unsigned prev_nents; + u8 n_rdma, n_rw_ctx; int ret, i; if (nbufs == 1) { @@ -970,6 +970,9 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, return -ENOMEM; } + n_rw_ctx = ioctx->n_rw_ctx; + n_rdma = ioctx->n_rdma; + for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) { struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i]; u64 remote_addr = be64_to_cpu(db->va); @@ -1016,6 +1019,9 @@ unwind: } if (ioctx->rw_ctxs != &ioctx->s_rw_ctx) kfree(ioctx->rw_ctxs); + ioctx->rw_ctxs = NULL; + ioctx->n_rw_ctx = n_rw_ctx; + ioctx->n_rdma = n_rdma; return ret; } @@ -1596,7 +1602,7 @@ static void srpt_handle_cmd(struct srpt_rdma_ch *ch, rc = target_init_cmd(cmd, ch->sess, &send_ioctx->sense_data[0], scsilun_to_int(&srp_cmd->lun), data_len, - TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF); + cmd->sam_task_attr, dir, TARGET_SCF_ACK_KREF); if (rc != 0) { pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc, srp_cmd->tag); @@ -1851,7 +1857,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) struct srpt_port *sport = ch->sport; struct srpt_device *sdev = sport->sdev; const struct ib_device_attr *attrs = &sdev->device->attrs; - int sq_size = sport->port_attrib.srp_sq_size; + u32 sq_size = sport->port_attrib.srp_sq_size; int i, ret; WARN_ON(ch->rq_size < 1); @@ -1912,13 +1918,13 @@ retry: bool retry = sq_size > MIN_SRPT_SQ_SIZE; if (retry) { - pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n", + pr_debug("failed to create queue pair with sq_size = %u (%d) - retrying\n", sq_size, ret); ib_cq_pool_put(ch->cq, ch->cq_size); sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE); goto retry; } else { - pr_err("failed to create queue pair with sq_size = %d (%d)\n", + pr_err("failed to create queue pair with sq_size = %u (%d)\n", sq_size, ret); goto err_destroy_cq; } @@ -1926,7 +1932,7 @@ retry: atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr); - pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d ch= %p\n", + pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %u ch= %p\n", __func__, ch->cq->cqe, qp_init->cap.max_send_sge, qp_init->cap.max_send_wr, ch); @@ -2299,7 +2305,7 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, * depth to avoid that the initiator driver has to report QUEUE_FULL * to the SCSI mid-layer. */ - ch->rq_size = min(MAX_SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr); + ch->rq_size = min(sdev->device->attrs.max_qp_wr, MAX_SRPT_RQ_SIZE); spin_lock_init(&ch->spinlock); ch->state = CH_CONNECTING; INIT_LIST_HEAD(&ch->cmd_wait_list); @@ -3137,7 +3143,7 @@ static int srpt_alloc_srq(struct srpt_device *sdev) return PTR_ERR(srq); } - pr_debug("create SRQ #wr= %d max_allow=%d dev= %s\n", sdev->srq_size, + pr_debug("create SRQ #wr= %d max_allow=%u dev= %s\n", sdev->srq_size, sdev->device->attrs.max_srq_wr, dev_name(&device->dev)); sdev->req_buf_cache = srpt_cache_get(srp_max_req_size); @@ -3952,7 +3958,7 @@ static int __init srpt_init_module(void) if (srpt_srq_size < MIN_SRPT_SRQ_SIZE || srpt_srq_size > MAX_SRPT_SRQ_SIZE) { - pr_err("invalid value %d for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n", + pr_err("invalid value %u for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n", srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE); goto out; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index c4ab4b5caa0a..fc8c50e8f365 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -1041,10 +1041,14 @@ static int ionic_get_ts_info(struct net_device *netdev, info->phc_index = ptp_clock_index(lif->phc->ptp); - info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_TX_HARDWARE | - SOF_TIMESTAMPING_RX_HARDWARE | - SOF_TIMESTAMPING_RAW_HARDWARE; + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE; + + if (!(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) + return 0; + + info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; /* tx modes */ diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h index 0a201422d0c5..6afc1fc3c7ed 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_if.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h @@ -1181,6 +1181,7 @@ enum ionic_eth_hw_features { IONIC_ETH_HW_TX_CSUM_GENEVE = BIT(18), IONIC_ETH_HW_TSO_GENEVE = BIT(19), IONIC_ETH_HW_TIMESTAMP = BIT(20), + IONIC_ETH_HW_RDMA_TIMESTAMP = BIT(21), }; /** diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index fd3ee9820531..af4aadd96cb9 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -1500,7 +1500,8 @@ static int ionic_set_nic_features(struct ionic_lif *lif, ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features); if (lif->phc) - ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP); + ctx.cmd.lif_setattr.features |= lif->ionic->ident.lif.eth.config.features & + cpu_to_le64(IONIC_ETH_HW_TIMESTAMP | IONIC_ETH_HW_RDMA_TIMESTAMP); err = ionic_adminq_post_wait(lif, &ctx); if (err) @@ -1551,6 +1552,8 @@ static int ionic_set_nic_features(struct ionic_lif *lif, dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n"); if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP) dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n"); + if (lif->hw_features & IONIC_ETH_HW_RDMA_TIMESTAMP) + dev_dbg(dev, "feature ETH_HW_RDMA_TIMESTAMP\n"); return 0; } diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h index d34692462036..e790f6b9441e 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h @@ -248,7 +248,7 @@ struct ionic_lif { }; struct ionic_phc { - spinlock_t lock; /* lock for cc and tc */ + spinlock_t lock; /* lock for state_page, cc and tc */ struct cyclecounter cc; struct timecounter tc; @@ -261,6 +261,7 @@ struct ionic_phc { long aux_work_delay; struct ptp_clock_info ptp_info; + struct ib_uverbs_clock_info *state_page; struct ptp_clock *ptp; struct ionic_lif *lif; }; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_phc.c b/drivers/net/ethernet/pensando/ionic/ionic_phc.c index 05b44fc482f8..4b8d80a55b72 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_phc.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_phc.c @@ -3,6 +3,7 @@ #include <linux/netdevice.h> #include <linux/etherdevice.h> +#include <uapi/rdma/ib_user_verbs.h> #include "ionic.h" #include "ionic_bus.h" @@ -77,7 +78,8 @@ static int ionic_lif_hwstamp_set_ts_config(struct ionic_lif *lif, bool rx_all; __le64 mask; - if (!lif->phc || !lif->phc->ptp) + if (!lif->phc || !lif->phc->ptp || + !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) return -EOPNOTSUPP; mutex_lock(&lif->phc->config_lock); @@ -210,7 +212,8 @@ int ionic_hwstamp_set(struct net_device *netdev, struct ionic_lif *lif = netdev_priv(netdev); int err; - if (!lif->phc || !lif->phc->ptp) + if (!lif->phc || !lif->phc->ptp || + !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) return -EOPNOTSUPP; mutex_lock(&lif->queue_lock); @@ -228,7 +231,8 @@ void ionic_lif_hwstamp_replay(struct ionic_lif *lif) { int err; - if (!lif->phc || !lif->phc->ptp) + if (!lif->phc || !lif->phc->ptp || + !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) return; mutex_lock(&lif->queue_lock); @@ -242,7 +246,8 @@ void ionic_lif_hwstamp_recreate_queues(struct ionic_lif *lif) { int err; - if (!lif->phc || !lif->phc->ptp) + if (!lif->phc || !lif->phc->ptp || + !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) return; mutex_lock(&lif->phc->config_lock); @@ -267,7 +272,8 @@ int ionic_hwstamp_get(struct net_device *netdev, { struct ionic_lif *lif = netdev_priv(netdev); - if (!lif->phc || !lif->phc->ptp) + if (!lif->phc || !lif->phc->ptp || + !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) return -EOPNOTSUPP; mutex_lock(&lif->phc->config_lock); @@ -329,6 +335,26 @@ static int ionic_setphc_cmd(struct ionic_phc *phc, struct ionic_admin_ctx *ctx) return ionic_adminq_post(phc->lif, ctx); } +static void ionic_phc_state_page_update(struct ionic_phc *phc) +{ + struct ib_uverbs_clock_info *state = phc->state_page; + u32 sign; + + /* read current sign */ + sign = smp_load_acquire(&state->sign) & ~1; + + /* make sign odd for updating */ + smp_store_mb(state->sign, sign | 1); + + state->cycles = phc->tc.cycle_last; + state->nsec = phc->tc.nsec; + state->frac = phc->tc.frac; + state->mult = phc->cc.mult; + + /* make sign the next even number for update completed */ + smp_store_release(&state->sign, sign + 2); +} + static int ionic_phc_adjfine(struct ptp_clock_info *info, long scaled_ppm) { struct ionic_phc *phc = container_of(info, struct ionic_phc, ptp_info); @@ -356,6 +382,8 @@ static int ionic_phc_adjfine(struct ptp_clock_info *info, long scaled_ppm) timecounter_read(&phc->tc); phc->cc.mult = adj; + ionic_phc_state_page_update(phc); + /* Setphc commands are posted in-order, sequenced by phc->lock. We * need to drop the lock before waiting for the command to complete. */ @@ -381,6 +409,8 @@ static int ionic_phc_adjtime(struct ptp_clock_info *info, s64 delta) timecounter_adjtime(&phc->tc, delta); + ionic_phc_state_page_update(phc); + /* Setphc commands are posted in-order, sequenced by phc->lock. We * need to drop the lock before waiting for the command to complete. */ @@ -410,6 +440,8 @@ static int ionic_phc_settime64(struct ptp_clock_info *info, timecounter_init(&phc->tc, &phc->cc, ns); + ionic_phc_state_page_update(phc); + /* Setphc commands are posted in-order, sequenced by phc->lock. We * need to drop the lock before waiting for the command to complete. */ @@ -467,6 +499,8 @@ static long ionic_phc_aux_work(struct ptp_clock_info *info) /* update point-in-time basis to now */ timecounter_read(&phc->tc); + ionic_phc_state_page_update(phc); + /* Setphc commands are posted in-order, sequenced by phc->lock. We * need to drop the lock before waiting for the command to complete. */ @@ -506,7 +540,8 @@ static const struct ptp_clock_info ionic_ptp_info = { void ionic_lif_register_phc(struct ionic_lif *lif) { - if (!lif->phc || !(lif->hw_features & IONIC_ETH_HW_TIMESTAMP)) + if (!lif->phc || + !(lif->hw_features & (IONIC_ETH_HW_TIMESTAMP | IONIC_ETH_HW_RDMA_TIMESTAMP))) return; lif->phc->ptp = ptp_clock_register(&lif->phc->ptp_info, lif->ionic->dev); @@ -545,13 +580,19 @@ void ionic_lif_alloc_phc(struct ionic_lif *lif) return; features = le64_to_cpu(ionic->ident.lif.eth.config.features); - if (!(features & IONIC_ETH_HW_TIMESTAMP)) + if (!(features & (IONIC_ETH_HW_TIMESTAMP | IONIC_ETH_HW_RDMA_TIMESTAMP))) return; phc = devm_kzalloc(ionic->dev, sizeof(*phc), GFP_KERNEL); if (!phc) return; + phc->state_page = (void *)get_zeroed_page(GFP_KERNEL); + if (!phc->state_page) { + devm_kfree(ionic->dev, phc); + return; + } + phc->lif = lif; phc->cc.read = ionic_cc_read; @@ -563,6 +604,7 @@ void ionic_lif_alloc_phc(struct ionic_lif *lif) dev_err(lif->ionic->dev, "Invalid device PHC mask multiplier %u, disabling HW timestamp support\n", phc->cc.mult); + free_page((unsigned long)phc->state_page); devm_kfree(lif->ionic->dev, phc); lif->phc = NULL; return; @@ -646,6 +688,12 @@ void ionic_lif_alloc_phc(struct ionic_lif *lif) */ phc->ptp_info.max_adj = NORMAL_PPB; + phc->state_page->mask = phc->cc.mask; + phc->state_page->shift = phc->cc.shift; + phc->state_page->overflow_period = delay; + + ionic_phc_state_page_update(phc); + lif->phc = phc; } @@ -656,6 +704,7 @@ void ionic_lif_free_phc(struct ionic_lif *lif) mutex_destroy(&lif->phc->config_lock); + free_page((unsigned long)lif->phc->state_page); devm_kfree(lif->ionic->dev, lif->phc); lif->phc = NULL; } diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 6909e3542794..56cd228af1d5 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -394,8 +394,10 @@ nvme_rdma_find_get_device(struct rdma_cm_id *cm_id) goto out_free_pd; } - ndev->num_inline_segments = min(NVME_RDMA_MAX_INLINE_SEGMENTS, - ndev->dev->attrs.max_send_sge - 1); + ndev->num_inline_segments = ndev->dev->attrs.max_send_sge; + if (ndev->num_inline_segments) + ndev->num_inline_segments--; + ndev->num_inline_segments = min(ndev->num_inline_segments, NVME_RDMA_MAX_INLINE_SEGMENTS); list_add(&ndev->entry, &device_list); out_unlock: mutex_unlock(&device_list_mutex); @@ -1847,7 +1849,7 @@ static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue) param.qp_num = queue->qp->qp_num; param.flow_control = 1; - param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom; + param.responder_resources = min(queue->device->dev->attrs.max_qp_rd_atom, U8_MAX); /* maximum retry count */ param.retry_count = 7; param.rnr_retry_count = 7; diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index ea1185b8267e..bdd28ce99faa 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -149,10 +149,10 @@ MODULE_PARM_DESC(use_srq, "Use shared receive queue."); static int srq_size_set(const char *val, const struct kernel_param *kp); static const struct kernel_param_ops srq_size_ops = { .set = srq_size_set, - .get = param_get_int, + .get = param_get_uint, }; -static int nvmet_rdma_srq_size = 1024; +static unsigned int nvmet_rdma_srq_size = 1024; module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644); MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)"); @@ -180,13 +180,14 @@ static const struct nvmet_fabrics_ops nvmet_rdma_ops; static int srq_size_set(const char *val, const struct kernel_param *kp) { - int n = 0, ret; + unsigned int n; + int ret; - ret = kstrtoint(val, 10, &n); + ret = kstrtouint(val, 10, &n); if (ret != 0 || n < 256) return -EINVAL; - return param_set_int(val, kp); + return param_set_uint(val, kp); } static int num_pages(int len) @@ -1153,8 +1154,8 @@ static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev) ndev->srq_size = min(ndev->device->attrs.max_srq_wr, nvmet_rdma_srq_size); - ndev->srq_count = min(ndev->device->num_comp_vectors, - ndev->device->attrs.max_srq); + ndev->srq_count = min_t(u32, ndev->device->num_comp_vectors, + ndev->device->attrs.max_srq); ndev->srqs = kzalloc_objs(*ndev->srqs, ndev->srq_count); if (!ndev->srqs) @@ -1199,7 +1200,7 @@ nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id) struct nvmet_port *nport = port->nport; struct nvmet_rdma_device *ndev; int inline_page_count; - int inline_sge_count; + u32 inline_sge_count; int ret; mutex_lock(&device_list_mutex); @@ -1215,7 +1216,9 @@ nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id) inline_page_count = num_pages(nport->inline_data_size); inline_sge_count = max(cm_id->device->attrs.max_sge_rd, - cm_id->device->attrs.max_recv_sge) - 1; + cm_id->device->attrs.max_recv_sge); + if (inline_sge_count) + inline_sge_count--; if (inline_page_count > inline_sge_count) { pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n", nport->inline_data_size, cm_id->device->name, @@ -1555,8 +1558,9 @@ static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id, param.rnr_retry_count = 7; param.flow_control = 1; - param.initiator_depth = min_t(u8, p->initiator_depth, - queue->dev->device->attrs.max_qp_init_rd_atom); + param.initiator_depth = min3(p->initiator_depth, + queue->dev->device->attrs.max_qp_init_rd_atom, + U8_MAX); param.private_data = &priv; param.private_data_len = sizeof(priv); priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0); diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c index 529740005838..44b681a20725 100644 --- a/fs/smb/smbdirect/accept.c +++ b/fs/smb/smbdirect/accept.c @@ -32,8 +32,9 @@ int smbdirect_accept_connect_request(struct smbdirect_socket *sc, /* * First set what the we as server are able to support */ - sp->initiator_depth = min_t(u8, sp->initiator_depth, - sc->ib.dev->attrs.max_qp_rd_atom); + sp->initiator_depth = min3(sp->initiator_depth, + sc->ib.dev->attrs.max_qp_rd_atom, + U8_MAX); peer_initiator_depth = param->initiator_depth; peer_responder_resources = param->responder_resources; diff --git a/fs/smb/smbdirect/connect.c b/fs/smb/smbdirect/connect.c index cd726b399afe..34a3e72c38fb 100644 --- a/fs/smb/smbdirect/connect.c +++ b/fs/smb/smbdirect/connect.c @@ -182,8 +182,9 @@ static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc) if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) sc->mr_io.type = IB_MR_TYPE_SG_GAPS; - sp->responder_resources = min_t(u8, sp->responder_resources, - sc->ib.dev->attrs.max_qp_rd_atom); + sp->responder_resources = min3(sp->responder_resources, + sc->ib.dev->attrs.max_qp_rd_atom, + U8_MAX); smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, "responder_resources=%d\n", sp->responder_resources); diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c index 8adf58097534..690acb84e1b5 100644 --- a/fs/smb/smbdirect/connection.c +++ b/fs/smb/smbdirect/connection.c @@ -287,7 +287,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc) qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) { pr_err("Possible CQE overrun: max_send_wr %d\n", qp_cap.max_send_wr); - pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", + pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n", IB_DEVICE_NAME_MAX, sc->ib.dev->name, sc->ib.dev->attrs.max_cqe, @@ -302,7 +302,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc) max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) { pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n", rdma_send_wr, qp_cap.max_send_wr, max_send_wr); - pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", + pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n", IB_DEVICE_NAME_MAX, sc->ib.dev->name, sc->ib.dev->attrs.max_cqe, @@ -316,7 +316,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc) qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) { pr_err("Possible CQE overrun: max_recv_wr %d\n", qp_cap.max_recv_wr); - pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", + pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n", IB_DEVICE_NAME_MAX, sc->ib.dev->name, sc->ib.dev->attrs.max_cqe, @@ -328,7 +328,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc) if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge || qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) { - pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", + pr_err("device %.*s max_send_sge/max_recv_sge = %u/%u too small\n", IB_DEVICE_NAME_MAX, sc->ib.dev->name, sc->ib.dev->attrs.max_send_sge, diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 5aadb47b3b0e..76aa5ec4ab40 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -77,8 +77,8 @@ struct svcxprt_rdma { struct rdma_cm_id *sc_cm_id; /* RDMA connection id */ struct list_head sc_accept_q; /* Conn. waiting accept */ struct rpcrdma_notification sc_rn; /* removal notification */ - int sc_ord; /* RDMA read limit */ - int sc_max_send_sges; + u32 sc_ord; /* RDMA read limit */ + unsigned int sc_max_send_sges; bool sc_snd_w_inv; /* OK to use Send With Invalidate */ atomic_t sc_sq_avail; /* SQEs ready to be consumed */ diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 31b3a86fe73a..1fe87fd1d769 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -250,7 +250,7 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, unsigned long offset, size_t size, int fd, int access, - struct dma_buf_attach_ops *ops) + const struct dma_buf_attach_ops *ops) { return ERR_PTR(-EOPNOTSUPP); } diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 794746de8db0..cb3b6163961b 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -407,36 +407,36 @@ struct ib_device_attr { u32 vendor_id; u32 vendor_part_id; u32 hw_ver; - int max_qp; - int max_qp_wr; + u32 max_qp; + u32 max_qp_wr; u64 device_cap_flags; u64 kernel_cap_flags; - int max_send_sge; - int max_recv_sge; - int max_sge_rd; - int max_cq; - int max_cqe; - int max_mr; - int max_pd; - int max_qp_rd_atom; - int max_ee_rd_atom; - int max_res_rd_atom; - int max_qp_init_rd_atom; - int max_ee_init_rd_atom; + u32 max_send_sge; + u32 max_recv_sge; + u32 max_sge_rd; + u32 max_cq; + u32 max_cqe; + u32 max_mr; + u32 max_pd; + u32 max_qp_rd_atom; + u32 max_ee_rd_atom; + u32 max_res_rd_atom; + u32 max_qp_init_rd_atom; + u32 max_ee_init_rd_atom; enum ib_atomic_cap atomic_cap; enum ib_atomic_cap masked_atomic_cap; - int max_ee; - int max_rdd; - int max_mw; - int max_raw_ipv6_qp; - int max_raw_ethy_qp; - int max_mcast_grp; - int max_mcast_qp_attach; - int max_total_mcast_qp_attach; - int max_ah; - int max_srq; - int max_srq_wr; - int max_srq_sge; + u32 max_ee; + u32 max_rdd; + u32 max_mw; + u32 max_raw_ipv6_qp; + u32 max_raw_ethy_qp; + u32 max_mcast_grp; + u32 max_mcast_qp_attach; + u32 max_total_mcast_qp_attach; + u32 max_ah; + u32 max_srq; + u32 max_srq_wr; + u32 max_srq_sge; unsigned int max_fast_reg_page_list_len; unsigned int max_pi_fast_reg_page_list_len; u16 max_pkeys; @@ -1746,6 +1746,42 @@ struct ib_cq { struct rdma_restrack_entry res; }; +enum ib_qp_attach_comp_cntr_op { + IB_QP_ATTACH_COMP_CNTR_OP_SEND = IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND, + IB_QP_ATTACH_COMP_CNTR_OP_RECV = IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV, + IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ = IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ, + IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ = IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ, + IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE = IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE, + IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE = IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE, +}; + +struct ib_comp_cntr_caps { + u64 max_value; + u32 max_counters; + u32 supported_qp_attach_ops; /* Bitmask of enum ib_qp_attach_comp_cntr_op */ +}; + +struct ib_comp_cntr { + struct ib_device *device; + struct ib_uobject *uobject; + atomic_t usecnt; + struct rdma_restrack_entry res; +}; + +enum ib_comp_cntr_entry { + IB_COMP_CNTR_ENTRY_COMP = IB_UVERBS_COMP_CNTR_ENTRY_COMP, + IB_COMP_CNTR_ENTRY_ERR = IB_UVERBS_COMP_CNTR_ENTRY_ERR, +}; + +enum ib_comp_cntr_modify_op { + IB_COMP_CNTR_MODIFY_OP_SET = IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, + IB_COMP_CNTR_MODIFY_OP_INC = IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, +}; + +struct ib_qp_attach_comp_cntr_attr { + u32 op_mask; /* Bitmask of enum ib_qp_attach_comp_cntr_op */ +}; + struct ib_srq { struct ib_device *device; struct ib_pd *pd; @@ -1916,6 +1952,8 @@ struct ib_qp { struct completion srq_completion; struct ib_xrcd *xrcd; /* XRC TGT QPs only */ struct list_head xrcd_list; + struct xarray comp_cntrs; /* op_mask -> comp_cntr */ + u32 comp_cntr_op_mask; /* count times opened, mcast attaches, flow attaches */ atomic_t usecnt; @@ -2629,6 +2667,8 @@ struct ib_device_ops { struct ib_udata *udata); int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); + int (*qp_attach_comp_cntr)(struct ib_qp *qp, struct ib_comp_cntr *cc, + struct ib_qp_attach_comp_cntr_attr *attr); int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata); @@ -2650,6 +2690,15 @@ struct ib_device_ops { * post_destroy_cq - Free all kernel resources */ void (*post_destroy_cq)(struct ib_cq *cq); + int (*create_comp_cntr)(struct ib_comp_cntr *cc, + struct uverbs_attr_bundle *attrs); + int (*destroy_comp_cntr)(struct ib_comp_cntr *cc); + int (*modify_comp_cntr)(struct ib_comp_cntr *cc, enum ib_comp_cntr_entry entry, + enum ib_comp_cntr_modify_op op, u64 value); + int (*read_comp_cntr)(struct ib_comp_cntr *cc, enum ib_comp_cntr_entry entry, u64 *value); + int (*query_comp_cntr_caps)(struct ib_device *dev, + struct ib_comp_cntr_caps *caps, + struct uverbs_attr_bundle *attrs); struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags); struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int mr_access_flags, @@ -2883,6 +2932,7 @@ struct ib_device_ops { DECLARE_RDMA_OBJ_SIZE(ib_ah); DECLARE_RDMA_OBJ_SIZE(ib_counters); DECLARE_RDMA_OBJ_SIZE(ib_cq); + DECLARE_RDMA_OBJ_SIZE(ib_comp_cntr); DECLARE_RDMA_OBJ_SIZE(ib_dmah); DECLARE_RDMA_OBJ_SIZE(ib_mw); DECLARE_RDMA_OBJ_SIZE(ib_pd); diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h index 451f99e3717d..47fec3083d28 100644 --- a/include/rdma/restrack.h +++ b/include/rdma/restrack.h @@ -61,6 +61,10 @@ enum rdma_restrack_type { */ RDMA_RESTRACK_DMAH, /** + * @RDMA_RESTRACK_COMP_CNTR: Completion Counter + */ + RDMA_RESTRACK_COMP_CNTR, + /** * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations */ RDMA_RESTRACK_MAX @@ -123,7 +127,7 @@ struct rdma_restrack_entry { u32 id; }; -int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, +u32 rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, bool show_details); /** * rdma_is_kernel_res() - check the owner of resource diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 24fd36213023..80f3ba6663d0 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -1151,4 +1151,24 @@ static inline int ib_respond_empty_udata(struct ib_udata *udata) return 0; } +/** + * ib_no_udata_io - Ensure no input data and zero fill the response buffer + * @udata: The system call's ib_udata struct + * + * Driver ops which do not accept any input data and do not provide any response + * data may call this at the beginning of their handler to fully adhere to the + * uAPI forward/backward compatibility rules. + * + * Return: Negative failure code if the op should be denied, 0 otherwise. + */ +static inline int ib_no_udata_io(struct ib_udata *udata) +{ + int ret = ib_is_udata_in_empty(udata); + + if (ret) + return ret; + + return ib_respond_empty_udata(udata); +} + #endif diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h index d5c18f8de182..c79b54aade23 100644 --- a/include/uapi/rdma/efa-abi.h +++ b/include/uapi/rdma/efa-abi.h @@ -133,6 +133,7 @@ enum { EFA_QUERY_DEVICE_CAPS_RDMA_WRITE = 1 << 5, EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV = 1 << 6, EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM = 1 << 7, + EFA_QUERY_DEVICE_CAPS_COMP_CNTR = 1 << 8, }; struct efa_ibv_ex_query_device_resp { @@ -163,4 +164,9 @@ enum efa_mr_methods { EFA_IB_METHOD_MR_QUERY = (1U << UVERBS_ID_NS_SHIFT), }; +enum efa_comp_cntr_create_attrs { + EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER = (1U << UVERBS_ID_NS_SHIFT), + EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, +}; + #endif /* EFA_ABI_USER_H */ diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index 839835bd4b23..6a3d59d03f54 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -57,6 +57,7 @@ enum uverbs_default_objects { UVERBS_OBJECT_ASYNC_EVENT, UVERBS_OBJECT_DMAH, UVERBS_OBJECT_DMABUF, + UVERBS_OBJECT_COMP_CNTR, }; enum { @@ -75,6 +76,7 @@ enum uverbs_methods_device { UVERBS_METHOD_QUERY_GID_TABLE, UVERBS_METHOD_QUERY_GID_ENTRY, UVERBS_METHOD_QUERY_PORT_SPEED, + UVERBS_METHOD_QUERY_COMP_CNTR_CAPS, }; enum uverbs_attrs_invoke_write_cmd_attr_ids { @@ -93,6 +95,12 @@ enum uverbs_attrs_query_port_speed_cmd_attr_ids { UVERBS_ATTR_QUERY_PORT_SPEED_RESP, }; +enum uverbs_attrs_query_comp_cntr_caps_attr_ids { + UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, +}; + enum uverbs_attrs_get_context_attr_ids { UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, @@ -169,9 +177,16 @@ enum uverbs_attrs_destroy_qp_cmd_attr_ids { UVERBS_ATTR_DESTROY_QP_RESP, }; +enum uverbs_attrs_qp_attach_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, +}; + enum uverbs_methods_qp { UVERBS_METHOD_QP_CREATE, UVERBS_METHOD_QP_DESTROY, + UVERBS_METHOD_QP_ATTACH_COMP_CNTR, }; enum uverbs_attrs_create_srq_cmd_attr_ids { @@ -438,4 +453,32 @@ enum uverbs_attrs_query_gid_entry_cmd_attr_ids { UVERBS_ATTR_QUERY_GID_ENTRY_RESP_ENTRY, }; +enum uverbs_methods_comp_cntr { + UVERBS_METHOD_COMP_CNTR_CREATE, + UVERBS_METHOD_COMP_CNTR_DESTROY, + UVERBS_METHOD_COMP_CNTR_MODIFY, + UVERBS_METHOD_COMP_CNTR_READ, +}; + +enum uverbs_attrs_create_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, +}; + +enum uverbs_attrs_destroy_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, +}; + +enum uverbs_attrs_modify_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, + UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, + UVERBS_ATTR_MODIFY_COMP_CNTR_OP, + UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, +}; + +enum uverbs_attrs_read_comp_cntr_cmd_attr_ids { + UVERBS_ATTR_READ_COMP_CNTR_HANDLE, + UVERBS_ATTR_READ_COMP_CNTR_ENTRY, + UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, +}; + #endif diff --git a/include/uapi/rdma/ib_user_ioctl_verbs.h b/include/uapi/rdma/ib_user_ioctl_verbs.h index 51030c27d479..21f86cc7bb1f 100644 --- a/include/uapi/rdma/ib_user_ioctl_verbs.h +++ b/include/uapi/rdma/ib_user_ioctl_verbs.h @@ -300,4 +300,23 @@ struct ib_uverbs_buffer_desc { __aligned_u64 length; }; +enum ib_uverbs_comp_cntr_entry { + IB_UVERBS_COMP_CNTR_ENTRY_COMP, + IB_UVERBS_COMP_CNTR_ENTRY_ERR, +}; + +enum ib_uverbs_comp_cntr_modify_op { + IB_UVERBS_COMP_CNTR_MODIFY_OP_SET, + IB_UVERBS_COMP_CNTR_MODIFY_OP_INC, +}; + +enum ib_uverbs_qp_attach_comp_cntr_op { + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND = 1 << 0, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV = 1 << 1, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ = 1 << 2, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ = 1 << 3, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE = 1 << 4, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE = 1 << 5, +}; + #endif diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h index d2aeadb6d2f9..565301594634 100644 --- a/include/uapi/rdma/ib_user_verbs.h +++ b/include/uapi/rdma/ib_user_verbs.h @@ -1379,4 +1379,37 @@ enum ib_uverbs_raw_packet_caps { IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP = 1 << 3, }; +/* + * struct ib_uverbs_clock_info - timecounter state shared with userspace + * + * Drivers that use a software timecounter over a free-running hardware + * cycle counter can map this page read-only into userspace, allowing + * conversion of hardware timestamps to system time without a syscall. + * + * Synchronization uses a sequence counter (@sign): the kernel sets bit 0 + * before updating, then advances by 2 after. Userspace must retry the read + * if @sign is odd or changed during the read. + * + * @sign: Sequence counter (bit 0 = update in progress) + * @resv: Reserved + * @nsec: Nanoseconds at last update + * @cycles: Cycle counter value at last update + * @frac: Fractional nanoseconds at last update + * @mult: Cycle-to-nanosecond multiplier + * @shift: Cycle-to-nanosecond shift + * @mask: Cycle counter bitmask + * @overflow_period: Max interval (nsec) between reads before counter wraps + */ +struct ib_uverbs_clock_info { + __u32 sign; + __u32 resv; + __aligned_u64 nsec; + __aligned_u64 cycles; + __aligned_u64 frac; + __u32 mult; + __u32 shift; + __aligned_u64 mask; + __aligned_u64 overflow_period; +}; + #endif /* IB_USER_VERBS_H */ diff --git a/include/uapi/rdma/ionic-abi.h b/include/uapi/rdma/ionic-abi.h index 7b589d3e9728..2c70ac149c4f 100644 --- a/include/uapi/rdma/ionic-abi.h +++ b/include/uapi/rdma/ionic-abi.h @@ -48,6 +48,7 @@ struct ionic_ctx_resp { __u8 expdb_qtypes; __u8 rsvd2[3]; + __aligned_u64 phc_offset; }; struct ionic_qdesc { diff --git a/include/uapi/rdma/irdma-abi.h b/include/uapi/rdma/irdma-abi.h index 36f20802bcc8..38155affc8b4 100644 --- a/include/uapi/rdma/irdma-abi.h +++ b/include/uapi/rdma/irdma-abi.h @@ -88,6 +88,7 @@ struct irdma_create_srq_resp { struct irdma_create_qp_req { __aligned_u64 user_wqe_bufs; __aligned_u64 user_compl_ctx; + __aligned_u64 legacy_dontuse[2]; }; struct irdma_mem_reg_req { diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h index a75bf32b8cfb..8336bf51b7c5 100644 --- a/include/uapi/rdma/mana-abi.h +++ b/include/uapi/rdma/mana-abi.h @@ -25,7 +25,7 @@ enum mana_ib_create_cq_flags { struct mana_ib_create_cq { __aligned_u64 buf_addr; - __u16 flags; + __u16 comp_mask; __u16 reserved0; __u32 reserved1; }; diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h index 8a6ad6c6841c..a39226cd62dc 100644 --- a/include/uapi/rdma/mlx5-abi.h +++ b/include/uapi/rdma/mlx5-abi.h @@ -472,17 +472,10 @@ struct mlx5_ib_modify_wq { __u32 reserved; }; -struct mlx5_ib_clock_info { - __u32 sign; - __u32 resv; - __aligned_u64 nsec; - __aligned_u64 cycles; - __aligned_u64 frac; - __u32 mult; - __u32 shift; - __aligned_u64 mask; - __aligned_u64 overflow_period; -}; +/* + * deprecated, see struct ib_uverbs_clock_info from ib_user_verbs.h + */ +#define mlx5_ib_clock_info ib_uverbs_clock_info enum mlx5_ib_mmap_cmd { MLX5_IB_MMAP_REGULAR_PAGE = 0, diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h index aac9782ddc09..3af946ecbac3 100644 --- a/include/uapi/rdma/rdma_netlink.h +++ b/include/uapi/rdma/rdma_netlink.h @@ -605,6 +605,11 @@ enum rdma_nldev_attr { RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY, /* u64 */ /* + * Resource summary entry maximum value. + */ + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, /* u64 */ + + /* * Always the end */ RDMA_NLDEV_ATTR_MAX diff --git a/net/rds/ib.c b/net/rds/ib.c index 8f9cf491984f..786f39169bc1 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -162,12 +162,12 @@ static int rds_ib_add_one(struct ib_device *device) IB_ODP_SUPPORT_READ); rds_ibdev->max_1m_mrs = device->attrs.max_mr ? - min_t(unsigned int, (device->attrs.max_mr / 2), - rds_ib_mr_1m_pool_size) : rds_ib_mr_1m_pool_size; + min(device->attrs.max_mr / 2, + rds_ib_mr_1m_pool_size) : rds_ib_mr_1m_pool_size; rds_ibdev->max_8k_mrs = device->attrs.max_mr ? - min_t(unsigned int, ((device->attrs.max_mr / 2) * RDS_MR_8K_SCALE), - rds_ib_mr_8k_pool_size) : rds_ib_mr_8k_pool_size; + min((device->attrs.max_mr / 2) * RDS_MR_8K_SCALE, + rds_ib_mr_8k_pool_size) : rds_ib_mr_8k_pool_size; rds_ibdev->max_initiator_depth = device->attrs.max_qp_init_rd_atom; rds_ibdev->max_responder_resources = device->attrs.max_qp_rd_atom; @@ -204,7 +204,7 @@ static int rds_ib_add_one(struct ib_device *device) goto put_dev; } - rdsdebug("RDS/IB: max_mr = %d, max_wrs = %d, max_sge = %d, max_1m_mrs = %d, max_8k_mrs = %d\n", + rdsdebug("RDS/IB: max_mr = %u, max_wrs = %d, max_sge = %d, max_1m_mrs = %d, max_8k_mrs = %d\n", device->attrs.max_mr, rds_ibdev->max_wrs, rds_ibdev->max_sge, rds_ibdev->max_1m_mrs, rds_ibdev->max_8k_mrs); diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index d46146887ba4..4feb0edc360c 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -173,11 +173,11 @@ static void rds_ib_cm_fill_conn_param(struct rds_connection *conn, memset(conn_param, 0, sizeof(struct rdma_conn_param)); - conn_param->responder_resources = - min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources); - conn_param->initiator_depth = - min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth); - conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7); + conn_param->responder_resources = min3(rds_ibdev->max_responder_resources, + max_responder_resources, U8_MAX); + conn_param->initiator_depth = min3(rds_ibdev->max_initiator_depth, + max_initiator_depth, U8_MAX); + conn_param->retry_count = min(rds_ib_retry_count, 7U); conn_param->rnr_retry_count = 7; if (dp) { diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index e5c71cf705a3..e83cef19e656 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -172,8 +172,9 @@ out_mr_err: int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) { const struct ib_device_attr *attrs = &device->attrs; - int max_qp_wr, depth, delta; unsigned int max_sge; + u32 max_qp_wr; + int depth, delta; if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) || attrs->max_fast_reg_page_list_len == 0) { @@ -229,10 +230,10 @@ int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device) } max_qp_wr = attrs->max_qp_wr; + if (max_qp_wr < RPCRDMA_BACKWARD_WRS + 1 + RPCRDMA_MIN_SLOT_TABLE) + return -ENOMEM; max_qp_wr -= RPCRDMA_BACKWARD_WRS; max_qp_wr -= 1; - if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE) - return -ENOMEM; if (ep->re_max_requests > max_qp_wr) ep->re_max_requests = max_qp_wr; ep->re_attr.cap.max_send_wr = ep->re_max_requests * depth; diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 093371f9d245..927269598ac2 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -574,8 +574,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) set_bit(RDMAXPRT_CONN_PENDING, &newxprt->sc_flags); memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = 0; - conn_param.initiator_depth = min_t(int, newxprt->sc_ord, - dev->attrs.max_qp_init_rd_atom); + conn_param.initiator_depth = min(newxprt->sc_ord, dev->attrs.max_qp_init_rd_atom); if (!conn_param.initiator_depth) { ret = -EINVAL; trace_svcrdma_initdepth_err(newxprt, ret); @@ -600,7 +599,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) dprintk(" local address : %pIS:%u\n", sap, rpc_get_port(sap)); sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr; dprintk(" remote address : %pIS:%u\n", sap, rpc_get_port(sap)); - dprintk(" max_sge : %d\n", newxprt->sc_max_send_sges); + dprintk(" max_sge : %u\n", newxprt->sc_max_send_sges); dprintk(" sq_depth : %d\n", newxprt->sc_sq_depth); dprintk(" rdma_rw_ctxs : %d\n", ctxts); dprintk(" max_requests : %d\n", newxprt->sc_max_requests); diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 04b286223b24..be335eed329d 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -465,7 +465,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) /* Client offers RDMA Read but does not initiate */ ep->re_remote_cma.initiator_depth = 0; ep->re_remote_cma.responder_resources = - min_t(int, U8_MAX, device->attrs.max_qp_rd_atom); + min(device->attrs.max_qp_rd_atom, U8_MAX); /* Limit transport retries so client can detect server * GID changes quickly. RPC layer handles re-establishing |