diff options
Diffstat (limited to 'drivers/infiniband/core')
25 files changed, 770 insertions, 106 deletions
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile index ab7a2197bc86..47ef6b0afd29 100644 --- a/drivers/infiniband/core/Makefile +++ b/drivers/infiniband/core/Makefile @@ -38,6 +38,7 @@ ib_umad-y := user_mad.o ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \ uverbs_std_types.o uverbs_ioctl.o \ uverbs_std_types_cq.o \ + uverbs_std_types_comp_cntr.o \ uverbs_std_types_dmabuf.o \ uverbs_std_types_dmah.o \ uverbs_std_types_flow_action.o uverbs_std_types_dm.o \ diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index e9fb7ad4c377..bdaa3eb83bcc 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -769,7 +769,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr) /* * sync canceling the work after removing it from the req_list - * guarentees no work is running and none will be started. + * guarantees no work is running and none will be started. */ cancel_delayed_work_sync(&found->work); kfree(found); diff --git a/drivers/infiniband/core/cgroup.c b/drivers/infiniband/core/cgroup.c index 1f037fe01450..8611b4e32cfb 100644 --- a/drivers/infiniband/core/cgroup.c +++ b/drivers/infiniband/core/cgroup.c @@ -17,6 +17,7 @@ void ib_device_register_rdmacg(struct ib_device *device) { device->cg_device.name = device->name; + device->cg_device.index = device->index; rdmacg_register_device(&device->cg_device); } diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index e88d3efb967b..cc24fddf9aaa 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -2104,7 +2104,7 @@ static void destroy_id_handler_unlock(struct rdma_id_private *id_priv) /* * Setting the state to destroyed under the handler mutex provides a * fence against calling handler callbacks. If this is invoked due to - * the failure of a handler callback then it guarentees that no future + * the failure of a handler callback then it guarantees that no future * handlers will be called. */ lockdep_assert_held(&id_priv->handler_mutex); diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c index 891e52afb8f4..c389d4e37b6b 100644 --- a/drivers/infiniband/core/cma_configfs.c +++ b/drivers/infiniband/core/cma_configfs.c @@ -65,6 +65,10 @@ static struct cma_dev_port_group *to_dev_port_group(struct config_item *item) return container_of(group, struct cma_dev_port_group, group); } +/* + * configfs is not net namespace aware, so a name shared by devices in + * different namespaces resolves to the first match here. + */ static bool filter_by_name(struct ib_device *ib_dev, void *cookie) { return !strcmp(dev_name(&ib_dev->dev), cookie); diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h index 19104c542b27..aaf330b0d333 100644 --- a/drivers/infiniband/core/core_priv.h +++ b/drivers/infiniband/core/core_priv.h @@ -356,7 +356,8 @@ void ib_port_unregister_client_groups(struct ib_device *ibdev, u32 port_num, const struct attribute_group **groups); int ib_device_set_netns_put(struct sk_buff *skb, - struct ib_device *dev, u32 ns_fd); + struct ib_device *dev, u32 ns_fd, const char *name, + struct netlink_ext_ack *extack); int rdma_nl_net_init(struct rdma_dev_net *rnet); void rdma_nl_net_exit(struct rdma_dev_net *rnet); diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index a9e189194c13..a2c85840c501 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -234,7 +234,6 @@ static void rdma_counter_free(struct rdma_counter *counter) mutex_unlock(&port_counter->lock); - rdma_restrack_del(&counter->res); rdma_free_hw_stats_struct(counter->stats); kfree(counter); } @@ -329,6 +328,7 @@ static void counter_release(struct kref *kref) counter = container_of(kref, struct rdma_counter, kref); counter_history_stat_update(counter); + rdma_restrack_del(&counter->res); counter->device->ops.counter_dealloc(counter); rdma_counter_free(counter); } @@ -490,7 +490,8 @@ static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, return NULL; counter = container_of(res, struct rdma_counter, res); - kref_get(&counter->kref); + if (!kref_get_unless_zero(&counter->kref)) + counter = NULL; rdma_restrack_put(res); return counter; diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index 3d7b6cddd131..12304c9a9403 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -327,6 +327,7 @@ void ib_free_cq(struct ib_cq *cq) if (WARN_ON_ONCE(cq->cqe_used)) return; + rdma_restrack_del(&cq->res); if (cq->device->ops.pre_destroy_cq) { ret = cq->device->ops.pre_destroy_cq(cq); WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail"); @@ -353,7 +354,6 @@ void ib_free_cq(struct ib_cq *cq) else ret = cq->device->ops.destroy_cq(cq, NULL); WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); - rdma_restrack_del(&cq->res); kfree(cq->wc); kfree(cq); } @@ -393,8 +393,7 @@ static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes, * a reasonable batch size so that we can share CQs between * multiple users instead of allocating a larger number of CQs. */ - nr_cqes = min_t(unsigned int, dev->attrs.max_cqe, - max(nr_cqes, IB_MAX_SHARED_CQ_SZ)); + nr_cqes = min(dev->attrs.max_cqe, max(nr_cqes, IB_MAX_SHARED_CQ_SZ)); nr_cqs = min_t(unsigned int, dev->num_comp_vectors, num_online_cpus()); for (i = 0; i < nr_cqs; i++) { cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx); diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index d954eda63134..7a3ed5ecac00 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -129,7 +129,7 @@ static DECLARE_RWSEM(rdma_nets_rwsem); bool ib_devices_shared_netns = true; module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444); MODULE_PARM_DESC(netns_mode, - "Share device among net namespaces; default=1 (shared)"); + "Share device among net namespaces; default=1 (shared). In exclusive mode device names are unique per net namespace"); /** * rdma_dev_access_netns() - Return whether an rdma device can be accessed * from a specified net namespace or not. @@ -268,7 +268,8 @@ static struct notifier_block ibdev_lsm_nb = { }; static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net, - struct net *net); + struct net *net, const char *requested_name, + const char *fallback_pattern); /* Pointer to the RCU head at the start of the ib_port_data array */ struct ib_port_data_rcu { @@ -351,13 +352,15 @@ void ib_device_put(struct ib_device *device) } EXPORT_SYMBOL(ib_device_put); -static struct ib_device *__ib_device_get_by_name(const char *name) +static struct ib_device *__ib_device_get_by_name(const char *name, + const struct net *net) { struct ib_device *device; unsigned long index; xa_for_each (&devices, index, device) - if (!strcmp(name, dev_name(&device->dev))) + if (rdma_dev_access_netns(device, net) && + !strcmp(name, dev_name(&device->dev))) return device; return NULL; @@ -395,7 +398,7 @@ int ib_device_rename(struct ib_device *ibdev, const char *name) return 0; } - if (__ib_device_get_by_name(name)) { + if (__ib_device_get_by_name(name, rdma_dev_net(ibdev))) { up_write(&devices_rwsem); return -EEXIST; } @@ -435,7 +438,13 @@ int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim) return 0; } -static int alloc_name(struct ib_device *ibdev, const char *name) +/* + * Pick a free index for the '%d' style @name pattern within net namespace + * @net. Returns the index on success or a negative errno. The caller builds + * the final unique device name from the returned index. + */ +static int __alloc_name_id(struct net *net, const char *name, + const struct ib_device *skip) { struct ib_device *device; unsigned long index; @@ -448,6 +457,8 @@ static int alloc_name(struct ib_device *ibdev, const char *name) xa_for_each (&devices, index, device) { char buf[IB_DEVICE_NAME_MAX]; + if (device == skip || !rdma_dev_access_netns(device, net)) + continue; if (sscanf(dev_name(&device->dev), name, &i) != 1) continue; if (i < 0 || i >= INT_MAX) @@ -462,15 +473,27 @@ static int alloc_name(struct ib_device *ibdev, const char *name) } rc = ida_alloc(&inuse, GFP_KERNEL); - if (rc < 0) - goto out; - - rc = dev_set_name(&ibdev->dev, name, rc); out: ida_destroy(&inuse); return rc; } +static int alloc_name_id(struct net *net, const char *name) +{ + return __alloc_name_id(net, name, NULL); +} + +static int alloc_name(struct ib_device *ibdev, const char *name) +{ + int id; + + id = alloc_name_id(rdma_dev_net(ibdev), name); + if (id < 0) + return id; + + return dev_set_name(&ibdev->dev, name, id); +} + static void ib_device_release(struct device *device) { struct ib_device *dev = container_of(device, struct ib_device, dev); @@ -1151,8 +1174,17 @@ static void rdma_dev_exit_net(struct net *net) /* * If the real device is in the NS then move it back to init. + * Provide a fallback pattern so a name conflict in init_net + * cannot make the teardown move fail. */ - rdma_dev_change_netns(dev, net, &init_net); + if (net_eq(net, read_pnet(&dev->coredev.rdma_net))) { + ret = rdma_dev_change_netns(dev, net, &init_net, + NULL, "ibdev%d"); + if (ret && ret != -ENODEV) + WARN(1, + "Failed to move RDMA device %s to init_net on netns exit: %d\n", + dev_name(&dev->dev), ret); + } put_device(&dev->dev); down_read(&devices_rwsem); @@ -1206,7 +1238,8 @@ static __net_init int rdma_dev_init_net(struct net *net) } /* - * Assign the unique string device name and the unique device index. This is + * Assign the unique string device name and the unique device index. The device + * name is unique within the net namespace the device is assigned to. This is * undone by ib_dealloc_device. */ static int assign_name(struct ib_device *device, const char *name) @@ -1223,7 +1256,8 @@ static int assign_name(struct ib_device *device, const char *name) if (ret) goto out; - if (__ib_device_get_by_name(dev_name(&device->dev))) { + if (__ib_device_get_by_name(dev_name(&device->dev), + rdma_dev_net(device))) { ret = -ENFILE; goto out; } @@ -1388,8 +1422,9 @@ out: /** * ib_register_device - Register an IB device with IB core * @device: Device to register - * @name: unique string device name. This may include a '%' which will - * cause a unique index to be added to the passed device name. + * @name: device name, unique within the device's net namespace. This may + * include a '%' which will cause a unique index to be added to the + * passed device name. * @dma_device: pointer to a DMA-capable device. If %NULL, then the IB * device will be used. In this case the caller should fully * setup the ibdev for DMA. This usually means using dma_virt_ops. @@ -1670,14 +1705,83 @@ void ib_unregister_device_queued(struct ib_device *ib_dev) } EXPORT_SYMBOL(ib_unregister_device_queued); +static bool rdma_dev_name_in_netns(struct ib_device *skip, struct net *net, + const char *name) +{ + struct ib_device *device; + unsigned long index; + + lockdep_assert_held_write(&devices_rwsem); + + xa_for_each(&devices, index, device) + if (device != skip && + rdma_dev_access_netns(device, net) && + !strcmp(name, dev_name(&device->dev))) + return true; + + return false; +} + +/* + * Choose the name @device should use in net namespace @net. @requested_name + * is used as a literal device name when set. Otherwise keep the current name + * when it is free, or use a trusted '%d' @fallback_pattern for teardown. The + * caller must hold the write side of devices_rwsem. + */ +static int rdma_dev_pick_netns_name(struct ib_device *device, struct net *net, + const char *requested_name, + const char *fallback_pattern, + char *buf, size_t buf_len, + const char **new_name) +{ + int id; + + lockdep_assert_held_write(&devices_rwsem); + + if (requested_name) { + if (!rdma_dev_name_in_netns(device, net, requested_name)) { + *new_name = requested_name; + return 0; + } + + return -EEXIST; + } + + if (!rdma_dev_name_in_netns(device, net, dev_name(&device->dev))) { + *new_name = dev_name(&device->dev); + return 0; + } + + if (!fallback_pattern) + return -EEXIST; + + snprintf(buf, buf_len, "ibdev%u", device->index); + if (!rdma_dev_name_in_netns(device, net, buf)) { + *new_name = buf; + return 0; + } + + id = __alloc_name_id(net, fallback_pattern, device); + if (id < 0) + return id; + snprintf(buf, buf_len, fallback_pattern, id); + *new_name = buf; + return 0; +} + /* * The caller must pass in a device that has the kref held and the refcount * released. If the device is in cur_net and still registered then it is moved * into net. + * + * Naming rules are handled by rdma_dev_pick_netns_name(). */ static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net, - struct net *net) + struct net *net, const char *requested_name, + const char *fallback_pattern) { + char buf[IB_DEVICE_NAME_MAX]; + const char *new_name; int ret2 = -EINVAL; int ret; @@ -1694,30 +1798,65 @@ static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net, goto out; } + if (!fallback_pattern) { + /* + * Reject a predictable name conflict before tearing anything + * down, so a doomed user move does not disable a live device. + */ + down_write(&devices_rwsem); + ret = rdma_dev_pick_netns_name(device, net, requested_name, + fallback_pattern, buf, + sizeof(buf), &new_name); + up_write(&devices_rwsem); + if (ret) + goto out; + } + kobject_uevent(&device->dev.kobj, KOBJ_REMOVE); disable_device(device); /* - * At this point no one can be using the device, so it is safe to - * change the namespace. + * Recompute the destination name under the write side of devices_rwsem + * now that the device is disabled, closing races with a concurrent + * registration or rename, then publish the new namespace at the sysfs + * level. */ - write_pnet(&device->coredev.rdma_net, net); + down_write(&devices_rwsem); + ret = rdma_dev_pick_netns_name(device, net, requested_name, + fallback_pattern, buf, sizeof(buf), + &new_name); + if (ret) { + if (fallback_pattern) { + WARN(1, + "%s: failed to pick device name during namespace teardown: %d\n", + __func__, ret); + write_pnet(&device->coredev.rdma_net, net); + ret = 0; + } + goto rename_done; + } - down_read(&devices_rwsem); - /* - * Currently rdma devices are system wide unique. So the device name - * is guaranteed free in the new namespace. Publish the new namespace - * at the sysfs level. - */ - ret = device_rename(&device->dev, dev_name(&device->dev)); - up_read(&devices_rwsem); + write_pnet(&device->coredev.rdma_net, net); + ret = device_rename(&device->dev, new_name); if (ret) { - dev_warn(&device->dev, - "%s: Couldn't rename device after namespace change\n", - __func__); - /* Try and put things back and re-enable the device */ - write_pnet(&device->coredev.rdma_net, cur_net); + if (fallback_pattern) { + WARN(1, + "%s: failed to rename device during namespace teardown: %d\n", + __func__, ret); + ret = 0; + } else { + dev_warn(&device->dev, + "%s: Couldn't rename device after namespace change\n", + __func__); + /* Try and put things back and re-enable the device */ + write_pnet(&device->coredev.rdma_net, cur_net); + } + } else { + strscpy(device->name, dev_name(&device->dev), + IB_DEVICE_NAME_MAX); } +rename_done: + up_write(&devices_rwsem); ret2 = enable_device_and_get(device); if (ret2) { @@ -1740,36 +1879,72 @@ out: } int ib_device_set_netns_put(struct sk_buff *skb, - struct ib_device *dev, u32 ns_fd) + struct ib_device *dev, u32 ns_fd, const char *name, + struct netlink_ext_ack *extack) { struct net *net; int ret; net = get_net_ns_by_fd(ns_fd); if (IS_ERR(net)) { + NL_SET_ERR_MSG(extack, "Invalid target net namespace fd"); ret = PTR_ERR(net); goto net_err; } if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) { + NL_SET_ERR_MSG(extack, + "Missing CAP_NET_ADMIN in the target net namespace"); ret = -EPERM; goto ns_err; } /* + * Moving a device to the namespace it already lives in is a no-op; a + * supplied name still renames it in place. + */ + if (net_eq(net, read_pnet(&dev->coredev.rdma_net))) { + ret = name ? ib_device_rename(dev, name) : 0; + + if (ret == -EEXIST) + NL_SET_ERR_MSG(extack, + "Device name already exists in the target net namespace"); + else if (ret == -EINVAL && name) + NL_SET_ERR_MSG(extack, + "Unable to use requested device name in the target net namespace"); + goto ns_err; + } + + /* * All the ib_clients, including uverbs, are reset when the namespace is * changed and this cannot be blocked waiting for userspace to do * something, so disassociation is mandatory. */ - if (!dev->ops.disassociate_ucontext || ib_devices_shared_netns) { + if (ib_devices_shared_netns) { + NL_SET_ERR_MSG(extack, + "Cannot change net namespace of RDMA device in shared netns mode"); + ret = -EOPNOTSUPP; + goto ns_err; + } + + if (!dev->ops.disassociate_ucontext) { + NL_SET_ERR_MSG(extack, + "Device does not support namespace changes (no disassociate support)"); ret = -EOPNOTSUPP; goto ns_err; } get_device(&dev->dev); ib_device_put(dev); - ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net); + ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net, name, + NULL); put_device(&dev->dev); + if (ret == -EEXIST) + NL_SET_ERR_MSG(extack, + "Device name already exists in the target net namespace"); + else if (ret == -EINVAL && name) + NL_SET_ERR_MSG(extack, + "Unable to use requested device name in the target net namespace"); put_net(net); return ret; @@ -2742,6 +2917,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, create_ah); SET_DEVICE_OP(dev_ops, create_counters); SET_DEVICE_OP(dev_ops, create_cq); + SET_DEVICE_OP(dev_ops, create_comp_cntr); SET_DEVICE_OP(dev_ops, create_user_cq); SET_DEVICE_OP(dev_ops, create_flow); SET_DEVICE_OP(dev_ops, create_qp); @@ -2762,6 +2938,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, destroy_ah); SET_DEVICE_OP(dev_ops, destroy_counters); SET_DEVICE_OP(dev_ops, destroy_cq); + SET_DEVICE_OP(dev_ops, destroy_comp_cntr); SET_DEVICE_OP(dev_ops, destroy_flow); SET_DEVICE_OP(dev_ops, destroy_flow_action); SET_DEVICE_OP(dev_ops, destroy_qp); @@ -2813,6 +2990,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, modify_hw_stat); SET_DEVICE_OP(dev_ops, modify_port); SET_DEVICE_OP(dev_ops, modify_qp); + SET_DEVICE_OP(dev_ops, qp_attach_comp_cntr); SET_DEVICE_OP(dev_ops, modify_srq); SET_DEVICE_OP(dev_ops, modify_wq); SET_DEVICE_OP(dev_ops, peek_cq); @@ -2826,6 +3004,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, post_srq_recv); SET_DEVICE_OP(dev_ops, process_mad); SET_DEVICE_OP(dev_ops, query_ah); + SET_DEVICE_OP(dev_ops, query_comp_cntr_caps); SET_DEVICE_OP(dev_ops, query_device); SET_DEVICE_OP(dev_ops, query_gid); SET_DEVICE_OP(dev_ops, query_pkey); @@ -2836,12 +3015,14 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, query_ucontext); SET_DEVICE_OP(dev_ops, rdma_netdev_get_params); SET_DEVICE_OP(dev_ops, read_counters); + SET_DEVICE_OP(dev_ops, read_comp_cntr); SET_DEVICE_OP(dev_ops, reg_dm_mr); SET_DEVICE_OP(dev_ops, reg_user_mr); SET_DEVICE_OP(dev_ops, reg_user_mr_dmabuf); SET_DEVICE_OP(dev_ops, req_notify_cq); SET_DEVICE_OP(dev_ops, rereg_user_mr); SET_DEVICE_OP(dev_ops, resize_user_cq); + SET_DEVICE_OP(dev_ops, modify_comp_cntr); SET_DEVICE_OP(dev_ops, set_vf_guid); SET_DEVICE_OP(dev_ops, set_vf_link_state); SET_DEVICE_OP(dev_ops, ufile_hw_cleanup); @@ -2850,6 +3031,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_OBJ_SIZE(dev_ops, ib_ah); SET_OBJ_SIZE(dev_ops, ib_counters); SET_OBJ_SIZE(dev_ops, ib_cq); + SET_OBJ_SIZE(dev_ops, ib_comp_cntr); SET_OBJ_SIZE(dev_ops, ib_dmah); SET_OBJ_SIZE(dev_ops, ib_mw); SET_OBJ_SIZE(dev_ops, ib_pd); diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c index 4625abd29ac0..1b10f2973ad9 100644 --- a/drivers/infiniband/core/iwpm_msg.c +++ b/drivers/infiniband/core/iwpm_msg.c @@ -268,7 +268,7 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client) if (ret) goto query_mapping_error; - /* If flags are required and we're not V4, then return a quite error */ + /* If flags are required and we're not V4, then return a quiet error */ if (pm_msg->flags && iwpm_ulib_version == IWPM_UABI_VERSION_MIN) { ret = -EINVAL; goto query_mapping_error_nowarn; diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 02a0a9c0a4a6..a4014a230639 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -188,6 +188,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY] = { .type = NLA_U64 }, + [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX] = { .type = NLA_U64 }, }; static int put_driver_name_print_type(struct sk_buff *msg, const char *name, @@ -413,7 +414,7 @@ out: } static int fill_res_info_entry(struct sk_buff *msg, - const char *name, u64 curr) + const char *name, u64 curr, u64 max) { struct nlattr *entry_attr; @@ -427,6 +428,9 @@ static int fill_res_info_entry(struct sk_buff *msg, if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, RDMA_NLDEV_ATTR_PAD)) goto err; + if (max && nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, + max, RDMA_NLDEV_ATTR_PAD)) + goto err; nla_nest_end(msg, entry_attr); return 0; @@ -447,10 +451,13 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device, [RDMA_RESTRACK_MR] = "mr", [RDMA_RESTRACK_CTX] = "ctx", [RDMA_RESTRACK_SRQ] = "srq", + [RDMA_RESTRACK_COMP_CNTR] = "comp_cntr", }; + struct ib_comp_cntr_caps comp_cntr_caps = {}; struct nlattr *table_attr; - int ret, i, curr; + u64 curr, max; + int ret, i; if (fill_nldev_handle(msg, device)) return -EMSGSIZE; @@ -459,11 +466,36 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device, if (!table_attr) return -EMSGSIZE; + if (device->ops.query_comp_cntr_caps) + device->ops.query_comp_cntr_caps(device, &comp_cntr_caps, NULL); + for (i = 0; i < RDMA_RESTRACK_MAX; i++) { if (!names[i]) continue; curr = rdma_restrack_count(device, i, show_details); - ret = fill_res_info_entry(msg, names[i], curr); + switch (i) { + case RDMA_RESTRACK_QP: + max = device->attrs.max_qp; + break; + case RDMA_RESTRACK_CQ: + max = device->attrs.max_cq; + break; + case RDMA_RESTRACK_MR: + max = device->attrs.max_mr; + break; + case RDMA_RESTRACK_PD: + max = device->attrs.max_pd; + break; + case RDMA_RESTRACK_SRQ: + max = device->attrs.max_srq; + break; + case RDMA_RESTRACK_COMP_CNTR: + max = comp_cntr_caps.max_counters; + break; + default: + max = 0; + } + ret = fill_res_info_entry(msg, names[i], curr, max); if (ret) goto err; } @@ -1154,6 +1186,24 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh, if (!device) return -EINVAL; + if (tb[RDMA_NLDEV_NET_NS_FD]) { + char name[IB_DEVICE_NAME_MAX] = {}; + u32 ns_fd; + + if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) { + nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME], + IB_DEVICE_NAME_MAX); + if (strlen(name) == 0) { + err = -EINVAL; + goto done; + } + } + ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]); + err = ib_device_set_netns_put(skb, device, ns_fd, + name[0] ? name : NULL, extack); + goto put_done; + } + if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) { char name[IB_DEVICE_NAME_MAX] = {}; @@ -1167,14 +1217,6 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh, goto done; } - if (tb[RDMA_NLDEV_NET_NS_FD]) { - u32 ns_fd; - - ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]); - err = ib_device_set_netns_put(skb, device, ns_fd); - goto put_done; - } - if (tb[RDMA_NLDEV_ATTR_DEV_DIM]) { u8 use_dim; @@ -2133,6 +2175,11 @@ static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[], nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS], rem) { + if (nla_len(entry_attr) != sizeof(u32)) { + ret = -EINVAL; + goto out; + } + index = nla_get_u32(entry_attr); if ((index >= stats->num_counters) || !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) { diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 56121103e9f4..2b91e8527287 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -159,6 +159,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile); extern const struct uapi_definition uverbs_def_obj_async_fd[]; extern const struct uapi_definition uverbs_def_obj_counters[]; +extern const struct uapi_definition uverbs_def_obj_comp_cntr[]; extern const struct uapi_definition uverbs_def_obj_cq[]; extern const struct uapi_definition uverbs_def_obj_device[]; extern const struct uapi_definition uverbs_def_obj_dm[]; diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index cfee2071586c..f89a81dad72f 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -61,7 +61,7 @@ void rdma_restrack_clean(struct ib_device *dev) * @type: actual type of object to operate * @show_details: count driver specific objects */ -int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, +u32 rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, bool show_details) { struct rdma_restrack_root *rt = &dev->res[type]; @@ -104,6 +104,8 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) return container_of(res, struct ib_srq, res)->device; case RDMA_RESTRACK_DMAH: return container_of(res, struct ib_dmah, res)->device; + case RDMA_RESTRACK_COMP_CNTR: + return container_of(res, struct ib_comp_cntr, res)->device; default: WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type); return NULL; @@ -129,6 +131,46 @@ static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, res->user = true; } +static struct rdma_restrack_root *res_to_rt(struct rdma_restrack_entry *res) +{ + struct ib_device *dev = res_to_dev(res); + + if (WARN_ON(!dev)) + return NULL; + + return &dev->res[res->type]; +} + +static void restrack_drain_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, + GFP_KERNEL); + WARN_ON(old != res); + } + + rdma_restrack_put(res); + wait_for_completion(&res->comp); +} + +static void restrack_restore_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + reinit_completion(&res->comp); + kref_init(&res->kref); + + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, + GFP_KERNEL); + WARN_ON(old); + } +} + /** * rdma_restrack_set_name() - set the task for this resource * @res: resource entry @@ -177,22 +219,23 @@ void rdma_restrack_new(struct rdma_restrack_entry *res, EXPORT_SYMBOL(rdma_restrack_new); /** - * rdma_restrack_add() - add object to the resource tracking database + * rdma_restrack_add() - add object to the resource tracking database. + * If this resource reuses an ID of a resource that was already destroyed + * after calling rdma_restrack_begin() but didn't yet call + * rdma_restrack_commit_del() it can result in an untracked QP. * @res: resource entry */ void rdma_restrack_add(struct rdma_restrack_entry *res) { - struct ib_device *dev = res_to_dev(res); struct rdma_restrack_root *rt; int ret = 0; - if (!dev) - return; - if (res->no_track) goto out; - rt = &dev->res[res->type]; + rt = res_to_rt(res); + if (!rt) + return; if (res->type == RDMA_RESTRACK_QP) { /* Special case to ensure that LQPN points to right QP */ @@ -229,6 +272,28 @@ out: } EXPORT_SYMBOL(rdma_restrack_add); +/** + * rdma_restrack_abort_del() - re-add object to the resource tracking database + * it can only be used after rdma_restrack_begin_del(). + * @res: resource entry + */ +void rdma_restrack_abort_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_restore_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_abort_del); + int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) { return kref_get_unless_zero(&res->kref); @@ -265,7 +330,7 @@ static void restrack_release(struct kref *kref) struct rdma_restrack_entry *res; res = container_of(kref, struct rdma_restrack_entry, kref); - if (res->task) { + if (res->task && !res->valid) { put_task_struct(res->task); res->task = NULL; } @@ -291,37 +356,20 @@ EXPORT_SYMBOL(rdma_restrack_put); */ void rdma_restrack_sync(struct rdma_restrack_entry *res) { - struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct task_struct *task; - struct ib_device *dev; if (!res->valid || res->no_track) return; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) return; - old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, GFP_KERNEL); - if (WARN_ON(old != res)) - return; - - task = res->task; - if (task) - get_task_struct(task); - rdma_restrack_put(res); - wait_for_completion(&res->comp); - reinit_completion(&res->comp); - if (task) - res->task = task; - kref_init(&res->kref); - - xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, GFP_KERNEL); + restrack_drain_res(rt, res); + restrack_restore_res(rt, res); } EXPORT_SYMBOL(rdma_restrack_sync); @@ -333,7 +381,6 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) { struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct ib_device *dev; if (!res->valid) { if (res->task) { @@ -346,12 +393,10 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) if (res->no_track) goto out; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; - old = xa_erase(&rt->xa, res->id); WARN_ON(old != res); @@ -359,5 +404,61 @@ out: res->valid = false; rdma_restrack_put(res); wait_for_completion(&res->comp); + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } } EXPORT_SYMBOL(rdma_restrack_del); + +/** + * rdma_restrack_begin_del() - invalidate the object from the resource tracking + * database but preserve its index in the array. + * Since this preserves the index in the array until rdma_restrack_commit_del() + * is called, if rdma_restrack_add() is called in between with an old QP ID it + * can result in an untracked QP. + * @res: resource entry + */ +void rdma_restrack_begin_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_drain_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_begin_del); + +/** + * rdma_restrack_commit_del() - delete object from the resource tracking + * database and free the task. + * @res: resource entry + */ +void rdma_restrack_commit_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt; + + if (!res->valid || res->no_track) + goto out; + + rt = res_to_rt(res); + if (!rt) + return; + + xa_erase(&rt->xa, res->id); + +out: + res->valid = false; + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } +} +EXPORT_SYMBOL(rdma_restrack_commit_del); diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 75b8d1005a98..2df78e084e10 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -26,8 +26,11 @@ struct rdma_restrack_root { int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); +void rdma_restrack_abort_del(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); void rdma_restrack_sync(struct rdma_restrack_entry *res); +void rdma_restrack_begin_del(struct rdma_restrack_entry *res); +void rdma_restrack_commit_del(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/security.c b/drivers/infiniband/core/security.c index 9af31d1d9d70..a82c46965416 100644 --- a/drivers/infiniband/core/security.c +++ b/drivers/infiniband/core/security.c @@ -700,6 +700,12 @@ int ib_mad_agent_security_setup(struct ib_mad_agent *agent, if (qp_type != IB_QPT_SMI) return 0; + /* + * SELinux labels an endport by (device name, port) from a global + * policy. If devices in different net namespaces share a name, they get + * the same label; distinguishing them would need net namespace support + * in the policy language and tooling. + */ spin_lock(&mad_agent_list_lock); ret = security_ib_endport_manage_subnet(agent->security, dev_name(&agent->device->dev), diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 73498723a5d5..81f44dadfa52 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -209,7 +209,8 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, mmgrab(mm); - page_list = (struct page **) __get_free_page(GFP_KERNEL); + /* TODO: switch to "fast and as large as possible" allocation helper */ + page_list = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!page_list) { ret = -ENOMEM; goto umem_kfree; @@ -269,7 +270,7 @@ umem_release: __ib_umem_release(device, umem, 0); atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm); out: - free_page((unsigned long) page_list); + kfree(page_list); umem_kfree: if (ret) { mmdrop(umem->owning_mm); diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index ad023c2d84d8..39b5564a4c35 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -181,7 +181,7 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, } EXPORT_SYMBOL(ib_umem_dmabuf_get); -static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { +static const struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { .allow_peer2peer = true, }; @@ -205,7 +205,7 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach) umem_dmabuf->revoked = 1; } -static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = { +static const struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = { .allow_peer2peer = true, .invalidate_mappings = ib_umem_dmabuf_revoke_locked, }; diff --git a/drivers/infiniband/core/uverbs_std_types_comp_cntr.c b/drivers/infiniband/core/uverbs_std_types_comp_cntr.c new file mode 100644 index 000000000000..2e7de84d94a6 --- /dev/null +++ b/drivers/infiniband/core/uverbs_std_types_comp_cntr.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#include <rdma/uverbs_std_types.h> +#include "rdma_core.h" +#include "uverbs.h" +#include "restrack.h" + +static int uverbs_free_comp_cntr(struct ib_uobject *uobject, enum rdma_remove_reason why, + struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr *cc = uobject->object; + int ret; + + if (atomic_read(&cc->usecnt)) + return -EBUSY; + + rdma_restrack_begin_del(&cc->res); + ret = cc->device->ops.destroy_comp_cntr(cc); + if (ret) { + rdma_restrack_abort_del(&cc->res); + return ret; + } + + rdma_restrack_commit_del(&cc->res); + kfree(cc); + return 0; +} + +static int UVERBS_HANDLER(UVERBS_METHOD_COMP_CNTR_CREATE)(struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, + UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE); + struct ib_device *ib_dev = attrs->context->device; + struct ib_comp_cntr *cc; + int ret; + + if (!ib_dev->ops.create_comp_cntr || + !ib_dev->ops.destroy_comp_cntr || + !ib_dev->ops.qp_attach_comp_cntr) + return -EOPNOTSUPP; + + cc = rdma_zalloc_drv_obj(ib_dev, ib_comp_cntr); + if (!cc) + return -ENOMEM; + + cc->device = ib_dev; + cc->uobject = uobj; + + rdma_restrack_new(&cc->res, RDMA_RESTRACK_COMP_CNTR); + rdma_restrack_set_name(&cc->res, NULL); + + ret = ib_dev->ops.create_comp_cntr(cc, attrs); + if (ret) + goto err_free; + + uobj->object = cc; + rdma_restrack_add(&cc->res); + uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE); + return 0; + +err_free: + rdma_restrack_put(&cc->res); + kfree(cc); + return ret; +} + +static int UVERBS_HANDLER(UVERBS_METHOD_COMP_CNTR_MODIFY)(struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr *cc = uverbs_attr_get_obj(attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE); + enum ib_comp_cntr_modify_op op; + enum ib_comp_cntr_entry entry; + u64 value; + int ret; + + if (!cc->device->ops.modify_comp_cntr) + return -EOPNOTSUPP; + + ret = uverbs_get_const(&entry, attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY); + if (ret) + return ret; + + ret = uverbs_get_const(&op, attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_OP); + if (ret) + return ret; + + ret = uverbs_copy_from(&value, attrs, UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE); + if (ret) + return ret; + + return cc->device->ops.modify_comp_cntr(cc, entry, op, value); +} + +static int UVERBS_HANDLER(UVERBS_METHOD_COMP_CNTR_READ)(struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr *cc = uverbs_attr_get_obj(attrs, UVERBS_ATTR_READ_COMP_CNTR_HANDLE); + enum ib_comp_cntr_entry entry; + u64 value = 0; + int ret; + + if (!cc->device->ops.read_comp_cntr) + return -EOPNOTSUPP; + + ret = uverbs_get_const(&entry, attrs, UVERBS_ATTR_READ_COMP_CNTR_ENTRY); + if (ret) + return ret; + + ret = cc->device->ops.read_comp_cntr(cc, entry, &value); + if (ret) + return ret; + + return uverbs_copy_to(attrs, UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, &value, sizeof(value)); +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_COMP_CNTR_CREATE, + UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_NEW, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD_DESTROY( + UVERBS_METHOD_COMP_CNTR_DESTROY, + UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_DESTROY, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_COMP_CNTR_MODIFY, + UVERBS_ATTR_IDR(UVERBS_ATTR_MODIFY_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_WRITE, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_MODIFY_COMP_CNTR_ENTRY, + enum ib_uverbs_comp_cntr_entry, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_MODIFY_COMP_CNTR_OP, + enum ib_uverbs_comp_cntr_modify_op, + UA_MANDATORY), + UVERBS_ATTR_PTR_IN(UVERBS_ATTR_MODIFY_COMP_CNTR_VALUE, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_COMP_CNTR_READ, + UVERBS_ATTR_IDR(UVERBS_ATTR_READ_COMP_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_READ_COMP_CNTR_ENTRY, + enum ib_uverbs_comp_cntr_entry, + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_READ_COMP_CNTR_RESP_VALUE, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_OBJECT( + UVERBS_OBJECT_COMP_CNTR, + UVERBS_TYPE_ALLOC_IDR(uverbs_free_comp_cntr), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_CREATE), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_DESTROY), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_MODIFY), + &UVERBS_METHOD(UVERBS_METHOD_COMP_CNTR_READ)); + +const struct uapi_definition uverbs_def_obj_comp_cntr[] = { + UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_COMP_CNTR, + UAPI_DEF_OBJ_NEEDS_FN(destroy_comp_cntr)), + {} +}; diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index 12ca15739cd2..ce0a7de00405 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -472,6 +472,42 @@ out: return ret; } +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_COMP_CNTR_CAPS)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_comp_cntr_caps caps = {}; + struct ib_ucontext *ucontext; + struct ib_device *ib_dev; + int ret; + + ucontext = ib_uverbs_get_ucontext(attrs); + if (IS_ERR(ucontext)) + return PTR_ERR(ucontext); + ib_dev = ucontext->device; + + if (!ib_dev->ops.query_comp_cntr_caps) + return -EOPNOTSUPP; + + ret = ib_dev->ops.query_comp_cntr_caps(ib_dev, &caps, attrs); + if (ret) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + &caps.max_counters, sizeof(caps.max_counters)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + &caps.max_value, sizeof(caps.max_value)); + if (IS_UVERBS_COPY_ERR(ret)) + return ret; + + ret = uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, + &caps.supported_qp_attach_ops, + sizeof(caps.supported_qp_attach_ops)); + return IS_UVERBS_COPY_ERR(ret) ? ret : 0; +} + DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_GET_CONTEXT, UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, @@ -542,6 +578,18 @@ DECLARE_UVERBS_NAMED_METHOD( netdev_ifindex), UA_MANDATORY)); +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QUERY_COMP_CNTR_CAPS, + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_COUNTERS, + UVERBS_ATTR_TYPE(u32), + UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_MAX_VALUE, + UVERBS_ATTR_TYPE(u64), + UA_OPTIONAL), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_COMP_CNTR_CAPS_SUPPORTED_QP_ATTACH_OPS, + UVERBS_ATTR_TYPE(u32), + UA_OPTIONAL)); + DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, &UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), @@ -550,7 +598,8 @@ DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT_SPEED), &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_TABLE), - &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY)); + &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY), + &UVERBS_METHOD(UVERBS_METHOD_QUERY_COMP_CNTR_CAPS)); const struct uapi_definition uverbs_def_obj_device[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE), diff --git a/drivers/infiniband/core/uverbs_std_types_dmah.c b/drivers/infiniband/core/uverbs_std_types_dmah.c index 97101e093826..9873ab49a601 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmah.c +++ b/drivers/infiniband/core/uverbs_std_types_dmah.c @@ -18,11 +18,14 @@ static int uverbs_free_dmah(struct ib_uobject *uobject, if (atomic_read(&dmah->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&dmah->res); ret = dmah->device->ops.dealloc_dmah(dmah, attrs); - if (ret) + if (ret) { + rdma_restrack_abort_del(&dmah->res); return ret; + } - rdma_restrack_del(&dmah->res); + rdma_restrack_commit_del(&dmah->res); kfree(dmah); return 0; } diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c index 570b9656801d..0c72f801e0d3 100644 --- a/drivers/infiniband/core/uverbs_std_types_mr.c +++ b/drivers/infiniband/core/uverbs_std_types_mr.c @@ -364,7 +364,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)( dmah, attrs); else mr = pd->device->ops.reg_user_mr(pd, addr, length, iova, - access_flags, dmah, NULL); + access_flags, dmah, + &attrs->driver_udata); if (IS_ERR(mr)) return PTR_ERR(mr); @@ -527,7 +528,8 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY), UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY, UVERBS_ATTR_TYPE(u32), - UA_MANDATORY)); + UA_MANDATORY), + UVERBS_ATTR_UHW()); DECLARE_UVERBS_NAMED_METHOD_DESTROY( UVERBS_METHOD_MR_DESTROY, diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c index 5767607dd420..30fc20fb251f 100644 --- a/drivers/infiniband/core/uverbs_std_types_qp.c +++ b/drivers/infiniband/core/uverbs_std_types_qp.c @@ -372,11 +372,76 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_qp_resp), UA_MANDATORY)); +static int UVERBS_HANDLER(UVERBS_METHOD_QP_ATTACH_COMP_CNTR)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *qp_uobj = uverbs_attr_get_uobject( + attrs, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE); + struct ib_comp_cntr *cc = uverbs_attr_get_obj( + attrs, UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE); + struct ib_qp_attach_comp_cntr_attr attr = {}; + struct ib_qp *qp = qp_uobj->object; + int ret; + + if (!cc->device->ops.qp_attach_comp_cntr) + return -EOPNOTSUPP; + + if (qp->real_qp != qp) + return -EINVAL; + + ret = uverbs_get_flags32(&attr.op_mask, attrs, + UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_SEND | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RECV | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_READ | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE | + IB_UVERBS_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE); + if (ret) + return ret; + + if (!attr.op_mask) + return -EINVAL; + + if (attr.op_mask & qp->comp_cntr_op_mask) + return -EBUSY; + + ret = xa_err(xa_store(&qp->comp_cntrs, attr.op_mask, cc, GFP_KERNEL)); + if (ret) + return ret; + + ret = qp->device->ops.qp_attach_comp_cntr(qp, cc, &attr); + if (ret) { + xa_erase(&qp->comp_cntrs, attr.op_mask); + return ret; + } + + atomic_inc(&cc->usecnt); + qp->comp_cntr_op_mask |= attr.op_mask; + + return 0; +} + +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QP_ATTACH_COMP_CNTR, + UVERBS_ATTR_IDR(UVERBS_ATTR_QP_ATTACH_COMP_CNTR_HANDLE, + UVERBS_OBJECT_QP, + UVERBS_ACCESS_WRITE, + UA_MANDATORY), + UVERBS_ATTR_IDR(UVERBS_ATTR_QP_ATTACH_COMP_CNTR_CNTR_HANDLE, + UVERBS_OBJECT_COMP_CNTR, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_QP_ATTACH_COMP_CNTR_OP_MASK, + enum ib_uverbs_qp_attach_comp_cntr_op, + UA_MANDATORY)); + DECLARE_UVERBS_NAMED_OBJECT( UVERBS_OBJECT_QP, UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp), &UVERBS_METHOD(UVERBS_METHOD_QP_CREATE), - &UVERBS_METHOD(UVERBS_METHOD_QP_DESTROY)); + &UVERBS_METHOD(UVERBS_METHOD_QP_DESTROY), + &UVERBS_METHOD(UVERBS_METHOD_QP_ATTACH_COMP_CNTR)); const struct uapi_definition uverbs_def_obj_qp[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP, diff --git a/drivers/infiniband/core/uverbs_std_types_srq.c b/drivers/infiniband/core/uverbs_std_types_srq.c index e5513f828bdc..0421bdd225df 100644 --- a/drivers/infiniband/core/uverbs_std_types_srq.c +++ b/drivers/infiniband/core/uverbs_std_types_srq.c @@ -192,6 +192,8 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_SRQ_NUM, UVERBS_ATTR_TYPE(u32), UA_OPTIONAL), + UVERBS_ATTR_UMEM(UVERBS_ATTR_CREATE_SRQ_BUF_UMEM, + UA_OPTIONAL), UVERBS_ATTR_UHW()); static int UVERBS_HANDLER(UVERBS_METHOD_SRQ_DESTROY)( diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index 4e2e556c8119..d150099b99d2 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -628,6 +628,7 @@ void uverbs_destroy_api(struct uverbs_api *uapi) static const struct uapi_definition uverbs_core_api[] = { UAPI_DEF_CHAIN(uverbs_def_obj_async_fd), UAPI_DEF_CHAIN(uverbs_def_obj_counters), + UAPI_DEF_CHAIN(uverbs_def_obj_comp_cntr), UAPI_DEF_CHAIN(uverbs_def_obj_cq), UAPI_DEF_CHAIN(uverbs_def_obj_device), UAPI_DEF_CHAIN(uverbs_def_obj_dm), diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 86811d31092c..04abc80c1327 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -392,6 +392,7 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) { int ret; + rdma_restrack_begin_del(&pd->res); if (pd->__internal_mr) { ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL); WARN_ON(ret); @@ -399,10 +400,12 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) } ret = pd->device->ops.dealloc_pd(pd, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&pd->res); return ret; + } - rdma_restrack_del(&pd->res); + rdma_restrack_commit_del(&pd->res); kfree(pd); return ret; } @@ -1140,16 +1143,20 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) if (atomic_read(&srq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&srq->res); + ret = srq->device->ops.destroy_srq(srq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&srq->res); return ret; + } atomic_dec(&srq->pd->usecnt); if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); - rdma_restrack_del(&srq->res); + rdma_restrack_commit_del(&srq->res); kfree(srq); return ret; @@ -1293,6 +1300,7 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, qp->qp_context = attr->qp_context; spin_lock_init(&qp->mr_lock); + xa_init(&qp->comp_cntrs); INIT_LIST_HEAD(&qp->rdma_mrs); INIT_LIST_HEAD(&qp->sig_mrs); init_completion(&qp->srq_completion); @@ -1327,6 +1335,7 @@ err_security: qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL); err_create: rdma_restrack_put(&qp->res); + xa_destroy(&qp->comp_cntrs); kfree(qp); return ERR_PTR(ret); @@ -2144,6 +2153,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr; const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr; struct ib_qp_security *sec; + struct ib_comp_cntr *cc; + unsigned long index; int ret; WARN_ON_ONCE(qp->mrs_used > 0); @@ -2154,6 +2165,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (qp->real_qp != qp) return __ib_destroy_shared_qp(qp); + rdma_restrack_begin_del(&qp->res); + sec = qp->qp_sec; if (sec) ib_destroy_qp_security_begin(sec); @@ -2166,6 +2179,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (ret) { if (sec) ib_destroy_qp_security_abort(sec); + rdma_restrack_abort_del(&qp->res); return ret; } @@ -2174,11 +2188,15 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (av_sgid_attr) rdma_put_gid_attr(av_sgid_attr); + xa_for_each(&qp->comp_cntrs, index, cc) + atomic_dec(&cc->usecnt); + xa_destroy(&qp->comp_cntrs); + ib_qp_usecnt_dec(qp); if (sec) ib_destroy_qp_security_end(sec); - rdma_restrack_del(&qp->res); + rdma_restrack_commit_del(&qp->res); kfree(qp); return ret; } @@ -2244,11 +2262,15 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (atomic_read(&cq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&cq->res); + ret = cq->device->ops.destroy_cq(cq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&cq->res); return ret; + } - rdma_restrack_del(&cq->res); + rdma_restrack_commit_del(&cq->res); kfree(cq); return ret; } |