Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git

# Conflicts:
#	drivers/infiniband/sw/rxe/rxe_verbs.c

101 files changed, 1756 insertions, 872 deletions

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 73170b15fc3d..337a49d1acf7 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -4823,6 +4823,29 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
 }
 EXPORT_SYMBOL(rdma_reject);
 
+static int cma_force_disconnect(struct rdma_id_private *id_priv)
+{
+	struct cma_work *work;
+
+	work = kzalloc_obj(*work);
+	if (!work)
+		return -ENOMEM;
+
+	INIT_WORK(&work->work, cma_work_handler);
+	work->old_state = RDMA_CM_CONNECT;
+	work->new_state = RDMA_CM_DISCONNECT;
+	work->event.event = RDMA_CM_EVENT_DISCONNECTED;
+	work->event.status = -ECONNABORTED;
+
+	cma_id_get(id_priv);
+	work->id = id_priv;
+
+	trace_cm_force_disconnect(id_priv);
+
+	queue_work(cma_wq, &work->work);
+	return 0;
+}
+
 int rdma_disconnect(struct rdma_cm_id *id)
 {
 	struct rdma_id_private *id_priv;
@@ -4838,12 +4861,15 @@ int rdma_disconnect(struct rdma_cm_id *id)
 			goto out;
 		/* Initiate or respond to a disconnect. */
 		trace_cm_disconnect(id_priv);
-		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
-			if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
-				trace_cm_sent_drep(id_priv);
-		} else {
+		if (!ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
 			trace_cm_sent_dreq(id_priv);
+			goto out;
+		}
+		if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) {
+			trace_cm_sent_drep(id_priv);
+			goto out;
 		}
+		ret = cma_force_disconnect(id_priv);
 	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
 		ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
 	} else
diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h
index 3456d5f3aa47..3d05d8e80856 100644
--- a/drivers/infiniband/core/cma_trace.h
+++ b/drivers/infiniband/core/cma_trace.h
@@ -61,6 +61,7 @@ DEFINE_CMA_FSM_EVENT(send_sidr_rep);
 DEFINE_CMA_FSM_EVENT(disconnect);
 DEFINE_CMA_FSM_EVENT(sent_drep);
 DEFINE_CMA_FSM_EVENT(sent_dreq);
+DEFINE_CMA_FSM_EVENT(force_disconnect);
 DEFINE_CMA_FSM_EVENT(id_destroy);
 
 TRACE_EVENT(cm_id_attach,
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 7a3ed5ecac00..29af9fe860e2 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -893,7 +893,7 @@ static int setup_port_data(struct ib_device *device)
  * ib_port_immutable_read() - Read rdma port's immutable data
  * @dev: IB device
  * @port: port number whose immutable data to read. It starts with index 1 and
- *        valid upto including rdma_end_port().
+ *        valid up to including rdma_end_port().
  */
 const struct ib_port_immutable*
 ib_port_immutable_read(struct ib_device *dev, unsigned int port)
@@ -3067,7 +3067,7 @@ int ib_add_sub_device(struct ib_device *parent,
 	sub->parent = parent;
 
 	mutex_lock(&parent->subdev_lock);
-	list_add_tail(&parent->subdev_list_head, &sub->subdev_list);
+	list_add_tail(&sub->subdev_list, &parent->subdev_list_head);
 	mutex_unlock(&parent->subdev_lock);
 
 	return ret;
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 0b7246ec559e..4b05867dad89 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -936,7 +936,7 @@ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv,
 /*
  * CM_ID <-- IDLE
  *
- * If in the ESTBLISHED or CLOSING states, the QP will have have been
+ * If in the ESTBLISHED or CLOSING states, the QP will have been
  * moved by the provider to the ERR state. Disassociate the CM_ID from
  * the QP,  move to IDLE, and remove the 'connected' reference.
  *
diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c
index 00fe241737ff..88aac911e8ee 100644
--- a/drivers/infiniband/core/lag.c
+++ b/drivers/infiniband/core/lag.c
@@ -59,7 +59,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev,
 		ip6h->version = 6;
 		ip6h->nexthdr = IPPROTO_UDP;
 		memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label,
-		       sizeof(*ip6h->flow_lbl));
+		       sizeof(ip6h->flow_lbl));
 		memcpy(&ip6h->saddr, ah_attr->grh.sgid_attr->gid.raw,
 		       sizeof(struct in6_addr));
 		memcpy(&ip6h->daddr, ah_attr->grh.dgid.raw,
diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c
index 17c4c52a19e4..f05e9ce93c33 100644
--- a/drivers/infiniband/core/mad_rmpp.c
+++ b/drivers/infiniband/core/mad_rmpp.c
@@ -245,12 +245,15 @@ static void recv_timeout_handler(struct work_struct *work)
 	struct ib_mad_recv_wc *rmpp_wc;
 	unsigned long flags;
 
-	spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
+	spin_lock_irqsave(&rmpp_recv->lock, flags);
 	if (rmpp_recv->state != RMPP_STATE_ACTIVE) {
-		spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
+		spin_unlock_irqrestore(&rmpp_recv->lock, flags);
 		return;
 	}
 	rmpp_recv->state = RMPP_STATE_TIMEOUT;
+	spin_unlock_irqrestore(&rmpp_recv->lock, flags);
+
+	spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
 	list_del(&rmpp_recv->list);
 	spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
 
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index bea7df3dd8f3..f549fd928c1c 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -189,7 +189,10 @@ static void release_group(struct mcast_group *group)
 
 	spin_lock_irqsave(&port->lock, flags);
 	if (atomic_dec_and_test(&group->refcount)) {
-		rb_erase(&group->node, &port->table);
+
+		if (!RB_EMPTY_NODE(&group->node))
+			rb_erase(&group->node, &port->table);
+
 		spin_unlock_irqrestore(&port->lock, flags);
 		kfree(group);
 		deref_port(port);
@@ -533,6 +536,8 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 		group->rec = *rec;
 		if (mgids_changed) {
 			rb_erase(&group->node, &group->port->table);
+
+			RB_CLEAR_NODE(&group->node);
 			is_mgid0 = !memcmp(&mgid0, &group->rec.mgid,
 					   sizeof(mgid0));
 			mcast_insert(group->port, group, is_mgid0);
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 4e8fbee34745..849a5db54e1c 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -700,7 +700,8 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
 		return -EMSGSIZE;
-	if (!rdma_is_kernel_res(res) &&
+
+	if (cq->uobject && cq->uobject->uevent.uobject.context &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 			cq->uobject->uevent.uobject.context->res.id))
 		return -EMSGSIZE;
@@ -790,7 +791,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id))
 		goto err;
 
-	if (!rdma_is_kernel_res(res) &&
+	if (pd->uobject && pd->uobject->context &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 			pd->uobject->context->res.id))
 		goto err;
@@ -1895,8 +1896,10 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
 		mutex_lock(&nldev_dellink_mutex);
 		err = device->link_ops->dellink(device);
 		mutex_unlock(&nldev_dellink_mutex);
-		if (err)
+		if (err) {
+			ib_device_put(device);
 			return err;
+		}
 	}
 
 	ib_unregister_device_and_put(device);
diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index f89a81dad72f..a8b838fdc7b5 100644
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -250,8 +250,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res)
 		ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);
 		if (ret)
 			res->id = 0;
-
-		if (qp->qp_type >= IB_QPT_DRIVER)
+		else if (qp->qp_type >= IB_QPT_DRIVER)
 			xa_set_mark(&rt->xa, res->id, RESTRACK_DD);
 	} else if (res->type == RDMA_RESTRACK_COUNTER) {
 		/* Special case to ensure that cntn points to right counter */
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 88110b9661f5..b1ab4f0359b0 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -55,7 +55,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 
 	if (dirty)
 		ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt,
-					   DMA_BIDIRECTIONAL, umem->dma_attrs);
+					   umem->dma_dir, umem->dma_attrs);
 
 	for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) {
 		unpin_user_page_range_dirty_lock(sg_page(sg),
@@ -159,9 +159,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
 }
 EXPORT_SYMBOL(ib_umem_find_best_pgsz);
 
+static inline enum dma_data_direction ib_access_dma_dir(int access)
+{
+	return ib_access_writable(access) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+}
+
 static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 					unsigned long addr, size_t size,
-					int access)
+					int access,
+					enum dma_data_direction dir)
 {
 	struct ib_umem *umem;
 	struct page **page_list;
@@ -202,6 +208,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 	 */
 	umem->iova = addr;
 	umem->writable   = ib_access_writable(access);
+	umem->dma_dir    = dir;
 	umem->owning_mm = mm = current->mm;
 	umem->dma_attrs = DMA_ATTR_REQUIRE_COHERENT;
 	if (access & IB_ACCESS_RELAXED_ORDERING)
@@ -261,7 +268,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 	}
 
 	ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt,
-				       DMA_BIDIRECTIONAL, umem->dma_attrs);
+				       dir, umem->dma_attrs);
 	if (ret)
 		goto umem_release;
 	goto out;
@@ -279,17 +286,16 @@ umem_kfree:
 	return ret ? ERR_PTR(ret) : umem;
 }
 
-/**
- * ib_umem_get_desc - Pin a umem from a buffer descriptor.
- * @device: IB device.
- * @desc:   buffer descriptor (VA or DMABUF).
- * @access: IB access flags.
+/*
+ * __ib_umem_get_desc_dir - core implementation for ib_umem_get_desc().
  *
- * Return: caller-owned umem on success, ERR_PTR(...) on error.
+ * @dir applies to VA buffers only; dmabuf direction is managed by the
+ * dmabuf subsystem and this argument is ignored for that type.
  */
-struct ib_umem *ib_umem_get_desc(struct ib_device *device,
-				 const struct ib_uverbs_buffer_desc *desc,
-				 int access)
+static struct ib_umem *
+__ib_umem_get_desc_dir(struct ib_device *device,
+		       const struct ib_uverbs_buffer_desc *desc,
+		       int access, enum dma_data_direction dir)
 {
 	struct ib_umem_dmabuf *umem_dmabuf;
 
@@ -310,11 +316,27 @@ struct ib_umem *ib_umem_get_desc(struct ib_device *device,
 		return &umem_dmabuf->umem;
 	case IB_UVERBS_BUFFER_TYPE_VA:
 		return __ib_umem_get_va(device, desc->addr, desc->length,
-					access);
+					access, dir);
 	default:
 		return ERR_PTR(-EINVAL);
 	}
 }
+
+/**
+ * ib_umem_get_desc - Pin a umem from a buffer descriptor.
+ * @device: IB device.
+ * @desc:   buffer descriptor (VA or DMABUF).
+ * @access: IB access flags.
+ *
+ * Return: caller-owned umem on success, ERR_PTR(...) on error.
+ */
+struct ib_umem *ib_umem_get_desc(struct ib_device *device,
+				 const struct ib_uverbs_buffer_desc *desc,
+				 int access)
+{
+	return __ib_umem_get_desc_dir(device, desc, access,
+				      ib_access_dma_dir(access));
+}
 EXPORT_SYMBOL(ib_umem_get_desc);
 
 /*
@@ -376,11 +398,12 @@ static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs,
 static struct ib_umem *
 ib_umem_get_desc_check(struct ib_device *device,
 		       const struct ib_uverbs_buffer_desc *desc,
-		       size_t min_size, int access)
+		       size_t min_size, int access,
+		       enum dma_data_direction dir)
 {
 	struct ib_umem *umem;
 
-	umem = ib_umem_get_desc(device, desc, access);
+	umem = __ib_umem_get_desc_dir(device, desc, access, dir);
 	if (IS_ERR(umem))
 		return umem;
 	if (umem->length < min_size) {
@@ -401,7 +424,7 @@ static struct ib_umem *
 ib_umem_get_from_attrs(struct ib_device *device,
 		       const struct uverbs_attr_bundle *attrs,
 		       u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler,
-		       size_t size, int access)
+		       size_t size, int access, enum dma_data_direction dir)
 {
 	struct ib_uverbs_buffer_desc desc = {};
 	int ret;
@@ -411,7 +434,7 @@ ib_umem_get_from_attrs(struct ib_device *device,
 		return NULL;
 	if (ret)
 		return ERR_PTR(ret);
-	return ib_umem_get_desc_check(device, &desc, size, access);
+	return ib_umem_get_desc_check(device, &desc, size, access, dir);
 }
 
 /*
@@ -431,7 +454,8 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device,
 			     const struct uverbs_attr_bundle *attrs,
 			     u16 attr_id,
 			     ib_umem_buf_desc_filler_t legacy_filler,
-			     u64 addr, size_t size, int access)
+			     u64 addr, size_t size, int access,
+			     enum dma_data_direction dir)
 {
 	struct ib_uverbs_buffer_desc desc = {};
 	int ret;
@@ -445,7 +469,7 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device,
 		};
 	else if (ret)
 		return ERR_PTR(ret);
-	return ib_umem_get_desc_check(device, &desc, size, access);
+	return ib_umem_get_desc_check(device, &desc, size, access, dir);
 }
 
 /**
@@ -464,7 +488,7 @@ struct ib_umem *ib_umem_get_attr(struct ib_device *device,
 				 u16 attr_id, size_t size, int access)
 {
 	return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size,
-				      access);
+				      access, ib_access_dma_dir(access));
 }
 EXPORT_SYMBOL(ib_umem_get_attr);
 
@@ -503,7 +527,8 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
 				       int access)
 {
 	return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr,
-					    size, access);
+					    size, access,
+					    ib_access_dma_dir(access));
 }
 EXPORT_SYMBOL(ib_umem_get_attr_or_va);
 
@@ -569,6 +594,9 @@ static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *att
  * must arrange its own backing (typically an in-kernel allocation)
  * when no source is available.
  *
+ * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it
+ * and the CPU only reads.
+ *
  * Return: caller-owned umem on success; NULL when no source supplied
  * a buffer; ERR_PTR(...) on error.
  */
@@ -579,7 +607,7 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device,
 	return ib_umem_get_from_attrs(device, attrs,
 				      UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
 				      uverbs_create_cq_get_buffer_desc,
-				      size, access);
+				      size, access, DMA_FROM_DEVICE);
 }
 EXPORT_SYMBOL(ib_umem_get_cq_buf);
 
@@ -595,6 +623,9 @@ EXPORT_SYMBOL(ib_umem_get_cq_buf);
  * Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the
  * UMEM attribute nor the legacy CQ buffer attributes are supplied.
  *
+ * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it
+ * and the CPU only reads.
+ *
  * See ib_umem_get_attr_or_va() for the note on @size's dual role and
  * the migration path for drivers that would distinguish a user-supplied
  * length from a driver-computed minimum.
@@ -608,7 +639,7 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device,
 	return ib_umem_get_from_attrs_or_va(device, attrs,
 					    UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
 					    uverbs_create_cq_get_buffer_desc,
-					    addr, size, access);
+					    addr, size, access, DMA_FROM_DEVICE);
 }
 EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va);
 
diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c
index ce0a7de00405..db65032e80b4 100644
--- a/drivers/infiniband/core/uverbs_std_types_device.c
+++ b/drivers/infiniband/core/uverbs_std_types_device.c
@@ -272,6 +272,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)(
 		return ret;
 	ret = ib_init_ucontext(attrs);
 	if (ret) {
+		rdma_restrack_put(&attrs->context->res);
 		kfree(attrs->context);
 		attrs->context = NULL;
 		return ret;
diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c
index 7ded8339346f..c23f455231bf 100644
--- a/drivers/infiniband/core/uverbs_std_types_wq.c
+++ b/drivers/infiniband/core/uverbs_std_types_wq.c
@@ -140,7 +140,7 @@ DECLARE_UVERBS_NAMED_METHOD(
 	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE,
 			UVERBS_OBJECT_CQ,
 			UVERBS_ACCESS_READ,
-			UA_OPTIONAL),
+			UA_MANDATORY),
 	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD,
 		       UVERBS_OBJECT_ASYNC_EVENT,
 		       UVERBS_ACCESS_READ,
diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c
index d150099b99d2..b67db3790f03 100644
--- a/drivers/infiniband/core/uverbs_uapi.c
+++ b/drivers/infiniband/core/uverbs_uapi.c
@@ -729,7 +729,7 @@ void uverbs_disassociate_api(struct uverbs_api *uapi)
 			 *
 			 * release_cleanup is the exception because
 			 * uverbs_uobject_fd_release() needs it. In this case
-			 * the module reference held by the fops will guarentee
+			 * the module reference held by the fops will guarantee
 			 * the type_class remains valid too.
 			 */
 			if (type_attrs &&
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index c43e25d37266..6d5825114c19 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2362,7 +2362,7 @@ EXPORT_SYMBOL(ib_dereg_mr_user);
  * @max_num_sg:    maximum sg entries available for registration.
  *
  * Notes:
- * Memory registeration page/sg lists must not exceed max_num_sg.
+ * Memory registration page/sg lists must not exceed max_num_sg.
  * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
  * max_num_sg * used_page_size.
  *
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index a43e678151d3..da471c63a3ba 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -216,6 +216,10 @@ struct bnxt_re_dev {
 	unsigned long			event_bitmap;
 	struct bnxt_qplib_cc_param	cc_param;
 	struct workqueue_struct		*dcb_wq;
+	/* Protects dcb_wq against bnxt_re_uninit_dcb_wq() destroying it
+	 * concurrently with bnxt_re_async_notifier() queuing work on it.
+	 */
+	spinlock_t			dcb_lock;
 	struct dentry                   *cc_config;
 	struct bnxt_re_dbg_cc_config_params *cc_config_params;
 	struct dentry			*cq_coal_cfg;
diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c
index 651cf9d0e0c7..575b97030587 100644
--- a/drivers/infiniband/hw/bnxt_re/hw_counters.c
+++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c
@@ -413,7 +413,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
 	}
 
 done:
-	return bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
+	return bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+					 rdev->is_virtfn) ?
 		BNXT_RE_NUM_EXT_COUNTERS : BNXT_RE_NUM_STD_COUNTERS;
 }
 
@@ -423,7 +424,8 @@ struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev,
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 	int num_counters = 0;
 
-	if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
+	if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+				     rdev->is_virtfn))
 		num_counters = BNXT_RE_NUM_EXT_COUNTERS;
 	else
 		num_counters = BNXT_RE_NUM_STD_COUNTERS;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index ccd2702db78b..ac15372376a2 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -165,10 +165,11 @@ static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
 		qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
 }
 
-static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
+static u32 bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 			     struct bnxt_qplib_sge *sg_list, int num)
 {
-	int i, total = 0;
+	u32 total = 0;
+	int i;
 
 	for (i = 0; i < num; i++) {
 		sg_list[i].addr = ib_sg_list[i].addr;
@@ -2235,6 +2236,14 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 		goto exit;
 	}
 
+	if (srq_init_attr->attr.max_sge > dev_attr->max_srq_sges) {
+		ibdev_err(&rdev->ibdev,
+			  "Create SRQ failed - max_sge %d exceeds supported %d",
+			  srq_init_attr->attr.max_sge, dev_attr->max_srq_sges);
+		rc = -EINVAL;
+		goto exit;
+	}
+
 	if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
 		rc = -EOPNOTSUPP;
 		goto exit;
@@ -2388,6 +2397,7 @@ int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
 	spin_lock_irqsave(&srq->lock, flags);
 	while (wr) {
 		/* Transcribe each ib_recv_wr to qplib_swqe */
+		wqe.flags = 0;
 		wqe.num_sge = wr->num_sge;
 		bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
 		wqe.wr_id = wr->wr_id;
@@ -2903,7 +2913,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
 	qp->send_psn &= BTH_PSN_MASK;
 	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
 	/* DETH */
-	/* Use the priviledged Q_Key for QP1 */
+	/* Use the privileged Q_Key for QP1 */
 	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
 	qp->qp1_hdr.deth.source_qpn = IB_QP1;
 
@@ -3168,8 +3178,9 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
 				wr->sg_list[i].addr;
 		sge_len = wr->sg_list[i].length;
 
-		if ((sge_len + wqe->inline_len) >
-		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
+		if (sge_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH ||
+		    ((sge_len + wqe->inline_len) >
+		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH)) {
 			ibdev_err(&rdev->ibdev,
 				  "Inline data size requested > supported value");
 			return -EINVAL;
@@ -3185,17 +3196,22 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
 
 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
 				   const struct ib_send_wr *wr,
-				   struct bnxt_qplib_swqe *wqe)
+				   struct bnxt_qplib_swqe *wqe,
+				   u32 *payload_sz)
 {
-	int payload_sz = 0;
+	int rc;
 
-	if (wr->send_flags & IB_SEND_INLINE)
-		payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
-	else
-		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
-					       wqe->num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		rc = bnxt_re_copy_inline_data(rdev, wr, wqe);
+		if (rc < 0)
+			return rc;
+		*payload_sz = rc;
+	} else {
+		*payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
+						wqe->num_sge);
+	}
 
-	return payload_sz;
+	return 0;
 }
 
 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
@@ -3218,7 +3234,8 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
 				       struct bnxt_re_qp *qp,
 				       const struct ib_send_wr *wr)
 {
-	int rc = 0, payload_sz = 0;
+	int rc = 0;
+	u32 payload_sz = 0;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->sq_lock, flags);
@@ -3234,11 +3251,9 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
 			goto bad;
 		}
 
-		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
-		if (payload_sz < 0) {
-			rc = -EINVAL;
+		rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz);
+		if (rc)
 			goto bad;
-		}
 		wqe.wr_id = wr->wr_id;
 
 		wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
@@ -3279,7 +3294,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
 {
 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 	struct bnxt_qplib_swqe wqe;
-	int rc = 0, payload_sz = 0;
+	int rc = 0;
+	u32 payload_sz = 0;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->sq_lock, flags);
@@ -3296,11 +3312,9 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
 			goto bad;
 		}
 
-		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
-		if (payload_sz < 0) {
-			rc = -EINVAL;
+		rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz);
+		if (rc)
 			goto bad;
-		}
 		wqe.wr_id = wr->wr_id;
 
 		switch (wr->opcode) {
@@ -3749,12 +3763,13 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe,
 	if (rc)
 		goto fail;
 
-	cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
-					 entries * sizeof(struct cq_base),
-					 IB_ACCESS_LOCAL_WRITE);
+	cq->resize_umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, NULL,
+						   req.cq_va,
+						   entries * sizeof(struct cq_base),
+						   IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(cq->resize_umem)) {
 		rc = PTR_ERR(cq->resize_umem);
-		ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n",
+		ibdev_err(&rdev->ibdev, "%s: ib_umem_get_cq_buf_or_va failed! rc = %pe\n",
 			  __func__, cq->resize_umem);
 		cq->resize_umem = NULL;
 		goto fail;
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 17654a9e23fe..fa0e1323a326 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -367,9 +367,15 @@ static int bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
 
 static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
 {
-	if (!rdev->dcb_wq)
-		return;
-	destroy_workqueue(rdev->dcb_wq);
+	struct workqueue_struct *dcb_wq;
+
+	spin_lock_bh(&rdev->dcb_lock);
+	dcb_wq = rdev->dcb_wq;
+	rdev->dcb_wq = NULL;
+	spin_unlock_bh(&rdev->dcb_lock);
+
+	if (dcb_wq)
+		destroy_workqueue(dcb_wq);
 }
 
 static void bnxt_re_dcb_wq_task(struct work_struct *work)
@@ -424,14 +430,23 @@ static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *c
 
 	switch (event_id) {
 	case ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:
+		spin_lock(&rdev->dcb_lock);
+		if (!rdev->dcb_wq) {
+			spin_unlock(&rdev->dcb_lock);
+			break;
+		}
+
 		dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC);
-		if (!dcb_work)
+		if (!dcb_work) {
+			spin_unlock(&rdev->dcb_lock);
 			break;
+		}
 
 		dcb_work->rdev = rdev;
 		memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl));
 		INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task);
 		queue_work(rdev->dcb_wq, &dcb_work->work);
+		spin_unlock(&rdev->dcb_lock);
 		break;
 	default:
 		break;
@@ -1448,6 +1463,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev,
 	INIT_LIST_HEAD(&rdev->qp_list);
 	mutex_init(&rdev->qp_lock);
 	mutex_init(&rdev->pacing.dbq_lock);
+	spin_lock_init(&rdev->dcb_lock);
 	atomic_set(&rdev->stats.res.qp_count, 0);
 	atomic_set(&rdev->stats.res.cq_count, 0);
 	atomic_set(&rdev->stats.res.srq_count, 0);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 756f8b5f042a..7ff587ce9126 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -45,6 +45,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/if_vlan.h>
 #include <linux/vmalloc.h>
+#include <linux/bitops.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/iter.h>
 
@@ -668,9 +669,9 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
 {
 	u32 bytes;
 
-	bytes = max >> 3;
+	bytes = BITS_TO_LONGS(max) * sizeof(unsigned long);
 	if (!bytes)
-		bytes = 1;
+		bytes = sizeof(unsigned long);
 	pdt->tbl = kmalloc(bytes, GFP_KERNEL);
 	if (!pdt->tbl)
 		return -ENOMEM;
@@ -848,9 +849,9 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
 	if (!dpit->app_tbl)
 		return -ENOMEM;
 
-	bytes = dpit->max >> 3;
+	bytes = BITS_TO_LONGS(dpit->max) * sizeof(unsigned long);
 	if (!bytes)
-		bytes = 1;
+		bytes = sizeof(unsigned long);
 
 	dpit->tbl = kmalloc(bytes, GFP_KERNEL);
 	if (!dpit->tbl) {
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index ec9eb52a8ebf..9aaa2b5204b8 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -160,7 +160,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
 
 	attr->max_srq = le16_to_cpu(sb->max_srq);
 	attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
-	attr->max_srq_sges = sb->max_srq_sge;
+	attr->max_srq_sges = min_t(u32, sb->max_srq_sge, BNXT_STATIC_MAX_SGE);
 	attr->max_pkey = 1;
 	attr->max_inline_data = attr->max_qp_sges * sizeof(struct sq_sge);
 	if (!bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx))
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index b3b45c49077d..a6f520f49d59 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -4006,8 +4006,10 @@ static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb,
 	int ret;
 
 	req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL);
-	if (!req_skb)
+	if (!req_skb) {
+		kfree_skb(skb);
 		return;
+	}
 	req = __skb_put_zero(req_skb, sizeof(*req));
 	req->op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL_F);
 	req->len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)));
@@ -4156,6 +4158,7 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
 		pdev = ip_dev_find(&init_net, iph->daddr);
 		if (!pdev) {
 			pr_err("%s - failed to find device!\n", __func__);
+			neigh_release(neigh);
 			goto free_dst;
 		}
 		e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh,
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index d1517f2560b9..2a7be0a1f168 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -1182,7 +1182,7 @@ void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx)
 	struct c4iw_cq *rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
 	unsigned long flag;
 
-	/* locking heirarchy: cq lock first, then qp lock. */
+	/* locking hierarchy: cq lock first, then qp lock. */
 	spin_lock_irqsave(&rchp->lock, flag);
 	spin_lock(&qhp->lock);
 
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index ab830764e3b4..30a38e633bfd 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -358,7 +358,8 @@ struct efa_admin_reg_mr_cmd {
 	 * 5:0 : phys_page_size_shift - page size is (1 <<
 	 *    phys_page_size_shift). Page size is used for
 	 *    building the Virtual to Physical address mapping
-	 * 6 : reserved - MBZ
+	 * 6 : relaxed_ordering_enable - Enable PCIe relaxed
+	 *    ordering for this memory region
 	 * 7 : mem_addr_phy_mode_en - Enable bit for physical
 	 *    memory registration (no translation), can be used
 	 *    only by privileged clients. If set, PBL must
@@ -1203,6 +1204,7 @@ struct efa_admin_modify_event_counter_resp {
 
 /* reg_mr_cmd */
 #define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(5, 0)
+#define EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE_MASK   BIT(6)
 #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK      BIT(7)
 #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK        BIT(0)
 #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK       BIT(1)
@@ -1231,7 +1233,6 @@ struct efa_admin_modify_event_counter_resp {
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK  BIT(3)
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4)
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5)
-#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_SHIFT 10
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_MASK BIT(10)
 
 /* feature_queue_attr_desc_2 */
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index 1b00f16b8ea8..07bfcb093df8 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -236,8 +236,15 @@ int efa_com_register_mr(struct efa_com_dev *edev,
 	mr_cmd.mr_length = params->mr_length_in_bytes;
 	EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,
 		params->page_shift);
+	if (params->relaxed_ordering)
+		EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE, 1);
 	mr_cmd.iova = params->iova;
-	mr_cmd.permissions = params->permissions;
+	if (params->permissions.local_write)
+		EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1);
+	if (params->permissions.remote_write)
+		EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE, 1);
+	if (params->permissions.remote_read)
+		EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE, 1);
 
 	if (params->inline_pbl) {
 		memcpy(mr_cmd.pbl.inline_pbl_array,
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index bca722f021ea..524407f36d43 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -204,8 +204,12 @@ struct efa_com_reg_mr_params {
 	 * address mapping
 	 */
 	u8 page_shift;
-	/* see permissions field of struct efa_admin_reg_mr_cmd */
-	u8 permissions;
+	struct {
+		u8 local_write : 1;
+		u8 remote_write : 1;
+		u8 remote_read : 1;
+	} permissions;
+	u8 relaxed_ordering;
 	u8 inline_pbl;
 	u8 indirect;
 };
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 2d28d68efe77..9e4435401a4c 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1676,7 +1676,10 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
 	params.pd = to_epd(ibpd)->pdn;
 	params.iova = virt_addr;
 	params.mr_length_in_bytes = length;
-	params.permissions = access_flags;
+	params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE);
+	params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE);
+	params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ);
+	params.relaxed_ordering = !!(access_flags & IB_ACCESS_RELAXED_ORDERING);
 
 	pg_sz = ib_umem_find_best_pgsz(mr->umem,
 				       dev->dev_attr.page_size_cap,
diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c
index 1f456327e63c..daec74f0aa6d 100644
--- a/drivers/infiniband/hw/erdma/erdma_cq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cq.c
@@ -8,8 +8,8 @@
 
 static void *get_next_valid_cqe(struct erdma_cq *cq)
 {
-	__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
-				      cq->depth, CQE_SHIFT);
+	__be32 *cqe = erdma_kmem_get_entry(
+		&cq->kern_cq.qbuf_mem, cq->kern_cq.ci, cq->depth, CQE_SHIFT);
 	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
 			      be32_to_cpu(READ_ONCE(*cqe)));
 
@@ -161,8 +161,8 @@ static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)
 	if (qtype == ERDMA_CQE_QTYPE_SQ) {
 		id_table = kern_qp->swr_tbl;
 		depth = qp->attrs.sq_size;
-		wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
-					  qp->attrs.sq_size, SQEBB_SHIFT);
+		wqe_hdr = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+					       qp->attrs.sq_size, SQEBB_SHIFT);
 		kern_qp->sq_ci =
 			FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +
 			wqe_idx + 1;
@@ -242,15 +242,15 @@ void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn)
 
 	while (ncqe > 0) {
 		cur_cq_ci = prev_cq_ci + ncqe - 1;
-		cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth,
-				      CQE_SHIFT);
+		cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, cur_cq_ci,
+					   cq->depth, CQE_SHIFT);
 
 		if (be32_to_cpu(cqe->qpn) == qpn) {
 			++nqp_cqe;
 		} else if (nqp_cqe) {
-			dst_cqe = get_queue_entry(cq->kern_cq.qbuf,
-						  cur_cq_ci + nqp_cqe,
-						  cq->depth, CQE_SHIFT);
+			dst_cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem,
+						       cur_cq_ci + nqp_cqe,
+						       cq->depth, CQE_SHIFT);
 			owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
 					  be32_to_cpu(dst_cqe->hdr));
 			cqe->hdr = cpu_to_be32(
diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 445182c6bc5d..62dc22388eea 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -50,7 +50,7 @@ static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
 	struct net_device *netdev;
 	int ret = -EPROBE_DEFER;
 
-	/* Already binded to a net_device, so we skip. */
+	/* Already bound to a net_device, so we skip. */
 	if (dev->netdev)
 		return 0;
 
diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c
index e002343832f7..fc335b674746 100644
--- a/drivers/infiniband/hw/erdma/erdma_qp.c
+++ b/drivers/infiniband/hw/erdma/erdma_qp.c
@@ -259,8 +259,8 @@ static void erdma_reset_qp(struct erdma_qp *qp)
 	qp->kern_qp.rq_ci = 0;
 	memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64));
 	memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64));
-	memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT);
-	memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT);
+	erdma_kmem_clear(&qp->kern_qp.sq_mem);
+	erdma_kmem_clear(&qp->kern_qp.rq_mem);
 	erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp));
 	if (qp->rcq != qp->scq)
 		erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp));
@@ -332,8 +332,8 @@ static int fill_inline_data(struct erdma_qp *qp,
 
 	wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 	sgl_offset &= (SQEBB_SIZE - 1);
-	data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
-			       SQEBB_SHIFT);
+	data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+				    qp->attrs.sq_size, SQEBB_SHIFT);
 
 	while (i < send_wr->num_sge) {
 		bytes += send_wr->sg_list[i].length;
@@ -356,8 +356,9 @@ static int fill_inline_data(struct erdma_qp *qp,
 			wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 			sgl_offset &= (SQEBB_SIZE - 1);
 
-			data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
-					       qp->attrs.sq_size, SQEBB_SHIFT);
+			data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
+						    wqe_idx, qp->attrs.sq_size,
+						    SQEBB_SHIFT);
 			if (!remain_size)
 				break;
 		}
@@ -385,8 +386,8 @@ static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
 	while (i < send_wr->num_sge) {
 		wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 		sgl_offset &= (SQEBB_SIZE - 1);
-		sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
-				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sgl = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+					   qp->attrs.sq_size, SQEBB_SHIFT);
 
 		bytes += send_wr->sg_list[i].length;
 		memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
@@ -463,8 +464,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 	    send_wr->opcode != IB_WR_SEND_WITH_IMM)
 		return -EINVAL;
 
-	entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
-				SQEBB_SHIFT);
+	entry = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx,
+				     qp->attrs.sq_size, SQEBB_SHIFT);
 
 	/* Clear the SQE header section. */
 	*entry = 0;
@@ -524,8 +525,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 		read_sqe->sink_to_h =
 			cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
 
-		sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
-				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
+					   qp->attrs.sq_size, SQEBB_SHIFT);
 		sge->addr = cpu_to_le64(rdma_wr->remote_addr);
 		sge->key = cpu_to_le32(rdma_wr->rkey);
 		sge->length = cpu_to_le32(send_wr->sg_list[0].length);
@@ -569,8 +570,9 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 		if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
 			attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
 			/* Copy SGLs to SQE content to accelerate */
-			memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
-					       qp->attrs.sq_size, SQEBB_SHIFT),
+			memcpy(erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
+						    idx + 1, qp->attrs.sq_size,
+						    SQEBB_SHIFT),
 			       mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents));
 			wqe_size = sizeof(struct erdma_reg_mr_sqe) +
 				   MTT_SIZE(mr->mem.mtt_nents);
@@ -605,8 +607,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 				cpu_to_le64(atomic_wr(send_wr)->compare_add);
 		}
 
-		sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
-				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
+					   qp->attrs.sq_size, SQEBB_SHIFT);
 		sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
 		sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
 		sge++;
@@ -702,8 +704,8 @@ static int erdma_post_recv_one(struct erdma_qp *qp,
 			       const struct ib_recv_wr *recv_wr)
 {
 	struct erdma_rqe *rqe =
-		get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
-				qp->attrs.rq_size, RQE_SHIFT);
+		erdma_kmem_get_entry(&qp->kern_qp.rq_mem, qp->kern_qp.rq_pi,
+				     qp->attrs.rq_size, RQE_SHIFT);
 
 	rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
 	rqe->qpn = cpu_to_le32(QP_ID(qp));
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index f18b88bba281..dd29d55d4122 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -41,7 +41,7 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
 	struct erdma_dev *dev = to_edev(qp->ibqp.device);
 	struct erdma_pd *pd = to_epd(qp->ibqp.pd);
 	struct erdma_cmdq_create_qp_req req;
-	struct erdma_uqp *user_qp;
+	struct erdma_mem *sq_mem, *rq_mem;
 	u64 resp0, resp1;
 	int err;
 
@@ -63,67 +63,46 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
 				      ERDMA_QPT_UD);
 
 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
-		u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
-
-		req.sq_cqn_mtt_cfg =
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-				   pgsz_range) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
-		req.rq_cqn_mtt_cfg =
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-				   pgsz_range) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
-
-		req.sq_mtt_cfg =
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
-				   ERDMA_MR_MTT_0LEVEL);
-		req.rq_mtt_cfg = req.sq_mtt_cfg;
-
-		req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
-		req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
+		sq_mem = &qp->kern_qp.sq_mem;
+		rq_mem = &qp->kern_qp.rq_mem;
 		req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
 		req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
 	} else {
-		user_qp = &qp->user_qp;
-		req.sq_cqn_mtt_cfg = FIELD_PREP(
-			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-			ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
+		sq_mem = &qp->user_qp.sq_mem;
+		rq_mem = &qp->user_qp.rq_mem;
+		req.sq_dbrec_dma = qp->user_qp.sq_dbrec_dma;
+		req.rq_dbrec_dma = qp->user_qp.rq_dbrec_dma;
+	}
+
+	req.sq_cqn_mtt_cfg =
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
+			   ilog2(sq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+	req.sq_cqn_mtt_cfg |=
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
+	req.rq_cqn_mtt_cfg =
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
+			   ilog2(rq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+	req.rq_cqn_mtt_cfg |=
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
+
+	req.sq_mtt_cfg =
+		sq_mem->page_offset |
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, sq_mem->mtt_nents);
+	req.rq_mtt_cfg =
+		rq_mem->page_offset |
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, rq_mem->mtt_nents);
+
+	assemble_qbuf_mtt_for_cmd(sq_mem, &req.sq_mtt_cfg, &req.sq_buf_addr,
+				  req.sq_mtt_entry);
+	assemble_qbuf_mtt_for_cmd(rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr,
+				  req.rq_mtt_entry);
+	if (uctx && uctx->ext_db.enable) {
 		req.sq_cqn_mtt_cfg |=
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
-
-		req.rq_cqn_mtt_cfg = FIELD_PREP(
-			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-			ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
-		req.rq_cqn_mtt_cfg |=
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
-
-		req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
-		req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
-					     user_qp->sq_mem.mtt_nents);
-
-		req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
-		req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
-					     user_qp->rq_mem.mtt_nents);
-
-		assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
-					  &req.sq_buf_addr, req.sq_mtt_entry);
-		assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
-					  &req.rq_buf_addr, req.rq_mtt_entry);
-
-		req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
-		req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
-
-		if (uctx->ext_db.enable) {
-			req.sq_cqn_mtt_cfg |=
-				FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
-			req.db_cfg =
-				FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
-					   uctx->ext_db.sdb_off) |
-				FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
-					   uctx->ext_db.rdb_off);
-		}
+			FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
+		req.db_cfg = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
+					uctx->ext_db.sdb_off) |
+			     FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
+					uctx->ext_db.rdb_off);
 	}
 
 	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
@@ -194,7 +173,6 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
 	struct erdma_cmdq_create_cq_req req;
 	struct erdma_mem *mem;
-	u32 page_size;
 
 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 				CMDQ_OPCODE_CREATE_CQ);
@@ -204,48 +182,34 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
 	req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
 
 	if (rdma_is_kernel_res(&cq->ibcq.res)) {
-		page_size = SZ_32M;
-		req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
-				       ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
-		req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
-		req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
-
-		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
-			    FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
-				       ERDMA_MR_MTT_0LEVEL);
-
-		req.first_page_offset = 0;
+		mem = &cq->kern_cq.qbuf_mem;
 		req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
 	} else {
 		mem = &cq->user_cq.qbuf_mem;
-		req.cfg0 |=
-			FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
-				   ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
-		if (mem->mtt_nents == 1) {
-			req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
-			req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
-			req.cfg1 |=
-				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
-					   ERDMA_MR_MTT_0LEVEL);
-		} else {
-			req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
-			req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
-			req.cfg1 |=
-				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
-					   ERDMA_MR_MTT_1LEVEL);
-		}
-		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
-				       mem->mtt_nents);
-
-		req.first_page_offset = mem->page_offset;
 		req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
+	}
 
-		if (uctx->ext_db.enable) {
-			req.cfg1 |= FIELD_PREP(
-				ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
-			req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
-					      uctx->ext_db.cdb_off);
-		}
+	req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
+			       ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+	if (mem->mtt_nents == 1) {
+		req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
+		req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
+				       ERDMA_MR_MTT_0LEVEL);
+	} else {
+		req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
+		req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
+				       ERDMA_MR_MTT_1LEVEL);
+	}
+	req.cfg1 |=
+		FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, mem->mtt_nents);
+	req.first_page_offset = mem->page_offset;
+
+	if (uctx && uctx->ext_db.enable) {
+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
+		req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
+				      uctx->ext_db.cdb_off);
 	}
 
 	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
@@ -505,96 +469,28 @@ static int erdma_qp_validate_attr(struct erdma_dev *dev,
 	return 0;
 }
 
-static void free_kernel_qp(struct erdma_qp *qp)
-{
-	struct erdma_dev *dev = qp->dev;
-
-	vfree(qp->kern_qp.swr_tbl);
-	vfree(qp->kern_qp.rwr_tbl);
-
-	if (qp->kern_qp.sq_buf)
-		dma_free_coherent(&dev->pdev->dev,
-				  qp->attrs.sq_size << SQEBB_SHIFT,
-				  qp->kern_qp.sq_buf,
-				  qp->kern_qp.sq_buf_dma_addr);
-
-	if (qp->kern_qp.sq_dbrec)
-		dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
-			      qp->kern_qp.sq_dbrec_dma);
-
-	if (qp->kern_qp.rq_buf)
-		dma_free_coherent(&dev->pdev->dev,
-				  qp->attrs.rq_size << RQE_SHIFT,
-				  qp->kern_qp.rq_buf,
-				  qp->kern_qp.rq_buf_dma_addr);
-
-	if (qp->kern_qp.rq_dbrec)
-		dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
-			      qp->kern_qp.rq_dbrec_dma);
-}
-
-static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
-			  struct ib_qp_init_attr *attrs)
+static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt)
 {
-	struct erdma_kqp *kqp = &qp->kern_qp;
-	int size;
-
-	if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
-		kqp->sig_all = 1;
-
-	kqp->sq_pi = 0;
-	kqp->sq_ci = 0;
-	kqp->rq_pi = 0;
-	kqp->rq_ci = 0;
-	kqp->hw_sq_db =
-		dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
-	kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
-
-	kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
-	kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
-	if (!kqp->swr_tbl || !kqp->rwr_tbl)
-		goto err_out;
-
-	size = qp->attrs.sq_size << SQEBB_SHIFT;
-	kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
-					 &kqp->sq_buf_dma_addr, GFP_KERNEL);
-	if (!kqp->sq_buf)
-		goto err_out;
-
-	kqp->sq_dbrec =
-		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
-	if (!kqp->sq_dbrec)
-		goto err_out;
-
-	size = qp->attrs.rq_size << RQE_SHIFT;
-	kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
-					 &kqp->rq_buf_dma_addr, GFP_KERNEL);
-	if (!kqp->rq_buf)
-		goto err_out;
-
-	kqp->rq_dbrec =
-		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
-	if (!kqp->rq_dbrec)
-		goto err_out;
-
-	return 0;
+	struct ib_block_iter biter;
+	u32 idx = 0;
 
-err_out:
-	free_kernel_qp(qp);
-	return -ENOMEM;
+	if (mem->type == ERDMA_UMEM) {
+		rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
+			mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
+	} else {
+		for (; idx < mem->page_cnt; idx++)
+			mtt->buf[idx] = mem->kmem.buf_list[idx].dma_addr;
+	}
 }
 
-static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_init_bottom_mtt(struct erdma_mem *mem)
 {
 	struct erdma_mtt *mtt = mem->mtt;
-	struct ib_block_iter biter;
-	u32 idx = 0;
 
 	while (mtt->low_level)
 		mtt = mtt->low_level;
 
-	rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
-		mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
+	erdma_init_mtt_leaf(mem, mtt);
 }
 
 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
@@ -826,54 +722,216 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
 	}
 }
 
-static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
-			   u64 start, u64 len, int access, u64 virt,
-			   unsigned long req_page_size, bool force_continuous)
+static void erdma_kmem_free(struct erdma_dev *dev, struct erdma_mem *mem)
 {
-	int ret = 0;
+	struct erdma_buf_list *buf_list = mem->kmem.buf_list;
+	u32 i;
 
-	mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
-	if (IS_ERR(mem->umem)) {
-		ret = PTR_ERR(mem->umem);
-		mem->umem = NULL;
-		return ret;
+	for (i = 0; i < mem->page_cnt; i++)
+		dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf,
+				  buf_list[i].dma_addr);
+
+	kfree(buf_list);
+}
+
+static int erdma_kmem_alloc(struct erdma_dev *dev, struct erdma_mem *mem,
+			    u32 page_cnt)
+{
+	struct erdma_buf_list *buf_list;
+	u32 i;
+
+	buf_list = kcalloc(page_cnt, sizeof(*buf_list), GFP_KERNEL);
+	if (!buf_list)
+		return -ENOMEM;
+
+	for (i = 0; i < page_cnt; i++) {
+		buf_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
+						     &buf_list[i].dma_addr,
+						     GFP_KERNEL);
+		if (!buf_list[i].buf)
+			goto err_free_pages;
+	}
+
+	mem->kmem.buf_list = buf_list;
+	return 0;
+
+err_free_pages:
+	while (i--)
+		dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf,
+				  buf_list[i].dma_addr);
+	kfree(buf_list);
+
+	return -ENOMEM;
+}
+
+static void erdma_mem_free(struct erdma_dev *dev, struct erdma_mem *mem)
+{
+	switch (mem->type) {
+	case ERDMA_UMEM:
+		ib_umem_release(mem->umem);
+		break;
+	case ERDMA_KMEM:
+		erdma_kmem_free(dev, mem);
+		break;
+	default:
+		break;
+	}
+}
+
+static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
+			  struct erdma_mem_init_attr *attr)
+{
+	struct erdma_mtt *mtt;
+	int ret;
+
+	mem->type = attr->type;
+
+	switch (mem->type) {
+	case ERDMA_UMEM:
+		if (attr->flags & ERDMA_MEM_FLAG_CQ_BUF)
+			mem->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
+							     attr->start,
+							     attr->len,
+							     attr->access);
+		else
+			mem->umem = ib_umem_get_va(&dev->ibdev, attr->start,
+						   attr->len, attr->access);
+		if (IS_ERR(mem->umem)) {
+			ret = PTR_ERR(mem->umem);
+			return ret;
+		}
+
+		mem->page_size = ib_umem_find_best_pgsz(
+			mem->umem, attr->req_page_size, attr->virt);
+		mem->mtt_nents =
+			ib_umem_num_dma_blocks(mem->umem, mem->page_size);
+		break;
+	case ERDMA_KMEM:
+		mem->page_size = PAGE_SIZE;
+		mem->mtt_nents = DIV_ROUND_UP(attr->len, PAGE_SIZE);
+		ret = erdma_kmem_alloc(dev, mem, mem->mtt_nents);
+		if (ret)
+			return ret;
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	mem->va = virt;
-	mem->len = len;
-	mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
-	mem->page_offset = start & (mem->page_size - 1);
-	mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
+	mem->va = attr->virt;
+	mem->len = attr->len;
+	mem->page_offset = attr->start & (mem->page_size - 1);
 	mem->page_cnt = mem->mtt_nents;
-	mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
-				    force_continuous);
-	if (IS_ERR(mem->mtt)) {
-		ret = PTR_ERR(mem->mtt);
-		goto error_ret;
+	mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
+			       !(attr->flags & ERDMA_MEM_FLAG_MR_BUF));
+	if (IS_ERR(mtt)) {
+		ret = PTR_ERR(mtt);
+		goto err_free_mem;
 	}
+	mem->mtt = mtt;
 
-	erdma_fill_bottom_mtt(dev, mem);
+	erdma_init_bottom_mtt(mem);
+	if (mtt->continuous)
+		dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma,
+					   mtt->size, DMA_TO_DEVICE);
 
 	return 0;
 
-error_ret:
-	if (mem->umem) {
-		ib_umem_release(mem->umem);
-		mem->umem = NULL;
-	}
+err_free_mem:
+	erdma_mem_free(dev, mem);
 
 	return ret;
 }
 
-static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
 {
-	if (mem->mtt)
-		erdma_destroy_mtt(dev, mem->mtt);
+	erdma_destroy_mtt(dev, mem->mtt);
+	erdma_mem_free(dev, mem);
+}
 
-	if (mem->umem) {
-		ib_umem_release(mem->umem);
-		mem->umem = NULL;
+static void free_kernel_qp(struct erdma_qp *qp)
+{
+	struct erdma_dev *dev = qp->dev;
+
+	vfree(qp->kern_qp.swr_tbl);
+	vfree(qp->kern_qp.rwr_tbl);
+
+	erdma_mem_uninit(dev, &qp->kern_qp.sq_mem);
+	dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
+		      qp->kern_qp.sq_dbrec_dma);
+	erdma_mem_uninit(dev, &qp->kern_qp.rq_mem);
+	dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
+		      qp->kern_qp.rq_dbrec_dma);
+}
+
+static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
+			  struct ib_qp_init_attr *attrs)
+{
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_KMEM,
+	};
+	struct erdma_kqp *kqp = &qp->kern_qp;
+	int ret = -ENOMEM;
+	int size;
+
+	if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
+		kqp->sig_all = 1;
+
+	kqp->sq_pi = 0;
+	kqp->sq_ci = 0;
+	kqp->rq_pi = 0;
+	kqp->rq_ci = 0;
+	kqp->hw_sq_db =
+		dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
+	kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
+
+	kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
+	if (!kqp->swr_tbl)
+		return -ENOMEM;
+
+	kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
+	if (!kqp->rwr_tbl)
+		goto err_free_swr_tbl;
+
+	size = qp->attrs.sq_size << SQEBB_SHIFT;
+	attr.len = size;
+	ret = erdma_mem_init(dev, &kqp->sq_mem, &attr);
+	if (ret)
+		goto err_free_rwr_tbl;
+
+	kqp->sq_dbrec =
+		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
+	if (!kqp->sq_dbrec) {
+		ret = -ENOMEM;
+		goto err_free_sq_mem;
 	}
+
+	size = qp->attrs.rq_size << RQE_SHIFT;
+	attr.len = size;
+	ret = erdma_mem_init(dev, &kqp->rq_mem, &attr);
+	if (ret)
+		goto err_free_sq_dbrec;
+
+	kqp->rq_dbrec =
+		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
+	if (!kqp->rq_dbrec) {
+		ret = -ENOMEM;
+		goto err_free_rq_mem;
+	}
+
+	return 0;
+
+err_free_rq_mem:
+	erdma_mem_uninit(dev, &kqp->rq_mem);
+err_free_sq_dbrec:
+	dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
+err_free_sq_mem:
+	erdma_mem_uninit(dev, &kqp->sq_mem);
+err_free_rwr_tbl:
+	vfree(kqp->rwr_tbl);
+err_free_swr_tbl:
+	vfree(kqp->swr_tbl);
+
+	return ret;
 }
 
 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
@@ -941,6 +999,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 			u64 va, u32 len, u64 dbrec_va)
 {
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_UMEM,
+	};
 	dma_addr_t dbrec_dma;
 	u32 rq_offset;
 	int ret;
@@ -949,45 +1010,47 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 		   qp->attrs.rq_size * RQE_SIZE))
 		return -EINVAL;
 
-	ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
-			      qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
-			      (SZ_1M - SZ_4K), true);
+	attr.start = va;
+	attr.virt = va;
+	attr.len = qp->attrs.sq_size << SQEBB_SHIFT;
+	attr.req_page_size = SZ_1M - SZ_4K;
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr);
 	if (ret)
 		return ret;
 
 	rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
 	qp->user_qp.rq_offset = rq_offset;
 
-	ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
-			      qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
-			      (SZ_1M - SZ_4K), true);
+	attr.start = va + rq_offset;
+	attr.virt = va + rq_offset;
+	attr.len = qp->attrs.rq_size << RQE_SHIFT;
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr);
 	if (ret)
-		goto put_sq_mtt;
+		goto uninit_sq_mem;
 
 	ret = erdma_map_user_dbrecords(uctx, dbrec_va,
-				       &qp->user_qp.user_dbr_page,
-				       &dbrec_dma);
+				       &qp->user_qp.user_dbr_page, &dbrec_dma);
 	if (ret)
-		goto put_rq_mtt;
+		goto uninit_rq_mem;
 
 	qp->user_qp.sq_dbrec_dma = dbrec_dma;
 	qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
 
 	return 0;
 
-put_rq_mtt:
-	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+uninit_rq_mem:
+	erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
 
-put_sq_mtt:
-	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
+uninit_sq_mem:
+	erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
 
 	return ret;
 }
 
 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
 {
-	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
-	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+	erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
+	erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
 	erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
 }
 
@@ -1234,8 +1297,11 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 				u64 virt, int access, struct ib_dmah *dmah,
 				struct ib_udata *udata)
 {
-	struct erdma_mr *mr = NULL;
 	struct erdma_dev *dev = to_edev(ibpd->device);
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_UMEM,
+	};
+	struct erdma_mr *mr = NULL;
 	u32 stag;
 	int ret;
 
@@ -1249,8 +1315,13 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
-			      SZ_2G - SZ_4K, false);
+	attr.start = start;
+	attr.virt = virt;
+	attr.len = len;
+	attr.req_page_size = SZ_2G - SZ_4K;
+	attr.access = access;
+	attr.flags = ERDMA_MEM_FLAG_MR_BUF;
+	ret = erdma_mem_init(dev, &mr->mem, &attr);
 	if (ret)
 		goto err_out_free;
 
@@ -1260,8 +1331,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 
 	mr->ibmr.lkey = mr->ibmr.rkey = stag;
 	mr->ibmr.pd = ibpd;
-	mr->mem.va = virt;
-	mr->mem.len = len;
 	mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
 	mr->valid = 1;
 	mr->type = ERDMA_MR_TYPE_NORMAL;
@@ -1277,7 +1346,7 @@ err_out_mr:
 		       mr->ibmr.lkey >> 8);
 
 err_out_put_mtt:
-	put_mtt_entries(dev, &mr->mem);
+	erdma_mem_uninit(dev, &mr->mem);
 
 err_out_free:
 	kfree(mr);
@@ -1314,7 +1383,8 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 
 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
 
-	put_mtt_entries(dev, &mr->mem);
+	if (mr->type != ERDMA_MR_TYPE_DMA)
+		erdma_mem_uninit(dev, &mr->mem);
 
 	kfree(mr);
 	return 0;
@@ -1349,13 +1419,12 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	wait_for_completion(&cq->free);
 
 	if (rdma_is_kernel_res(&cq->ibcq.res)) {
-		dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
-				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+		erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
 		dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
 			      cq->kern_cq.dbrec_dma);
 	} else {
 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 	}
 
 	return 0;
@@ -1407,8 +1476,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
 		free_kernel_qp(qp);
 	} else {
-		put_mtt_entries(dev, &qp->user_qp.sq_mem);
-		put_mtt_entries(dev, &qp->user_qp.rq_mem);
+		erdma_mem_uninit(dev, &qp->user_qp.sq_mem);
+		erdma_mem_uninit(dev, &qp->user_qp.rq_mem);
 		erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
 	}
 
@@ -1926,12 +1995,19 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 			      struct erdma_ureq_create_cq *ureq)
 {
-	int ret;
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_UMEM,
+	};
+	int ret;
 
-	ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
-			      ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
-			      true);
+	attr.start = ureq->qbuf_va;
+	attr.virt = ureq->qbuf_va;
+	attr.len = ureq->qbuf_len;
+	attr.req_page_size = SZ_64M - SZ_4K;
+	attr.access = IB_ACCESS_LOCAL_WRITE;
+	attr.flags = ERDMA_MEM_FLAG_CQ_BUF;
+	ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr);
 	if (ret)
 		return ret;
 
@@ -1939,7 +2015,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 				       &cq->user_cq.user_dbr_page,
 				       &cq->user_cq.dbrec_dma);
 	if (ret)
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 
 	return ret;
 }
@@ -1947,12 +2023,15 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 static int erdma_init_kernel_cq(struct erdma_cq *cq)
 {
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_KMEM,
+		.len = cq->depth << CQE_SHIFT,
+	};
+	int ret;
 
-	cq->kern_cq.qbuf =
-		dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
-				   &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
-	if (!cq->kern_cq.qbuf)
-		return -ENOMEM;
+	ret = erdma_mem_init(dev, &cq->kern_cq.qbuf_mem, &attr);
+	if (ret)
+		return ret;
 
 	cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
 					    &cq->kern_cq.dbrec_dma);
@@ -1966,8 +2045,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq)
 	return 0;
 
 err_out:
-	dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
-			  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+	erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
 
 	return -ENOMEM;
 }
@@ -2032,10 +2110,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 err_free_res:
 	if (!rdma_is_kernel_res(&ibcq->res)) {
 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 	} else {
-		dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
-				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+		erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
 		dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
 			      cq->kern_cq.dbrec_dma);
 	}
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index c73cecf92f61..0542e10a0c99 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -111,19 +111,71 @@ struct erdma_mtt {
 	struct erdma_mtt *low_level;
 };
 
+enum erdma_mem_type {
+	ERDMA_NO_MEM = 0,
+	ERDMA_UMEM = 1,
+	ERDMA_KMEM = 2,
+};
+
+struct erdma_buf_list {
+	void *buf;
+	dma_addr_t dma_addr;
+};
+
+struct erdma_kmem {
+	struct erdma_buf_list *buf_list;
+};
+
+enum erdma_mem_flags {
+	ERDMA_MEM_FLAG_MR_BUF = (1 << 0),
+	ERDMA_MEM_FLAG_CQ_BUF = (1 << 1),
+};
+
+struct erdma_mem_init_attr {
+	enum erdma_mem_type type;
+	u64 start;
+	u64 virt;
+	u64 len;
+	unsigned long req_page_size;
+	int access;
+	u32 flags;
+};
+
 struct erdma_mem {
-	struct ib_umem *umem;
-	struct erdma_mtt *mtt;
+	enum erdma_mem_type type;
+	union {
+		struct ib_umem *umem;
+		struct erdma_kmem kmem;
+	};
 
 	u32 page_size;
 	u32 page_offset;
 	u32 page_cnt;
 	u32 mtt_nents;
 
+	struct erdma_mtt *mtt;
+
 	u64 va;
 	u64 len;
 };
 
+static inline void *erdma_kmem_get_entry(struct erdma_mem *mem, u32 idx,
+					 u32 depth, u32 shift)
+{
+	u32 offset = (idx & (depth - 1)) << shift;
+
+	return (u8 *)mem->kmem.buf_list[offset >> PAGE_SHIFT].buf +
+	       offset_in_page(offset);
+}
+
+static inline void erdma_kmem_clear(struct erdma_mem *mem)
+{
+	u32 i;
+
+	for (i = 0; i < mem->page_cnt; i++)
+		memset(mem->kmem.buf_list[i].buf, 0, PAGE_SIZE);
+}
+
 struct erdma_mr {
 	struct ib_mr ibmr;
 	struct erdma_mem mem;
@@ -183,11 +235,8 @@ struct erdma_kqp {
 	void __iomem *hw_sq_db;
 	void __iomem *hw_rq_db;
 
-	void *sq_buf;
-	dma_addr_t sq_buf_dma_addr;
-
-	void *rq_buf;
-	dma_addr_t rq_buf_dma_addr;
+	struct erdma_mem sq_mem;
+	struct erdma_mem rq_mem;
 
 	void *sq_dbrec;
 	void *rq_dbrec;
@@ -320,8 +369,7 @@ struct erdma_qp {
 };
 
 struct erdma_kcq_info {
-	void *qbuf;
-	dma_addr_t qbuf_dma_addr;
+	struct erdma_mem qbuf_mem;
 	u32 ci;
 	u32 cmdsn;
 	u32 notify_cnt;
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index 2b20907c9c12..d4bf75dcd102 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -386,7 +386,7 @@ static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
 
 /*
  * This function creates the table for looking up CPUs for completion vectors.
- * num_comp_vectors needs to have been initilized before calling this function.
+ * num_comp_vectors needs to have been initialized before calling this function.
  */
 static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
 					  struct hfi1_affinity_node *entry)
diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c
index 3c228aeaaf81..a69de0a91238 100644
--- a/drivers/infiniband/hw/hfi1/firmware.c
+++ b/drivers/infiniband/hw/hfi1/firmware.c
@@ -77,7 +77,7 @@ struct css_header {
 /* size of platform configuration partition */
 #define MAX_PLATFORM_CONFIG_FILE_SIZE 4096
 
-/* size of file of plaform configuration encoded in format version 4 */
+/* size of file of platform configuration encoded in format version 4 */
 #define PLATFORM_CONFIG_FORMAT_4_FILE_SIZE 528
 
 /* the file itself */
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index 1154b8cc713c..106bed4932c8 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -53,17 +53,9 @@ int hfi1_pcie_init(struct pci_dev *pdev)
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (ret) {
-		/*
-		 * If the 64 bit setup fails, try 32 bit.  Some systems
-		 * do not setup 64 bit maps on systems with 2GB or less
-		 * memory installed.
-		 */
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (ret) {
-			dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
-				ERR_PTR(ret));
-			goto bail;
-		}
+		dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
+			ERR_PTR(ret));
+		goto bail;
 	}
 
 	pci_set_master(pdev);
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c
index f2b12ae13a58..8d8272705fb7 100644
--- a/drivers/infiniband/hw/hns/hns_roce_bond.c
+++ b/drivers/infiniband/hw/hns/hns_roce_bond.c
@@ -81,7 +81,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,
 {
 	struct net_device *active_dev;
 	struct net_device *old_dev;
-	int i, ret = 0;
+	int i = 0, ret = 0;
 
 	if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
 		rcu_read_lock();
@@ -89,7 +89,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,
 			bond_option_active_slave_get_rcu(netdev_priv(bond_grp->upper_dev));
 		rcu_read_unlock();
 	} else {
-		for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) {
+		for (; i < ROCE_BOND_FUNC_MAX; i++) {
 			active_dev = bond_grp->bond_func_info[i].net_dev;
 			if (active_dev &&
 			    ib_get_curr_port_state(active_dev) == IB_PORT_ACTIVE)
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 1dd0efb5620d..7dfaa01cda67 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -261,10 +261,11 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
 	buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size;
 	buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num;
 	buf_attr.region_count = 1;
+	buf_attr.user_access = IB_ACCESS_LOCAL_WRITE;
 
 	ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
 				  hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT,
-				  udata, addr);
+				  udata, addr, true);
 	if (ret)
 		ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret);
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c
index 103b8c9ca969..4ef0b882ea2f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c
+++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c
@@ -338,6 +338,60 @@ static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev,
 	}
 }
 
+static int gsi_sl_debugfs_show(struct seq_file *file, void *offset)
+{
+	struct hns_roce_dev *hr_dev = file->private;
+
+	seq_printf(file, "%u\n", hr_dev->gsi_sl);
+
+	return 0;
+}
+
+static ssize_t gsi_sl_debugfs_store(char *buf, size_t count, void *data)
+{
+	struct hns_roce_dev *hr_dev = data;
+	struct net_device *netdev;
+	int ret;
+	u8 val;
+
+	netdev = ib_device_get_netdev(&hr_dev->ib_dev, 1);
+	if (!netdev)
+		return -ENODEV;
+
+	if (ib_get_curr_port_state(netdev) == IB_PORT_ACTIVE) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+
+	ret = kstrtou8(buf, 0, &val);
+	if (ret)
+		goto out;
+
+	if (!check_sl_valid(hr_dev, val)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	hr_dev->gsi_sl = val;
+
+out:
+	dev_put(netdev);
+	return ret ? : count;
+}
+
+static void create_gsi_sl_debugfs(struct hns_roce_dev *hr_dev,
+				  struct dentry *parent)
+{
+	struct hns_debugfs_seqfile *seqfile = &hr_dev->dbgfs.gsi_sl;
+
+	seqfile->read = gsi_sl_debugfs_show;
+	seqfile->write = gsi_sl_debugfs_store;
+	seqfile->data = hr_dev;
+
+	debugfs_create_file("gsi_sl", 0600, parent, seqfile,
+			    &hns_debugfs_seqfile_fops);
+}
+
 /* debugfs for device */
 void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
 {
@@ -348,6 +402,7 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
 
 	create_sw_stat_debugfs(hr_dev, dbgfs->root);
 	create_cc_param_debugfs(hr_dev, dbgfs->root);
+	create_gsi_sl_debugfs(hr_dev, dbgfs->root);
 }
 
 void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h
index 116b1e8b6677..286704711785 100644
--- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h
+++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h
@@ -47,6 +47,7 @@ struct hns_roce_dev_debugfs {
 	struct dentry *root;
 	struct hns_sw_stat_debugfs sw_stat_root;
 	struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM];
+	struct hns_debugfs_seqfile gsi_sl;
 };
 
 struct hns_roce_dev;
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index eb7b1865e4c7..5890ee69247c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -654,6 +654,7 @@ struct hns_roce_qp {
 	u8			priority;
 	spinlock_t flush_lock;
 	struct hns_roce_dip *dip;
+	bool ud_sl_set;
 };
 
 struct hns_roce_ib_iboe {
@@ -737,7 +738,7 @@ enum hns_roce_scc_algo {
 struct hns_roce_caps {
 	u64		fw_ver;
 	u8		num_ports;
-	int		gid_table_len[HNS_ROCE_MAX_PORTS];
+	u32		gid_table_len[HNS_ROCE_MAX_PORTS];
 	int		pkey_table_len[HNS_ROCE_MAX_PORTS];
 	int		local_ca_ack_delay;
 	int		num_uars;
@@ -1047,6 +1048,7 @@ struct hns_roce_dev {
 	struct hns_roce_dev_debugfs dbgfs;
 	atomic64_t *dfx_cnt;
 	struct hns_roce_scc_param *scc_param;
+	u8 gsi_sl;
 };
 
 enum hns_roce_trace_type {
@@ -1230,7 +1232,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 			struct hns_roce_buf_attr *buf_attr,
 			unsigned int page_shift, struct ib_udata *udata,
-			unsigned long user_addr);
+			unsigned long user_addr, bool is_cq);
 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
 			  struct hns_roce_mtr *mtr);
 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 27cc7df55ee7..a8f319a57f25 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -431,7 +431,8 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 	return 0;
 }
 
-static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
+static int fill_ud_av(struct hns_roce_qp *qp,
+		      struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 		      struct hns_roce_ah *ah)
 {
 	struct ib_device *ib_dev = ah->ibah.device;
@@ -441,7 +442,13 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit);
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass);
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel);
-	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl);
+	if (!qp->ud_sl_set || qp->ibqp.qp_type == IB_QPT_GSI) {
+		qp->sl = qp->ibqp.qp_type == IB_QPT_GSI ?
+				hr_dev->gsi_sl : ah->av.sl;
+		qp->ud_sl_set = true;
+	}
+
+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, qp->sl);
 
 	ud_sq_wqe->sgid_index = ah->av.gid_index;
 
@@ -491,12 +498,10 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 			  qp->qkey : ud_wr(wr)->remote_qkey);
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn);
 
-	ret = fill_ud_av(ud_sq_wqe, ah);
+	ret = fill_ud_av(qp, ud_sq_wqe, ah);
 	if (ret)
 		return ret;
 
-	qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
-
 	set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
 
 	/*
@@ -2416,8 +2421,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev)
 					 caps->gmv_bt_num *
 					 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz));
 
-		caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
-							  caps->gmv_entry_sz);
+		caps->gmv_entry_num = caps->gid_table_len[0];
 	} else {
 		u32 func_num = max_t(u32, 1, hr_dev->func_num);
 
@@ -3824,7 +3828,8 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
 	 * Now backwards through the CQ, removing CQ entries
 	 * that match our QP by overwriting them with next entries.
 	 */
-	while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
+	while (prod_index != hr_cq->cons_index) {
+		--prod_index;
 		cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
 		if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
 			if (srq && hr_reg_read(cqe, CQE_S_R)) {
@@ -4353,18 +4358,39 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
 static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
 			       dma_addr_t base_addr)
 {
+	u32 obj_num_per_bt = HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz;
+	u32 chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + PAGE_SHIFT);
+	u32 bt_num_per_chunk = chunk_size / HNS_HW_PAGE_SIZE;
+	u32 first = obj / obj_num_per_bt;
+	u32 last = min(first + bt_num_per_chunk, hr_dev->caps.gmv_bt_num);
 	struct hns_roce_cmq_desc desc;
-	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
-	u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);
-	u64 addr = to_hr_hw_page_addr(base_addr);
+	struct hns_roce_cmq_req *req;
+	u64 addr;
+	int ret;
+	u32 i;
 
-	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
+	/* The GMV BT entry of hardware covers a fixed 4K region, so a buffer
+	 * chunk larger than 4K must be registered to hardware with one BT
+	 * entry per 4K block, otherwise the GMV entries beyond the first
+	 * 4K of the chunk are unreachable.
+	 */
+	for (i = first; i < last; i++) {
+		hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT,
+					      false);
+		req = (struct hns_roce_cmq_req *)desc.data;
 
-	hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
-	hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
-	hr_reg_write(req, CFG_GMV_BT_IDX, idx);
+		addr = to_hr_hw_page_addr(base_addr +
+					  (u64)(i - first) * HNS_HW_PAGE_SIZE);
+		hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
+		hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
+		hr_reg_write(req, CFG_GMV_BT_IDX, i);
 
-	return hns_roce_cmq_send(hr_dev, &desc, 1);
+		ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
@@ -5327,7 +5353,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp,
 	hr_reg_clear(qpc_mask, QPC_VLAN_ID);
 
 	if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
-		ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
+		ibdev_err(ibdev, "sgid_index(%u) too large. max is %u\n",
 			  grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
 		return -EINVAL;
 	}
@@ -5603,6 +5629,7 @@ static void v2_set_flushed_fields(struct ib_qp *ibqp,
 	hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);
 	hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);
 	hr_qp->state = IB_QPS_ERR;
+	hr_qp->ud_sl_set = false;
 	spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
 
 	if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */
@@ -6913,7 +6940,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 
 	err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
 				  hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
-				  0);
+				  0, false);
 	if (err)
 		dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 7d41ae897458..4799f667eeda 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -112,7 +112,7 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
 
 	err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,
 				  hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT,
-				  udata, start);
+				  udata, start, false);
 	if (err) {
 		ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err);
 		return err;
@@ -586,7 +586,8 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
 
 static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 			  struct hns_roce_buf_attr *buf_attr,
-			  struct ib_udata *udata, unsigned long user_addr)
+			  struct ib_udata *udata, unsigned long user_addr,
+			  bool is_cq)
 {
 	struct ib_device *ibdev = &hr_dev->ib_dev;
 	size_t total_size;
@@ -595,8 +596,14 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 
 	if (udata) {
 		mtr->kmem = NULL;
-		mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
-					   buf_attr->user_access);
+		if (is_cq)
+			mtr->umem = ib_umem_get_cq_buf_or_va(ibdev, NULL,
+							     user_addr,
+							     total_size,
+							     buf_attr->user_access);
+		else
+			mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
+						   buf_attr->user_access);
 		if (IS_ERR(mtr->umem)) {
 			ibdev_err(ibdev, "failed to get umem, ret = %pe.\n",
 				  mtr->umem);
@@ -1035,11 +1042,13 @@ static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
  * @ba_page_shift: page shift for multi-hop base address table
  * @udata: user space context, if it's NULL, means kernel space
  * @user_addr: userspace virtual address to start at
+ * @is_cq: true when @mtr backs a CQ buffer, which the device writes
+ *         completion entries into
  */
 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 			struct hns_roce_buf_attr *buf_attr,
 			unsigned int ba_page_shift, struct ib_udata *udata,
-			unsigned long user_addr)
+			unsigned long user_addr, bool is_cq)
 {
 	struct ib_device *ibdev = &hr_dev->ib_dev;
 	int ret;
@@ -1052,7 +1061,8 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 		mtr->umem = NULL;
 		mtr->kmem = NULL;
 	} else {
-		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);
+		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr,
+				     is_cq);
 		if (ret) {
 			ibdev_err(ibdev,
 				  "failed to alloc mtr bufs, ret = %d.\n", ret);
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index e333a8c4acb5..c6c0dfdbdbc3 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -808,7 +808,7 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 	}
 	ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
 				  PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
-				  udata, addr);
+				  udata, addr, false);
 	if (ret) {
 		ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
 		goto err_inline;
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index 4a54394f96be..b5e9ac9cfd59 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -180,7 +180,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 
 	ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
 				  hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT,
-				  udata, addr);
+				  udata, addr, false);
 	if (ret) {
 		ibdev_err(ibdev,
 			  "failed to alloc SRQ idx mtr, ret = %d.\n", ret);
@@ -235,7 +235,7 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,
 
 	ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
 				  hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,
-				  udata, addr);
+				  udata, addr, false);
 	if (ret)
 		ibdev_err(ibdev,
 			  "failed to alloc SRQ buf mtr, ret = %d.\n", ret);
diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c
index a70ef59a8064..2b68f377e1fa 100644
--- a/drivers/infiniband/hw/ionic/ionic_controlpath.c
+++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c
@@ -110,8 +110,9 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
 		if (rc)
 			goto err_qdesc;
 
-		cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr,
-					  req_cq->size, IB_ACCESS_LOCAL_WRITE);
+		cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
+						    req_cq->addr, req_cq->size,
+						    IB_ACCESS_LOCAL_WRITE);
 		if (IS_ERR(cq->umem)) {
 			rc = PTR_ERR(cq->umem);
 			goto err_qdesc;
@@ -418,6 +419,8 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
 
 	resp.udma_count = dev->lif_cfg.udma_count;
 	resp.expdb_mask = dev->lif_cfg.expdb_mask;
+	resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit;
+	resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS;
 
 	if (dev->lif_cfg.sq_expdb)
 		resp.expdb_qtypes |= IONIC_EXPDB_SQ;
@@ -1388,7 +1391,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
 			       struct ionic_qp *qp,
 			       struct ionic_tbl_buf *sq_buf,
 			       struct ionic_tbl_buf *rq_buf,
-			       struct ib_qp_init_attr *attr)
+			       struct ib_qp_init_attr *attr,
+			       u32 ionic_flags)
 {
 	const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject);
 	const u32 flags = to_ionic_qp_flags(0, 0,
@@ -1404,7 +1408,12 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN),
 			.cmd.create_qp = {
 				.pd_id = cpu_to_le32(pd->pdid),
-				.priv_flags = cpu_to_be32(flags),
+				/* User-supplied ionic_flags are passed through to
+				 * firmware, which validates and rejects any
+				 * unsupported or unauthorized bits.
+				 */
+				.priv_flags = cpu_to_be32(flags |
+						(ionic_flags & IONIC_QP_USER_FLAGS_MASK)),
 				.type_state = to_ionic_qp_type(attr->qp_type),
 				.dbid_flags = cpu_to_le16(dbid),
 				.id_ver = cpu_to_le32(qp->qpid),
@@ -2221,6 +2230,12 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		rc = ib_copy_validate_udata_in(udata, req, rsvd);
 		if (rc)
 			return rc;
+
+		if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK)
+			return -EINVAL;
+
+		if (req.rsvd_pad)
+			return -EINVAL;
 	} else {
 		req.sq_spec = IONIC_SPEC_HIGH;
 		req.rq_spec = IONIC_SPEC_HIGH;
@@ -2301,7 +2316,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 	rc = ionic_create_qp_cmd(dev, pd,
 				 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx),
 				 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx),
-				 qp, &sq_buf, &rq_buf, attr);
+				 qp, &sq_buf, &rq_buf, attr, req.ionic_flags);
 	if (rc)
 		goto err_cmd;
 
@@ -2336,7 +2351,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 						    &resp.sq_cmb_offset);
 			if (!qp->mmap_sq_cmb) {
 				rc = -ENOMEM;
-				goto err_mmap_sq;
+				goto err_qp_cmd;
 			}
 
 			resp.sq_cmb = qp->sq_cmb;
@@ -2373,7 +2388,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 						    &resp.rq_cmb_offset);
 			if (!qp->mmap_rq_cmb) {
 				rc = -ENOMEM;
-				goto err_mmap_rq;
+				goto err_qp_cmd;
 			}
 
 			resp.rq_cmb = qp->rq_cmb;
@@ -2381,7 +2396,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 
 		rc = ib_respond_udata(udata, resp);
 		if (rc)
-			goto err_resp;
+			goto err_qp_cmd;
 	}
 
 	ionic_pgtbl_unbuf(dev, &rq_buf);
@@ -2399,7 +2414,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		else
 			rc = xa_err(entry);
 
-		goto err_resp;
+		goto err_qp_cmd;
 	}
 
 	if (qp->has_sq) {
@@ -2430,13 +2445,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 
 	return 0;
 
-err_resp:
-	if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE))
-		rdma_user_mmap_entry_remove(qp->mmap_rq_cmb);
-err_mmap_rq:
-	if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE))
-		rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
-err_mmap_sq:
+err_qp_cmd:
 	ionic_destroy_qp_cmd(dev, qp->qpid);
 err_cmd:
 	ionic_pgtbl_unbuf(dev, &rq_buf);
diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h
index ee23062a1762..d0a5bce85ed8 100644
--- a/drivers/infiniband/hw/ionic/ionic_fw.h
+++ b/drivers/infiniband/hw/ionic/ionic_fw.h
@@ -105,6 +105,8 @@ enum ionic_qp_flags {
 	IONIC_QPF_SQ_CMB		= BIT(13),
 	IONIC_QPF_RQ_CMB		= BIT(14),
 	IONIC_QPF_PRIVILEGED		= BIT(15),
+
+	IONIC_QP_USER_FLAGS_MASK	= GENMASK(31, 16),
 };
 
 static inline int from_ionic_qp_flags(int flags)
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c
index cba7809ec3d9..fa79732c2f7d 100644
--- a/drivers/infiniband/hw/ionic/ionic_ibdev.c
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c
@@ -320,8 +320,8 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)
 
 	ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg);
 
-	xa_init_flags(&dev->qp_tbl, GFP_ATOMIC);
-	xa_init_flags(&dev->cq_tbl, GFP_ATOMIC);
+	xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ);
 
 	ionic_init_resids(dev);
 
diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
index 8e48c00d4959..1e37bd09490f 100644
--- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
+++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
@@ -86,6 +86,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)
 	cfg->udma_count = 2;
 
 	cfg->max_stride = ident->rdma.max_stride;
+	cfg->rcq_sign_bit = ident->rdma.rcq_sign_bit;
 	cfg->expdb_mask = ionic_get_expdb(lif);
 
 	cfg->sq_expdb =
diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
index 2b29e646c193..d7835ac27896 100644
--- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
+++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
@@ -57,6 +57,7 @@ struct ionic_lif_cfg {
 	bool sq_expdb;
 	bool rq_expdb;
 	u8 expdb_mask;
+	u8 rcq_sign_bit;
 };
 
 void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg);
diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
index 335ae3c82e17..0db73fd7d01b 100644
--- a/drivers/infiniband/hw/irdma/ctrl.c
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -1689,11 +1689,9 @@ static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev,
  * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
  * @qp: sc qp struct
  * @info: fast mr info
- * @post_sq: flag for cqp db to ring
  */
 int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
-			      struct irdma_fast_reg_stag_info *info,
-			      bool post_sq)
+			      struct irdma_fast_reg_stag_info *info)
 {
 	u64 temp, hdr;
 	__le64 *wqe;
@@ -1723,9 +1721,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 		  info->wr_id, wqe_idx,
 		  &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
 
-	temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
-		(uintptr_t)info->va : info->fbo;
-	set_64bit_val(wqe, 0, temp);
+	set_64bit_val(wqe, 0, info->va);
 
 	temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
 			 info->first_pm_pbl_index >> 16);
@@ -1742,7 +1738,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 	      FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
 	      FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
 	      FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
-	      FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
+	      IRDMAQPSQ_VABASEDTO |
 	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
 	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
 	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
@@ -1755,8 +1751,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 	print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,
 			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);
 
-	if (post_sq)
-		irdma_uk_qp_post_wr(&qp->qp_uk);
+	irdma_uk_qp_post_wr(&qp->qp_uk);
 
 	return 0;
 }
@@ -2625,11 +2620,9 @@ static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
  * @qp: sc qp
  * @info: dlush information
  * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
  */
 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
-			   struct irdma_qp_flush_info *info, u64 scratch,
-			   bool post_sq)
+			   struct irdma_qp_flush_info *info, u64 scratch)
 {
 	u64 temp = 0;
 	__le64 *wqe;
@@ -2697,8 +2690,7 @@ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
 
 	print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,
 			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
-	if (post_sq)
-		irdma_sc_cqp_post_sq(cqp);
+	irdma_sc_cqp_post_sq(cqp);
 
 	return 0;
 }
@@ -6114,8 +6106,7 @@ static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
 	case IRDMA_OP_QP_FLUSH_WQES:
 		status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
 						&pcmdinfo->in.u.qp_flush_wqes.info,
-						pcmdinfo->in.u.qp_flush_wqes.scratch,
-						pcmdinfo->post_sq);
+						pcmdinfo->in.u.qp_flush_wqes.scratch);
 		break;
 	case IRDMA_OP_GEN_AE:
 		status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index c345cc654256..e28f5f4876cd 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -1252,12 +1252,21 @@ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
 	}
 
 	msix_vec->ceq_id = ceq_id;
-	if (rf->sc_dev.privileged)
+	if (rf->sc_dev.privileged) {
 		rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id,
 						  msix_vec->idx, true);
-	else
+	} else {
 		status = irdma_vchnl_req_ceq_vec_map(&rf->sc_dev, ceq_id,
 						     msix_vec->idx);
+		if (!status)
+			return 0;
+
+		if (rf->msix_shared && !ceq_id)
+			irdma_destroy_irq(rf, msix_vec, rf);
+		else
+			irdma_destroy_irq(rf, msix_vec, iwceq);
+	}
+
 	return status;
 }
 
@@ -2742,7 +2751,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
 	hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
 	memcpy(hw_info, info, sizeof(*hw_info));
 	cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
-	cqp_info->post_sq = 1;
 	cqp_info->in.u.qp_flush_wqes.qp = qp;
 	cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
 	status = irdma_handle_cqp_op(rf, cqp_request);
@@ -2789,7 +2797,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
 				hw_info = &new_req->info.in.u.qp_flush_wqes.info;
 				memcpy(hw_info, info, sizeof(*hw_info));
 				cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
-				cqp_info->post_sq = 1;
 				cqp_info->in.u.qp_flush_wqes.qp = qp;
 				cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)new_req;
 
diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c
index 4b451d8482a4..f405ef761f84 100644
--- a/drivers/infiniband/hw/irdma/icrdma_if.c
+++ b/drivers/infiniband/hw/irdma/icrdma_if.c
@@ -318,14 +318,15 @@ static void icrdma_remove(struct auxiliary_device *aux_dev)
 		container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
 	struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
 	struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);
-	u8 rdma_ver = iwdev->rf->rdma_ver;
+	struct irdma_pci_f *rf = iwdev->rf;
+	u8 rdma_ver = rf->rdma_ver;
 
 	ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false);
 	irdma_ib_unregister_device(iwdev);
-	icrdma_deinit_interrupts(iwdev->rf, cdev_info);
-	mutex_destroy(&iwdev->rf->ah_tbl_lock);
+	icrdma_deinit_interrupts(rf, cdev_info);
+	mutex_destroy(&rf->ah_tbl_lock);
 
-	kfree(iwdev->rf);
+	kfree(rf);
 
 	pr_debug("INIT: Gen[%d] func[%d] device remove success\n",
 		 rdma_ver, PCI_FUNC(cdev_info->pdev->devfn));
diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
index 95957d52883d..106915b51e7c 100644
--- a/drivers/infiniband/hw/irdma/main.c
+++ b/drivers/infiniband/hw/irdma/main.c
@@ -80,6 +80,8 @@ static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev,
 		return -ENOMEM;
 	}
 	iwdev = ib_alloc_device(irdma_device, ibdev);
+	if (!iwdev)
+		return -ENOMEM;
 	/* Fill iwdev info */
 	iwdev->is_vport = true;
 	iwdev->rf = rf;
diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c
index 28dfad7f940c..0c2c5e96a05a 100644
--- a/drivers/infiniband/hw/irdma/pble.c
+++ b/drivers/infiniband/hw/irdma/pble.c
@@ -330,7 +330,7 @@ static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
 	if (root->addr)
 		irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo);
 
-	kfree(lvl2->leafmem.va);
+	kvfree(lvl2->leafmem.va);
 	lvl2->leaf = NULL;
 }
 
@@ -358,7 +358,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 	lvl2->leaf_cnt = total;
 
 	lvl2->leafmem.size = (sizeof(*leaf) * total);
-	lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
+	lvl2->leafmem.va = kvzalloc(lvl2->leafmem.size, GFP_KERNEL);
 	if (!lvl2->leafmem.va)
 		return -ENOMEM;
 
@@ -367,7 +367,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 	ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo,
 				       total << 3, &root->addr, &fpm_addr);
 	if (ret_code) {
-		kfree(lvl2->leafmem.va);
+		kvfree(lvl2->leafmem.va);
 		lvl2->leaf = NULL;
 		return -ENOMEM;
 	}
@@ -482,7 +482,7 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 
 		status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
 		/* if level1_only, only go through it once */
-		if (!status || lvl)
+		if (!status || lvl == PBLE_LEVEL_1)
 			break;
 	}
 
diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h
index 324cfbf21764..561a0287b945 100644
--- a/drivers/infiniband/hw/irdma/protos.h
+++ b/drivers/infiniband/hw/irdma/protos.h
@@ -19,8 +19,7 @@ void irdma_sc_rt_init(struct irdma_sc_dev *dev);
 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
 __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
 int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
-			      struct irdma_fast_reg_stag_info *info,
-			      bool post_sq);
+			      struct irdma_fast_reg_stag_info *info);
 /* HMC/FPM functions */
 int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);
 /* stats misc */
diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h
index 5557d9338796..8b4d2c0b283f 100644
--- a/drivers/infiniband/hw/irdma/type.h
+++ b/drivers/infiniband/hw/irdma/type.h
@@ -1173,13 +1173,11 @@ struct irdma_reg_ns_stag_info {
 struct irdma_fast_reg_stag_info {
 	u64 wr_id;
 	u64 reg_addr_pa;
-	u64 fbo;
-	void *va;
+	u64 va;
 	u64 total_len;
 	u32 page_size;
 	u32 chunk_size;
 	u32 first_pm_pbl_index;
-	enum irdma_addressing_type addr_type;
 	irdma_stag_index stag_idx;
 	u16 access_rights;
 	u32 pd_id;
@@ -1383,8 +1381,7 @@ int irdma_sc_qp_create(struct irdma_sc_qp *qp,
 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
 			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
-			   struct irdma_qp_flush_info *info, u64 scratch,
-			   bool post_sq);
+			   struct irdma_qp_flush_info *info, u64 scratch);
 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
 		       struct irdma_modify_qp_info *info, u64 scratch,
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 5d71300da0ff..22a8511fffea 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -4276,16 +4276,15 @@ static int irdma_post_send(struct ib_qp *ibqp,
 			stag_info.stag_idx = reg_wr(ib_wr)->key >> 8;
 			stag_info.page_size = reg_wr(ib_wr)->mr->page_size;
 			stag_info.wr_id = ib_wr->wr_id;
-			stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;
-			stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
+			stag_info.va = iwmr->ibmr.iova;
 			stag_info.total_len = iwmr->ibmr.length;
 			stag_info.reg_addr_pa = *palloc->level1.addr;
 			stag_info.first_pm_pbl_index = palloc->level1.idx;
 			stag_info.local_fence = true;
 			if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
 				stag_info.chunk_size = 1;
-			err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
-							true);
+			err = irdma_sc_mr_fast_register(&iwqp->sc_qp,
+							&stag_info);
 			break;
 		}
 		default:
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index d4e5e3f91268..843e2c64b32b 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -5,6 +5,58 @@
 
 #include "mana_ib.h"
 
+static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
+{
+	switch (vendor_error) {
+	case VENDOR_ERR_OK:
+		return IB_WC_SUCCESS;
+	case VENDOR_ERR_RX_PKT_LEN:
+	case VENDOR_ERR_RX_MSG_LEN_OVFL:
+		return IB_WC_LOC_LEN_ERR;
+	case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE:
+	case VENDOR_ERR_TX_PCIE_WQE:
+	case VENDOR_ERR_TX_PCIE_MSG:
+	case VENDOR_ERR_RX_MALFORMED_WQE:
+	case VENDOR_ERR_TX_GDMA_INVALID_STATE:
+	case VENDOR_ERR_TX_MISBEHAVING_CLIENT:
+	case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE:
+	case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD:
+	case VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED:
+	case VENDOR_ERR_TX_RDMA_WQE_LEN_ERR:
+	case VENDOR_ERR_TX_RDMA_MTU_ERR:
+		return IB_WC_LOC_QP_OP_ERR;
+	case VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION:
+	case VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE:
+	case VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR:
+	case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION:
+	case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE:
+	case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR:
+	case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE:
+	case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG:
+		return IB_WC_LOC_PROT_ERR;
+	case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT:
+	case VENDOR_ERR_RX_GFID:
+		return IB_WC_LOC_ACCESS_ERR;
+	case VENDOR_ERR_RX_MISBEHAVING_CLIENT:
+	case VENDOR_ERR_RX_CLIENT_ID:
+	case VENDOR_ERR_RX_PCIE:
+	case VENDOR_ERR_RX_NO_AVAIL_WQE:
+	case VENDOR_ERR_RX_ATB_WQE_MISCONFIG:
+	case VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT:
+	case VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE:
+	case VENDOR_ERR_TX_RDMA_INVALID_STATE:
+	case VENDOR_ERR_TX_RDMA_INVALID_NPT:
+	case VENDOR_ERR_TX_RDMA_INVALID_SGID:
+	case VENDOR_ERR_TX_RDMA_VFID_MISMATCH:
+		return IB_WC_FATAL_ERR;
+	case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE:
+	case VENDOR_ERR_SW_FLUSHED:
+		return IB_WC_WR_FLUSH_ERR;
+	default:
+		return IB_WC_GENERAL_ERR;
+	}
+}
+
 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct uverbs_attr_bundle *attrs)
 {
@@ -40,7 +92,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 		cq->cqe = attr->cqe;
 		err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
-					   &cq->queue);
+					   &cq->queue, true);
 		if (err) {
 			ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
 			return err;
@@ -64,6 +116,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		doorbell = mdev->gdma_dev->doorbell;
 	}
 
+	ibcq->cqe = cq->cqe;
+	cq->poll_credit = (cq->cqe << (GDMA_CQE_OWNER_BITS - 1)) - 1;
+
 	if (is_rnic_cq) {
 		err = mana_ib_gd_create_cq(mdev, cq, doorbell);
 		if (err) {
@@ -86,8 +141,8 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	}
 
 	spin_lock_init(&cq->cq_lock);
-	INIT_LIST_HEAD(&cq->list_send_qp);
-	INIT_LIST_HEAD(&cq->list_recv_qp);
+	INIT_LIST_HEAD(&cq->send_err_qp_list);
+	INIT_LIST_HEAD(&cq->recv_err_qp_list);
 
 	return 0;
 
@@ -174,171 +229,290 @@ void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 	gc->cq_table[cq->queue.id] = NULL;
 }
 
+static inline bool gdma_cq_idx_produced(struct gdma_queue *gdma_cq, uint32_t idx)
+{
+	struct gdma_mem_info *gmi = &gdma_cq->mem_info;
+	u32 num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE;
+	u32 expected_bits = (idx / num_cqe) & GDMA_CQE_OWNER_MASK;
+	u32 offset = (idx % num_cqe) * GDMA_CQE_SIZE;
+	struct gdma_cqe *cqe;
+
+	if (gmi->nr_pages)
+		cqe = gmi->pages_va[offset / PAGE_SIZE] +
+		      (offset & (PAGE_SIZE - 1));
+	else
+		cqe = gdma_cq->queue_mem_ptr + offset;
+
+	return cqe->cqe_info.owner_bits == expected_bits;
+}
+
+static inline void mana_ib_cq_doorbell(struct mana_ib_cq *cq, uint8_t arm)
+{
+	struct mana_ib_dev *mdev = container_of(cq->ibcq.device, struct mana_ib_dev, ib_dev);
+	struct gdma_queue *gdma_cq = cq->queue.kmem;
+	u32 num_cqe, max_credit, idx;
+
+	num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE;
+	max_credit = num_cqe << (GDMA_CQE_OWNER_BITS - 1);
+	idx = gdma_cq->head;
+
+	if (cq->poll_credit >= max_credit) {
+		if (gdma_cq_idx_produced(gdma_cq, idx + cq->poll_credit - max_credit))
+			cq->poll_credit++;
+		else
+			return;
+	} else {
+		/* Set index of already polled CQE for unarm */
+		cq->poll_credit = max_credit - (arm ? 0 : 1);
+	}
+
+	idx += (cq->poll_credit - max_credit);
+	idx %= (num_cqe << GDMA_CQE_OWNER_BITS);
+
+	mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), gdma_cq, idx, arm, 0);
+}
+
 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 {
 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 	struct gdma_queue *gdma_cq = cq->queue.kmem;
+	unsigned long irq_flags;
 
 	if (!gdma_cq)
 		return -EINVAL;
 
-	mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
+	spin_lock_irqsave(&cq->cq_lock, irq_flags);
+	mana_ib_cq_doorbell(cq, SET_ARM_BIT);
+	spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
+
 	return 0;
 }
 
-static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
-{
-	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
-	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
-	struct ud_sq_shadow_wqe *shadow_wqe;
+struct mana_cq_poll {
+	struct ib_wc *wc;
+	int budget;
+	int produced;
+};
 
-	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
-	if (!shadow_wqe)
-		return;
+static struct ib_wc *mana_fill_wc(struct mana_ib_qp *qp,
+				  struct mana_cq_poll *poll,
+				  const struct shadow_wqe_header *wqe,
+				  enum ib_wc_opcode opcode, u32 vendor_error)
+{
+	struct ib_wc *wc = &poll->wc[poll->produced++];
 
-	shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
+	memset(wc, 0, sizeof(*wc));
+	wc->wr_id = wqe->wr_id;
+	wc->status = vendor_error_to_wc_error(vendor_error);
+	wc->opcode = opcode;
+	wc->vendor_err = vendor_error;
+	wc->qp = &qp->ibqp;
 
-	wq->tail += shadow_wqe->header.posted_wqe_size;
-	shadow_queue_advance_next_to_complete(&qp->shadow_sq);
+	return wc;
 }
 
-static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
+static void mana_complete_send(struct mana_ib_qp *qp,
+			       struct mana_cq_poll *poll, u32 vendor_error)
 {
-	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
-	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
-	struct ud_rq_shadow_wqe *shadow_wqe;
+	struct shadow_queue *shadow = &qp->shadow_sq;
+	struct shadow_wqe_header *wqe = shadow_queue_get_next_to_consume(shadow);
+	struct gdma_queue *queue;
 
-	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
-	if (!shadow_wqe)
+	if (!wqe)
 		return;
 
-	shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
-	shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
-	shadow_wqe->header.error_code = IB_WC_SUCCESS;
+	if (vendor_error || !(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
+		mana_fill_wc(qp, poll, wqe, wqe->send_opcode, vendor_error);
 
-	wq->tail += shadow_wqe->header.posted_wqe_size;
-	shadow_queue_advance_next_to_complete(&qp->shadow_rq);
+	queue = mana_qp_get_sq(qp)->kmem;
+	queue->tail += wqe->wqe_size_in_bu;
+	shadow_queue_advance_consumer(shadow);
 }
 
-static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
+static void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct mana_rdma_cqe *rdma_cqe,
+			     struct mana_cq_poll *poll)
 {
-	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
+	u32 offset = rdma_cqe->ud_send.tx_wqe_offset & MANA_WQE_OFFSET_MASK;
+	struct shadow_queue *shadow = &qp->shadow_sq;
+	struct shadow_wqe_header *wqe;
+	u64 idx = shadow->cons_idx;
+	u32 to_complete = 0;
+	u64 prod_idx;
+
+	/* Pair with posting's release of the initialized shadow entries. */
+	prod_idx = smp_load_acquire(&shadow->prod_idx);
+	/* Find the target before retiring any entries: the CQE may be stale. */
+	for (; idx != prod_idx; idx++) {
+		wqe = shadow_queue_get_element(shadow, idx);
+		to_complete++;
+		if (wqe->wqe_offset_or_psn == offset)
+			break;
+		if (!(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
+			return;
+	}
+	if (idx == prod_idx)
+		return;
 
-	if (!qp)
+	for (; to_complete; to_complete--)
+		mana_complete_send(qp, poll, VENDOR_ERR_OK);
+}
+
+static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe,
+			  struct mana_cq_poll *poll)
+{
+	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
+	u32 offset = rdma_cqe->ud_recv.rx_wqe_offset / GDMA_WQE_BU_SIZE;
+	struct mana_ib_queue *rq = mana_qp_get_rq(qp);
+	struct gdma_queue *wq = rq->kmem;
+	struct shadow_wqe_header *wqe;
+	struct ib_wc *wc;
+
+	wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq);
+	if (!wqe || wqe->wqe_offset_or_psn != (offset & MANA_WQE_OFFSET_MASK))
 		return;
 
-	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
-		if (cqe->is_sq)
-			handle_ud_sq_cqe(qp, cqe);
-		else
-			handle_ud_rq_cqe(qp, cqe);
+	wc = mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_OK);
+	switch (rdma_cqe->cqe_type) {
+	case CQE_TYPE_UD_SEND_IMM:
+		wc->ex.imm_data = cpu_to_be32(rdma_cqe->ud_recv.imm_data);
+		wc->wc_flags |= IB_WC_WITH_IMM;
+		fallthrough;
+	case CQE_TYPE_UD_SEND:
+		wc->byte_len = rdma_cqe->ud_recv.msg_len;
+		wc->src_qp = rdma_cqe->ud_recv.src_qpn;
+		wc->wc_flags |= IB_WC_GRH;
+		break;
+	default:
+		break;
 	}
 
-	mana_put_qp_ref(qp);
+	wq->tail += wqe->wqe_size_in_bu;
+	shadow_queue_advance_consumer(&qp->shadow_rq);
 }
 
-static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
-				       const struct shadow_wqe_header *shadow_wqe)
+static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
+			    struct mana_cq_poll *poll)
 {
-	const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
+	struct gdma_comp *cqe = &cq->pending_cqe;
+	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
+	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
 
-	wc->wr_id = shadow_wqe->wr_id;
-	wc->status = shadow_wqe->error_code;
-	wc->opcode = shadow_wqe->opcode;
-	wc->vendor_err = shadow_wqe->error_code;
-	wc->wc_flags = 0;
-	wc->qp = &qp->ibqp;
-	wc->pkey_index = 0;
+	if (!qp)
+		return true;
 
-	if (shadow_wqe->opcode == IB_WC_RECV) {
-		wc->byte_len = ud_wqe->byte_len;
-		wc->src_qp = ud_wqe->src_qpn;
-		wc->wc_flags |= IB_WC_GRH;
+	switch (rdma_cqe->cqe_type) {
+	case CQE_TYPE_UD_SEND:
+		if (cqe->is_sq) {
+			handle_ud_sq_cqe(qp, rdma_cqe, poll);
+			break;
+		}
+		fallthrough;
+	case CQE_TYPE_UD_SEND_IMM:
+		handle_rq_cqe(qp, cqe, poll);
+		break;
+	default:
+		ibdev_warn_ratelimited(qp->ibqp.device, "Unexpected CQE type %u\n",
+				       rdma_cqe->cqe_type);
+		break;
 	}
+	mana_put_qp_ref(qp);
+	return true;
 }
 
-static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
+static void mana_flush_completions(struct mana_ib_cq *cq, struct mana_cq_poll *poll)
 {
-	struct shadow_wqe_header *shadow_wqe;
+	struct shadow_wqe_header *wqe;
 	struct mana_ib_qp *qp;
-	int wc_index = 0;
 
-	/* process send shadow queue completions  */
-	list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
-		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
-				!= NULL) {
-			if (wc_index >= nwc)
-				goto out;
+	if (poll->produced >= poll->budget)
+		return;
 
-			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
-			shadow_queue_advance_consumer(&qp->shadow_sq);
-			wc_index++;
-		}
+	list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) {
+		while (poll->produced < poll->budget &&
+		       shadow_queue_get_next_to_consume(&qp->shadow_sq))
+			mana_complete_send(qp, poll, VENDOR_ERR_SW_FLUSHED);
+		if (poll->produced == poll->budget)
+			return;
 	}
 
-	/* process recv shadow queue completions */
-	list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
-		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
-				!= NULL) {
-			if (wc_index >= nwc)
-				goto out;
-
-			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
+	list_for_each_entry(qp, &cq->recv_err_qp_list, recv_err_node) {
+		while (poll->produced < poll->budget &&
+		       (wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))) {
+			mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_SW_FLUSHED);
 			shadow_queue_advance_consumer(&qp->shadow_rq);
-			wc_index++;
 		}
+		if (poll->produced == poll->budget)
+			return;
 	}
-
-out:
-	return wc_index;
 }
 
-void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
+static void mana_drain_gsi_sq(struct mana_ib_qp *qp)
 {
-	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false);
-	struct ud_sq_shadow_wqe *shadow_wqe;
-	struct mana_ib_cq *cq;
+	struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
 	unsigned long flags;
 
-	if (!qp)
-		return;
-
-	cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
-
 	spin_lock_irqsave(&cq->cq_lock, flags);
-	while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq))
-			!= NULL) {
-		shadow_wqe->header.error_code = IB_WC_GENERAL_ERR;
-		shadow_queue_advance_next_to_complete(&qp->shadow_sq);
-	}
+	if (list_empty(&qp->send_err_node))
+		list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 
 	if (cq->ibcq.comp_handler)
 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
 
-	mana_put_qp_ref(qp);
+void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
+{
+	struct mana_ib_qp *qp;
+	u32 port;
+
+	/* One GSI QP per port, indexed in the QP table by (port << 24 | MANA_GSI_QPN) */
+	for (port = 1; port <= mdev->ib_dev.phys_port_cnt; port++) {
+		qp = mana_get_qp_ref(mdev, (port << 24) | MANA_GSI_QPN, false);
+		if (!qp)
+			continue;
+
+		mana_drain_gsi_sq(qp);
+		mana_put_qp_ref(qp);
+	}
 }
 
 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 {
 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 	struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
+	struct mana_cq_poll poll = { .wc = wc, .budget = num_entries, .produced = 0 };
 	struct gdma_queue *queue = cq->queue.kmem;
-	struct gdma_comp gdma_cqe;
 	unsigned long flags;
-	int num_polled = 0;
-	int comp_read, i;
+	bool consumed;
+	int comp_read;
+
+	if (!queue)
+		return -EINVAL;
 
 	spin_lock_irqsave(&cq->cq_lock, flags);
-	for (i = 0; i < num_entries; i++) {
-		comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
-		if (comp_read < 1)
-			break;
-		mana_handle_cqe(mdev, &gdma_cqe);
+	while (poll.produced < poll.budget) {
+		if (!cq->has_pending_cqe) {
+			comp_read = mana_gd_poll_cq(queue, &cq->pending_cqe, 1);
+			if (comp_read < 0) {
+				if (!poll.produced)
+					poll.produced = comp_read;
+				goto out;
+			}
+			if (!comp_read)
+				break;
+
+			cq->poll_credit--;
+			if (!cq->poll_credit)
+				mana_ib_cq_doorbell(cq, 0);
+		}
+
+		consumed = mana_handle_cqe(cq, mdev, &poll);
+		cq->has_pending_cqe = !consumed;
 	}
 
-	num_polled = mana_process_completions(cq, num_entries, wc);
+	mana_flush_completions(cq, &poll);
+out:
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 
-	return num_polled;
+	return poll.produced;
 }
diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index a8d19586a78a..62c7803ca6c6 100644
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -103,8 +103,7 @@ static int mana_ib_netdev_event(struct notifier_block *this,
 				ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
 
 				/* mana_get_primary_netdev() returns ndev with refcount held */
-				if (ndev)
-					netdev_put(ndev, &dev->dev_tracker);
+				netdev_put(ndev, &dev->dev_tracker);
 
 				return NOTIFY_OK;
 			default:
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index e4414d208d4a..858bbc325eb2 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -247,6 +247,9 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
 
 	ucontext->doorbell = doorbell_page;
 	ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT;
+	ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT;
+	if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT)
+		ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT;
 	ret = ib_respond_udata(udata, ucmd_resp);
 	if (ret)
 		return ret;
@@ -292,7 +295,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu
 }
 
 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
-			 struct mana_ib_queue *queue)
+			 struct mana_ib_queue *queue, bool is_cq)
 {
 	struct ib_umem *umem;
 	int err;
@@ -301,7 +304,12 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
 	queue->id = INVALID_QUEUE_ID;
 	queue->gdma_region = GDMA_INVALID_DMA_REGION;
 
-	umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
+	if (is_cq)
+		umem = ib_umem_get_cq_buf_or_va(&mdev->ib_dev, NULL, addr,
+						size, IB_ACCESS_LOCAL_WRITE);
+	else
+		umem = ib_umem_get_va(&mdev->ib_dev, addr, size,
+				      IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(umem)) {
 		ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem);
 		return PTR_ERR(umem);
@@ -592,7 +600,11 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 
 	if (mana_ib_is_rnic(dev)) {
-		if (port_num == 1) {
+		bool port_supports_cm = (port_num == 1 ||
+			(dev->adapter_caps.feature_flags &
+			 MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT));
+
+		if (port_supports_cm) {
 			immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 			immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 		} else {
@@ -670,10 +682,14 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
 	ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width);
 	props->pkey_tbl_len = 1;
 	if (mana_ib_is_rnic(dev)) {
+		bool port_supports_cm = (port == 1 ||
+			(dev->adapter_caps.feature_flags &
+			 MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT));
+
 		props->gid_tbl_len = 16;
 		props->ip_gids = true;
 		props->max_msg_sz = SZ_16M;
-		if (port == 1)
+		if (port_supports_cm)
 			props->port_cap_flags = IB_PORT_CM_SUP;
 	}
 
@@ -1059,6 +1075,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	req.max_recv_sge = attr->cap.max_recv_sge;
 	req.flags = flags;
 
+	if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE)
+		req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
+
 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
 	if (err)
 		return err;
@@ -1134,6 +1153,7 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	struct gdma_context *gc = mdev_to_gc(mdev);
 	struct mana_rnic_create_udqp_resp resp = {};
 	struct mana_rnic_create_udqp_req req = {};
+	struct net_device *ndev;
 	int err, i;
 
 	mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp));
@@ -1150,6 +1170,21 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	req.max_send_sge = attr->cap.max_send_sge;
 	req.max_recv_sge = attr->cap.max_recv_sge;
 	req.qp_type = type;
+
+	/* For GSI QPs, pass the vNIC MAC so the SoC can associate the QP with
+	 * the correct port, transition to INIT, and allocate a per-port QPN.
+	 * MAC must be in reversed byte order.
+	 */
+	if (type == IB_QPT_GSI &&
+	    (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)) {
+		ndev = mana_ib_get_netdev(&mdev->ib_dev, attr->port_num);
+		if (ndev) {
+			copy_in_reverse(req.mac, ndev->dev_addr, ETH_ALEN);
+			req.flags = MANA_UD_QP_FLAG_CREATE_IN_INIT;
+			req.hdr.req.msg_version = GDMA_MESSAGE_V2;
+		}
+	}
+
 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
 	if (err)
 		return err;
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index f69866696776..9173ca64d2e1 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -24,8 +24,12 @@
 /* MANA doesn't have any limit for MR size */
 #define MANA_IB_MAX_MR_SIZE	U64_MAX
 
-/* Send queue ID mask */
-#define MANA_SENDQ_MASK	BIT(31)
+/*
+ * Send queue ID mask. Queue IDs are 2-bit aligned (see MANA_QID_SUBTYPE_MASK),
+ * so bit 0 is always free to tag send queues in the lookup table. This keeps
+ * the whole top byte available to index per-port GSI QPs by (port << 24).
+ */
+#define MANA_SENDQ_MASK	BIT(0)
 /* Queue ID encodes type in the lower 2 bits */
 #define MANA_QID_SUBTYPE_MASK 0x3
 
@@ -162,17 +166,20 @@ struct mana_ib_cq {
 	struct mana_ib_queue queue;
 	/* protects CQ polling */
 	spinlock_t cq_lock;
-	struct list_head list_send_qp;
-	struct list_head list_recv_qp;
+	struct list_head send_err_qp_list;
+	struct list_head recv_err_qp_list;
+	struct gdma_comp pending_cqe;
+	bool has_pending_cqe;
 	int cqe;
 	u32 comp_vector;
+	u32 poll_credit;
 	mana_handle_t  cq_handle;
 };
 
 enum mana_rc_queue_type {
 	MANA_RC_SEND_QUEUE_REQUESTER = 0,
 	MANA_RC_SEND_QUEUE_RESPONDER,
-	MANA_RC_SEND_QUEUE_FMR,
+	MANA_RC_SEND_QUEUE_MMQ,
 	MANA_RC_RECV_QUEUE_REQUESTER,
 	MANA_RC_RECV_QUEUE_RESPONDER,
 	MANA_RC_QUEUE_TYPE_MAX,
@@ -180,6 +187,7 @@ enum mana_rc_queue_type {
 
 struct mana_ib_rc_qp {
 	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
+	u32 wqe_size_in_bu;
 };
 
 enum mana_uc_queue_type {
@@ -201,7 +209,6 @@ enum mana_ud_queue_type {
 
 struct mana_ib_ud_qp {
 	struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX];
-	u32 sq_psn;
 };
 
 struct mana_ib_qp {
@@ -217,9 +224,14 @@ struct mana_ib_qp {
 
 	/* The port on the IB device, starting with 1 */
 	u32 port;
+	u32 sq_psn;
+	bool sq_sig_all;
+
+	/* Serializes QP modification and error-list transitions. */
+	struct mutex modify_lock;
 
-	struct list_head cq_send_list;
-	struct list_head cq_recv_list;
+	struct list_head send_err_node;
+	struct list_head recv_err_node;
 	struct shadow_queue shadow_rq;
 	struct shadow_queue shadow_sq;
 
@@ -262,6 +274,9 @@ enum mana_ib_adapter_features {
 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
 	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
 	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
+	MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
+	MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14),
+	MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15),
 };
 
 struct mana_ib_query_adapter_caps_resp {
@@ -372,7 +387,9 @@ struct mana_rnic_destroy_cq_resp {
 }; /* HW Data */
 
 enum mana_rnic_create_rc_flags {
-	MANA_RC_FLAG_NO_FMR = 2,
+	MANA_RC_FLAG_NO_MMQ = BIT(1),
+	MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3),
+	MANA_RC_FLAG_MSN_IN_WQE = BIT(4),
 };
 
 struct mana_rnic_create_qp_req {
@@ -389,7 +406,8 @@ struct mana_rnic_create_qp_req {
 	u32 max_recv_wr;
 	u32 max_send_sge;
 	u32 max_recv_sge;
-	u32 reserved;
+	u8 wqe_size_in_bu;
+	u8 reserved[3];
 }; /* HW Data */
 
 struct mana_rnic_create_qp_resp {
@@ -445,8 +463,14 @@ struct mana_rnic_create_udqp_req {
 	u32 max_recv_wr;
 	u32 max_send_sge;
 	u32 max_recv_sge;
+	u8 mac[ETH_ALEN];     /* V2: port MAC for multi-port GSI */
+	u16 flags;            /* V2: MANA_UD_QP_FLAG_* */
 }; /* HW Data */
 
+enum mana_ud_qp_flags {
+	MANA_UD_QP_FLAG_CREATE_IN_INIT = BIT(0),
+};
+
 struct mana_rnic_create_udqp_resp {
 	struct gdma_resp_hdr hdr;
 	mana_handle_t qp_handle;
@@ -508,7 +532,16 @@ struct mana_rnic_set_qp_state_resp {
 
 enum WQE_OPCODE_TYPES {
 	WQE_TYPE_UD_SEND = 0,
+	WQE_TYPE_RC_SEND = 2,
+	WQE_TYPE_RC_SEND_IMM = 3,
+	WQE_TYPE_RC_SEND_INV = 4,
+	WQE_TYPE_WRITE = 5,
+	WQE_TYPE_WRITE_IMM = 6,
+	WQE_TYPE_READ = 7,
 	WQE_TYPE_UD_RECV = 8,
+	WQE_TYPE_RC_RECV = 9,
+	WQE_TYPE_REG_MR = 10,
+	WQE_TYPE_LOCAL_INV = 12,
 }; /* HW DATA */
 
 struct rdma_send_oob {
@@ -527,7 +560,83 @@ struct rdma_send_oob {
 			u32 reserved1;
 			u32 reserved2;
 		} ud_send;
+		union {
+			u32 immediate;
+			u32 invalidate_key;
+		} rc_send;
+		struct {
+			u32 address_hi;
+			u32 address_low;
+			u32 rkey;
+			u32 dma_len;
+		} rdma;
+		struct {
+			u32 mkey;
+		} mm;
 	};
+	union {
+		u32 immediate_ext;
+		struct {
+			u16 rsn;
+			u16 reserved;
+		} read;
+	};
+	u32 fsn : 24;
+	u32 reserved2   : 8;
+}; /* HW DATA */
+
+struct rdma_recv_oob {
+	u32 psn_start   : 24;
+	u32 reserved1   : 8;
+	u32 msn         : 24;
+	u32 reserved2   : 8;
+}; /* HW DATA */
+
+enum mana_ib_error_code {
+	VENDOR_ERR_OK					= 0x0,
+	VENDOR_ERR_RX_PKT_LEN                           = 0x05,
+	VENDOR_ERR_RX_MSG_LEN_OVFL                      = 0x102,
+	VENDOR_ERR_RX_MISBEHAVING_CLIENT                = 0x108,
+	VENDOR_ERR_RX_MALFORMED_WQE                     = 0x109,
+	VENDOR_ERR_RX_CLIENT_ID                         = 0x10a,
+	VENDOR_ERR_RX_GFID                              = 0x10b,
+	VENDOR_ERR_RX_PCIE                              = 0x10c,
+	VENDOR_ERR_RX_NO_AVAIL_WQE                      = 0x111,
+	VENDOR_ERR_RX_ATB_SGE_MISSCONFIG                = 0x143,
+	VENDOR_ERR_RX_ATB_WQE_MISCONFIG                 = 0x145,
+	VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT                = 0x183,
+	VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT                = 0x185,
+	VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE                = 0x1c3,
+	VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE                = 0x1c5,
+	VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE              = 0x1c7,
+	VENDOR_ERR_TX_GDMA_CORRUPTED_WQE                = 0x201,
+	VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION          = 0x202,
+	VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE                = 0x203,
+	VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR                = 0x204,
+	VENDOR_ERR_TX_PCIE_WQE                          = 0x205,
+	VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION          = 0x206,
+	VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE                = 0x207,
+	VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR                = 0x208,
+	VENDOR_ERR_TX_PCIE_MSG                          = 0x209,
+	VENDOR_ERR_TX_GDMA_INVALID_STATE                = 0x20a,
+	VENDOR_ERR_TX_MISBEHAVING_CLIENT                = 0x20b,
+	VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE           = 0x210,
+	VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD          = 0x211,
+	VENDOR_ERR_TX_RDMA_INVALID_STATE                = 0x212,
+	VENDOR_ERR_TX_RDMA_INVALID_NPT                  = 0x213,
+	VENDOR_ERR_TX_RDMA_INVALID_SGID                 = 0x214,
+	VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED              = 0x215,
+	VENDOR_ERR_TX_RDMA_WQE_LEN_ERR                  = 0x216,
+	VENDOR_ERR_TX_RDMA_MTU_ERR                      = 0x217,
+	VENDOR_ERR_TX_RDMA_VFID_MISMATCH                = 0x218,
+	VENDOR_ERR_HW_MAX                               = 0x3ff,
+	/* SW vendor errors */
+	VENDOR_ERR_SW_FLUSHED				= 0xfff,
+};
+
+enum mana_ib_cqe_type {
+	CQE_TYPE_UD_SEND = 1,
+	CQE_TYPE_UD_SEND_IMM = 2,
 }; /* HW DATA */
 
 struct mana_rdma_cqe {
@@ -538,8 +647,7 @@ struct mana_rdma_cqe {
 		};
 		struct {
 			u32 cqe_type		: 8;
-			u32 vendor_error	: 9;
-			u32 reserved1		: 15;
+			u32 reserved1		: 24;
 			u32 sge_offset		: 5;
 			u32 tx_wqe_offset	: 27;
 		} ud_send;
@@ -714,7 +822,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
 int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,
 				struct mana_ib_queue *queue);
 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
-			 struct mana_ib_queue *queue);
+			 struct mana_ib_queue *queue, bool is_cq);
 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
 
 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 8b7be1255c0d..15c5d6cefd62 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -340,7 +340,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
 		  ucmd.sq_buf_addr, ucmd.port);
 
-	err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq);
+	err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq,
+				   false);
 	if (err) {
 		ibdev_dbg(&mdev->ib_dev,
 			  "Failed to create queue for create qp-raw, err %d\n", err);
@@ -422,6 +423,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
 	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);
 }
 
+static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size)
+{
+	return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size));
+}
+
 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
 {
 	u32 queue_size;
@@ -509,8 +515,20 @@ static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
 	if (err)
 		goto err_remove_sq;
 
+	/* GSI QPs are additionally indexed by (port << 24 | MANA_GSI_QPN) so the
+	 * per-port GSI SQ drain can find each of them by iterating ports.
+	 */
+	if (qp->ibqp.qp_type == IB_QPT_GSI) {
+		err = mana_table_store_qp_qid(mdev, qp,
+					      (qp->port << 24) | MANA_GSI_QPN, false);
+		if (err)
+			goto err_remove_rq;
+	}
+
 	return 0;
 
+err_remove_rq:
+	mana_table_remove_qp_qid(mdev, rq->id, false);
 err_remove_sq:
 	mana_table_remove_qp_qid(mdev, sq->id, true);
 	mana_table_drain_qp_ref(qp);
@@ -529,6 +547,8 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp
 
 	mana_table_remove_qp_qid(mdev, sq->id, true);
 	mana_table_remove_qp_qid(mdev, rq->id, false);
+	if (qp->ibqp.qp_type == IB_QPT_GSI)
+		mana_table_remove_qp_qid(mdev, (qp->port << 24) | MANA_GSI_QPN, false);
 	mana_table_drain_qp_ref(qp);
 }
 
@@ -549,27 +569,42 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
 	doorbell = mana_ucontext->doorbell;
-	flags = MANA_RC_FLAG_NO_FMR;
-	err = ib_copy_validate_udata_in(udata, ucmd, queue_size);
+	flags = MANA_RC_FLAG_NO_MMQ;
+	err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size,
+					   MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE);
 	if (err)
 		return err;
 
 	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-		/* skip FMR for user-level RC QPs */
-		if (i == MANA_RC_SEND_QUEUE_FMR) {
-			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
-			qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+		if (i == MANA_RC_SEND_QUEUE_MMQ) {
+			if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) {
+				flags &= ~MANA_RC_FLAG_NO_MMQ;
+				err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size,
+							   &qp->rc_qp.queues[i], false);
+				if (err)
+					goto destroy_queues;
+			} else {
+				qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
+				qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+			}
 			continue;
 		}
 		err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
-					   &qp->rc_qp.queues[i]);
-		if (err) {
-			ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
+					   &qp->rc_qp.queues[i], false);
+		if (err)
 			goto destroy_queues;
-		}
 		j++;
 	}
 
+	if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) {
+		u32 wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge,
+						      INLINE_OOB_EXTRA_LARGE_SIZE);
+		flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
+		if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)
+			flags |= MANA_RC_FLAG_MSN_IN_WQE;
+		qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
+	}
+
 	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create rc qp  %d\n", err);
@@ -580,8 +615,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	if (udata) {
 		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-			if (i == MANA_RC_SEND_QUEUE_FMR)
+			if (i == MANA_RC_SEND_QUEUE_MMQ) {
+				resp.mmq_id = qp->rc_qp.queues[i].id;
 				continue;
+			}
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
 			j++;
 		}
@@ -627,7 +664,7 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; ++i) {
 		err = mana_ib_create_queue(mdev, ucmd.queue_buf[i], ucmd.queue_size[i],
-					   &qp->uc_qp.queues[i]);
+					   &qp->uc_qp.queues[i], false);
 		if (err)
 			goto destroy_queues;
 	}
@@ -658,33 +695,23 @@ destroy_queues:
 	return err;
 }
 
-static void mana_add_qp_to_cqs(struct mana_ib_qp *qp)
+static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset)
 {
 	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
 	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
 	unsigned long flags;
 
 	spin_lock_irqsave(&send_cq->cq_lock, flags);
-	list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
+	list_del_init(&qp->send_err_node);
+	/* Keep shadow reset serialized with hardware polling and SW flushing. */
+	if (reset)
+		reset_shadow_queue(&qp->shadow_sq);
 	spin_unlock_irqrestore(&send_cq->cq_lock, flags);
 
 	spin_lock_irqsave(&recv_cq->cq_lock, flags);
-	list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
-	spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
-}
-
-static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp)
-{
-	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
-	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
-	unsigned long flags;
-
-	spin_lock_irqsave(&send_cq->cq_lock, flags);
-	list_del(&qp->cq_send_list);
-	spin_unlock_irqrestore(&send_cq->cq_lock, flags);
-
-	spin_lock_irqsave(&recv_cq->cq_lock, flags);
-	list_del(&qp->cq_recv_list);
+	list_del_init(&qp->recv_err_node);
+	if (reset)
+		reset_shadow_queue(&qp->shadow_rq);
 	spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
 }
 
@@ -712,13 +739,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	doorbell = mdev->gdma_dev->doorbell;
 
 	err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr,
-				  sizeof(struct ud_rq_shadow_wqe));
+				  sizeof(struct shadow_wqe_header));
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err);
 		goto destroy_queues;
 	}
 	err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr,
-				  sizeof(struct ud_sq_shadow_wqe));
+				  sizeof(struct shadow_wqe_header));
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err);
 		goto destroy_shadow_queues;
@@ -729,7 +756,8 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 		ibdev_err(&mdev->ib_dev, "Failed to create ud qp  %d\n", err);
 		goto destroy_shadow_queues;
 	}
-	qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
+	qp->ibqp.qp_num = (qp->ibqp.qp_type == IB_QPT_GSI) ?
+			  MANA_GSI_QPN : qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
 	qp->port = attr->port_num;
 
 	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
@@ -739,8 +767,6 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	if (err)
 		goto destroy_qp;
 
-	mana_add_qp_to_cqs(qp);
-
 	return 0;
 
 destroy_qp:
@@ -757,6 +783,13 @@ destroy_queues:
 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		      struct ib_udata *udata)
 {
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	qp->sq_sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR;
+	mutex_init(&qp->modify_lock);
+	INIT_LIST_HEAD(&qp->send_err_node);
+	INIT_LIST_HEAD(&qp->recv_err_node);
+
 	switch (attr->qp_type) {
 	case IB_QPT_RAW_PACKET:
 		/* When rwq_ind_tbl is used, it's for creating WQs for RSS */
@@ -854,19 +887,56 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	return 0;
 }
 
+static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+				    int attr_mask, struct ib_udata *udata)
+{
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	if (udata)
+		return;
+
+	if (attr_mask & IB_QP_STATE) {
+		switch (attr->qp_state) {
+		case IB_QPS_RESET:
+			mana_remove_qp_from_cqs(qp, true);
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (attr_mask & IB_QP_SQ_PSN)
+		qp->sq_psn = attr->sq_psn;
+}
+
 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		      int attr_mask, struct ib_udata *udata)
 {
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+	int ret;
+
+	mutex_lock(&qp->modify_lock);
+
 	switch (ibqp->qp_type) {
 	case IB_QPT_RC:
 	case IB_QPT_UC:
 	case IB_QPT_UD:
 	case IB_QPT_GSI:
-		return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
+		ret = mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
+		if (ret)
+			goto out_unlock;
+		break;
 	default:
-		ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type);
-		return -EOPNOTSUPP;
+		ret = -EOPNOTSUPP;
+		goto out_unlock;
 	}
+
+	mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata);
+
+out_unlock:
+	mutex_unlock(&qp->modify_lock);
+
+	return ret;
 }
 
 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
@@ -1003,8 +1073,8 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
 	if (err)
 		return err;
 
-	mana_remove_qp_from_cqs(qp);
 	mana_table_remove_qp(mdev, qp);
+	mana_remove_qp_from_cqs(qp, false);
 
 	destroy_shadow_queue(&qp->shadow_rq);
 	destroy_shadow_queue(&qp->shadow_sq);
diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h
index a4b3818f9c39..b5d6bb28bd45 100644
--- a/drivers/infiniband/hw/mana/shadow_queue.h
+++ b/drivers/infiniband/hw/mana/shadow_queue.h
@@ -6,21 +6,19 @@
 #ifndef _MANA_SHADOW_QUEUE_H_
 #define _MANA_SHADOW_QUEUE_H_
 
-struct shadow_wqe_header {
-	u16 opcode;
-	u16 error_code;
-	u32 posted_wqe_size;
-	u64 wr_id;
-};
+#include <linux/build_bug.h>
 
-struct ud_rq_shadow_wqe {
-	struct shadow_wqe_header header;
-	u32 byte_len;
-	u32 src_qpn;
-};
+#define MANA_WQ_FENCE_WC		BIT(0)
+#define MANA_WQ_NO_SIGNAL_WC		BIT(1)
+#define MANA_WQE_OFFSET_MASK	GENMASK(23, 0)
 
-struct ud_sq_shadow_wqe {
-	struct shadow_wqe_header header;
+struct shadow_wqe_header {
+	u64 wr_id;
+	u64 wqe_offset_or_psn : 24;
+	u64 wqe_size_in_bu : 8;
+	u64 fsn : 24;
+	u64 send_opcode : 4;
+	u64 flags : 2;
 };
 
 struct shadow_queue {
@@ -28,8 +26,6 @@ struct shadow_queue {
 	u64 prod_idx;
 	/* Unmasked consumer index, Incremented on cq polling */
 	u64 cons_idx;
-	/* Unmasked index of next-to-complete (from HW) shadow WQE */
-	u64 next_to_complete_idx;
 	/* queue size in wqes */
 	u32 length;
 	/* distance between elements in bytes */
@@ -50,6 +46,12 @@ static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t lengt
 	return 0;
 }
 
+static inline void reset_shadow_queue(struct shadow_queue *queue)
+{
+	queue->prod_idx = 0;
+	queue->cons_idx = 0;
+}
+
 static inline void destroy_shadow_queue(struct shadow_queue *queue)
 {
 	kvfree(queue->buffer);
@@ -57,12 +59,14 @@ static inline void destroy_shadow_queue(struct shadow_queue *queue)
 
 static inline bool shadow_queue_full(struct shadow_queue *queue)
 {
-	return (queue->prod_idx - queue->cons_idx) >= queue->length;
+	/* Do not reuse an entry until the poller has finished reading it. */
+	return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length;
 }
 
 static inline bool shadow_queue_empty(struct shadow_queue *queue)
 {
-	return queue->prod_idx == queue->cons_idx;
+	/* Pair with posting's release of the initialized shadow WQE. */
+	return smp_load_acquire(&queue->prod_idx) == queue->cons_idx;
 }
 
 static inline void *
@@ -82,34 +86,23 @@ shadow_queue_producer_entry(struct shadow_queue *queue)
 static inline void *
 shadow_queue_get_next_to_consume(const struct shadow_queue *queue)
 {
-	if (queue->cons_idx == queue->next_to_complete_idx)
+	/* The producer publishes the WQE before advancing prod_idx. */
+	if (queue->cons_idx == smp_load_acquire(&queue->prod_idx))
 		return NULL;
 
 	return shadow_queue_get_element(queue, queue->cons_idx);
 }
 
-static inline void *
-shadow_queue_get_next_to_complete(struct shadow_queue *queue)
-{
-	if (queue->next_to_complete_idx == queue->prod_idx)
-		return NULL;
-
-	return shadow_queue_get_element(queue, queue->next_to_complete_idx);
-}
-
 static inline void shadow_queue_advance_producer(struct shadow_queue *queue)
 {
-	queue->prod_idx++;
+	/* Publish all WQE fields to CQ polling on another CPU. */
+	smp_store_release(&queue->prod_idx, queue->prod_idx + 1);
 }
 
 static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)
 {
-	queue->cons_idx++;
-}
-
-static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue)
-{
-	queue->next_to_complete_idx++;
+	/* Finish WC generation and queue-tail updates before allowing reuse. */
+	smp_store_release(&queue->cons_idx, queue->cons_idx + 1);
 }
 
 #endif
diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c
index 6b066d605dcb..4042f26632c0 100644
--- a/drivers/infiniband/hw/mana/wq.c
+++ b/drivers/infiniband/hw/mana/wq.c
@@ -25,7 +25,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
 
 	ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
 
-	err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue);
+	err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue,
+				   false);
 	if (err) {
 		ibdev_dbg(&mdev->ib_dev,
 			  "Failed to create queue for create wq, %d\n", err);
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 36a1d506f08f..250d673f58f2 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -7,29 +7,30 @@
 
 #define MAX_WR_SGL_NUM (2)
 
-static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
+static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
 {
-	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
-	struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
+	struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0};
+	struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
 	struct gdma_posted_wqe_info wqe_info = {0};
-	struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM];
+	struct gdma_queue *queue = ib_rq->kmem;
 	struct gdma_wqe_request wqe_req = {0};
-	struct ud_rq_shadow_wqe *shadow_wqe;
-	int err, i;
+	struct shadow_wqe_header *shadow_wqe;
+	int err;
 
 	if (shadow_queue_full(&qp->shadow_rq))
 		return -EINVAL;
 
-	if (wr->num_sge > MAX_WR_SGL_NUM)
+	if (wr->num_sge > MAX_RX_WQE_SGL_ENTRIES)
 		return -EINVAL;
 
-	for (i = 0; i < wr->num_sge; ++i) {
-		gdma_sgl[i].address = wr->sg_list[i].addr;
-		gdma_sgl[i].mem_key = wr->sg_list[i].lkey;
-		gdma_sgl[i].size = wr->sg_list[i].length;
-	}
 	wqe_req.num_sge = wr->num_sge;
-	wqe_req.sgl = gdma_sgl;
+	wqe_req.ib_sgl = wr->sg_list;
+	wqe_req.flags = GDMA_WR_IB_SGL;
+
+	if (wr->num_sge == 0) {
+		wqe_req.ib_sgl = &mana_ib_dummy_sge;
+		wqe_req.num_sge = 1;
+	}
 
 	err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info);
 	if (err)
@@ -37,42 +38,55 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *
 
 	shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq);
 	memset(shadow_wqe, 0, sizeof(*shadow_wqe));
-	shadow_wqe->header.opcode = IB_WC_RECV;
-	shadow_wqe->header.wr_id = wr->wr_id;
-	shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wr_id = wr->wr_id;
+	shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset;
 	shadow_queue_advance_producer(&qp->shadow_rq);
 
-	mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
 	return 0;
 }
 
-int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
-		      const struct ib_recv_wr **bad_wr)
+static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr,
+				const struct ib_recv_wr **bad_wr)
 {
-	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+	struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
+	struct gdma_queue *rq = ib_rq->kmem;
+	bool ring_rq = false;
 	int err = 0;
 
 	for (; wr; wr = wr->next) {
-		switch (ibqp->qp_type) {
-		case IB_QPT_UD:
-		case IB_QPT_GSI:
-			err = mana_ib_post_recv_ud(qp, wr);
-			if (unlikely(err)) {
-				*bad_wr = wr;
-				return err;
-			}
+		err = mana_ib_post_rq(qp, wr);
+		if (unlikely(err)) {
+			*bad_wr = wr;
 			break;
-		default:
-			ibdev_dbg(ibqp->device, "Posting recv wr on qp type %u is not supported\n",
-				  ibqp->qp_type);
-			return -EINVAL;
 		}
+		ring_rq = true;
 	}
 
+	if (ring_rq)
+		mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), rq);
+
 	return err;
 }
 
-static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
+int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr)
+{
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	switch (ibqp->qp_type) {
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return mana_ib_post_recv_ud(qp, wr, bad_wr);
+	default:
+		/* Unsupported QP type */
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+}
+
+static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
 {
 	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
 	struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah);
@@ -82,14 +96,12 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 	struct gdma_posted_wqe_info wqe_info = {0};
 	struct gdma_wqe_request wqe_req = {0};
 	struct rdma_send_oob send_oob = {0};
-	struct ud_sq_shadow_wqe *shadow_wqe;
+	struct shadow_wqe_header *shadow_wqe;
+	bool signaled = qp->sq_sig_all || (wr->wr.send_flags & IB_SEND_SIGNALED);
 	int err, i;
 
-	if (!ndev) {
-		ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n",
-			  qp->port, qp->ibqp.qp_num);
+	if (!ndev)
 		return -EINVAL;
-	}
 
 	if (wr->wr.opcode != IB_WR_SEND)
 		return -EINVAL;
@@ -111,16 +123,16 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 
 	wqe_req.num_sge = wr->wr.num_sge + 1;
 	wqe_req.sgl = gdma_sgl;
-	wqe_req.inline_oob_size = sizeof(struct rdma_send_oob);
+	wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE;
 	wqe_req.inline_oob_data = &send_oob;
 	wqe_req.flags = GDMA_WR_OOB_IN_SGL;
 	wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu));
 
 	send_oob.wqe_type = WQE_TYPE_UD_SEND;
 	send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE);
-	send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED);
+	send_oob.signaled = signaled;
 	send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
-	send_oob.psn = qp->ud_qp.sq_psn;
+	send_oob.psn = qp->sq_psn;
 	send_oob.ssn_or_rqpn = wr->remote_qpn;
 	send_oob.ud_send.remote_qkey =
 		qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey;
@@ -129,40 +141,64 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 	if (err)
 		return err;
 
-	qp->ud_qp.sq_psn++;
+	qp->sq_psn++;
 	shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq);
 	memset(shadow_wqe, 0, sizeof(*shadow_wqe));
-	shadow_wqe->header.opcode = IB_WC_SEND;
-	shadow_wqe->header.wr_id = wr->wr.wr_id;
-	shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wr_id = wr->wr.wr_id;
+	shadow_wqe->flags = signaled ? 0 : MANA_WQ_NO_SIGNAL_WC;
+	shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset;
 	shadow_queue_advance_producer(&qp->shadow_sq);
 
-	mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
 	return 0;
 }
 
-int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
-		      const struct ib_send_wr **bad_wr)
+static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr,
+				const struct ib_send_wr **bad_wr)
 {
+	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+	struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
+	struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
+	unsigned long flags;
+	bool ring_sq = false;
+	bool flush_send;
 	int err = 0;
-	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
 
 	for (; wr; wr = wr->next) {
-		switch (ibqp->qp_type) {
-		case IB_QPT_UD:
-		case IB_QPT_GSI:
-			err = mana_ib_post_send_ud(qp, ud_wr(wr));
-			if (unlikely(err)) {
-				*bad_wr = wr;
-				return err;
-			}
+		err = mana_ib_post_send_ud_one(qp, ud_wr(wr));
+		if (unlikely(err)) {
+			*bad_wr = wr;
 			break;
-		default:
-			ibdev_dbg(ibqp->device, "Posting send wr on qp type %u is not supported\n",
-				  ibqp->qp_type);
-			return -EINVAL;
 		}
+		ring_sq = true;
+	}
+
+	if (ring_sq) {
+		mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
+
+		spin_lock_irqsave(&cq->cq_lock, flags);
+		flush_send = !list_empty(&qp->send_err_node);
+		spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+		if (flush_send && cq->ibcq.comp_handler)
+			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 	}
 
 	return err;
 }
+
+int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr)
+{
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	switch (ibqp->qp_type) {
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return mana_ib_post_send_ud(qp, wr, bad_wr);
+	default:
+		/* Unsupported QP type */
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+}
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 887912469742..81dc47e57352 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -185,9 +185,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
 			goto err_umem;
 		}
 	} else {
-		cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
-					  entries * cqe_size,
-					  IB_ACCESS_LOCAL_WRITE);
+		cq->umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
+						    ucmd.buf_addr,
+						    entries * cqe_size,
+						    IB_ACCESS_LOCAL_WRITE);
 		if (IS_ERR(cq->umem)) {
 			err = PTR_ERR(cq->umem);
 			goto err_cq;
@@ -354,9 +355,10 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
 	if (!cq->resize_buf)
 		return -ENOMEM;
 
-	cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
-					 entries * cqe_size,
-					 IB_ACCESS_LOCAL_WRITE);
+	cq->resize_umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
+						   ucmd.buf_addr,
+						   entries * cqe_size,
+						   IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(cq->resize_umem)) {
 		err = PTR_ERR(cq->resize_umem);
 		goto err_buf;
@@ -989,7 +991,8 @@ void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
 	 * Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
-	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
+	while (prod_index != cq->mcq.cons_index) {
+		--prod_index;
 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 		cqe += cqe_inc;
 
diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
index dc7c0daaaba5..bba178ffe1b2 100644
--- a/drivers/infiniband/hw/mlx4/mcg.c
+++ b/drivers/infiniband/hw/mlx4/mcg.c
@@ -433,7 +433,7 @@ static u16 cmp_rec(struct ib_sa_mcmember_data *src,
 }
 
 /* release group, return 1 if this was last release and group is destroyed
- * timout work is canceled sync */
+ * timeout work is canceled sync */
 static int release_group(struct mcast_group *group, int from_timeout_handler)
 {
 	struct mlx4_ib_demux_ctx *ctx = group->demux;
@@ -694,7 +694,7 @@ static void mlx4_ib_mcg_work_handler(struct work_struct *work)
 			cur_join_state = group->rec.scope_join_state & 0xf;
 
 			if (method == IB_MGMT_METHOD_GET_RESP) {
-				/* successfull join */
+				/* successful join */
 				if (!cur_join_state && resp_join_state)
 					--rc;
 			} else if (!resp_join_state)
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 49b4bf148a4a..072b77a2c8a6 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -1169,7 +1169,8 @@ void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq)
 	/* Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
-	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
+	while (prod_index != cq->mcq.cons_index) {
+		--prod_index;
 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 		cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
 		if (is_equal_rsn(cqe64, rsn)) {
@@ -1245,9 +1246,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 	if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
 		return -EINVAL;
 
-	umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
-			      (size_t)ucmd.cqe_size * entries,
-			      IB_ACCESS_LOCAL_WRITE);
+	umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, ucmd.buf_addr,
+					(size_t)ucmd.cqe_size * entries,
+					IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(umem)) {
 		err = PTR_ERR(umem);
 		return err;
diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index d57484245c38..93a56ee2c4ef 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -65,13 +65,9 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (err) {
-		dev_warn(&pdev->dev,
-			 "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err);
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err);
-			return err;
-		}
+		dev_err(&pdev->dev,
+			"Can't set 64-bit PCI DMA mask, err=%d\n", err);
+		return err;
 	}
 
 	dma_set_max_seg_size(&pdev->dev, SZ_2G);
diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c
index b155baee0017..e63fdb4bc3e3 100644
--- a/drivers/infiniband/hw/mlx5/fs.c
+++ b/drivers/infiniband/hw/mlx5/fs.c
@@ -17,6 +17,7 @@
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/fs_helpers.h>
 #include <linux/mlx5/eswitch.h>
+#include <linux/overflow.h>
 #include <net/inet_ecn.h>
 #include "mlx5_ib.h"
 #include "counters.h"
@@ -1647,36 +1648,32 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de
 							  struct ib_flow_attr *flow_attr,
 							  struct mlx5_flow_destination *dst)
 {
+	DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_wc, flows, 1);
+	DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_uc, flows, 1);
+
 	struct mlx5_ib_flow_handler *handler_ucast = NULL;
 	struct mlx5_ib_flow_handler *handler = NULL;
 
-	static struct {
-		struct ib_flow_spec_eth eth_flow;
-		struct ib_flow_attr	flow_attr;
-	} leftovers_wc = { .flow_attr = { .num_of_specs = 1,
-					  .size = sizeof(leftovers_wc) },
-			   .eth_flow = {
-				   .type = IB_FLOW_SPEC_ETH,
-				   .size = sizeof(struct ib_flow_spec_eth),
-				   .mask = { .dst_mac = { 0x1 } },
-				   .val = { .dst_mac = { 0x1 } } } };
-
-	static struct {
-		struct ib_flow_spec_eth eth_flow;
-		struct ib_flow_attr	flow_attr;
-	} leftovers_uc = { .flow_attr = { .num_of_specs = 1,
-					  .size = sizeof(leftovers_uc) },
-			   .eth_flow = {
-				   .type = IB_FLOW_SPEC_ETH,
-				   .size = sizeof(struct ib_flow_spec_eth),
-				   .mask = { .dst_mac = { 0x1 } },
-				   .val = { .dst_mac = {} } } };
-
-	handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
+	flow_attr_wc->num_of_specs = 1;
+	flow_attr_wc->size = sizeof(struct ib_flow_attr) +
+			      sizeof(struct ib_flow_spec_eth);
+	flow_attr_wc->flows[0].eth.type = IB_FLOW_SPEC_ETH;
+	flow_attr_wc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth);
+	flow_attr_wc->flows[0].eth.mask.dst_mac[0] = 0x1;
+	flow_attr_wc->flows[0].eth.val.dst_mac[0] = 0x1;
+
+	flow_attr_uc->num_of_specs = 1;
+	flow_attr_uc->size = sizeof(struct ib_flow_attr) +
+			      sizeof(struct ib_flow_spec_eth);
+	flow_attr_uc->flows[0].eth.type = IB_FLOW_SPEC_ETH;
+	flow_attr_uc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth);
+	flow_attr_uc->flows[0].eth.mask.dst_mac[0] = 0x1;
+
+	handler = create_flow_rule(dev, ft_prio, flow_attr_wc, dst);
 	if (!IS_ERR(handler) &&
 	    flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
 		handler_ucast = create_flow_rule(dev, ft_prio,
-						 &leftovers_uc.flow_attr, dst);
+						 flow_attr_uc, dst);
 		if (IS_ERR(handler_ucast)) {
 			mlx5_del_flow_rules(handler->rule);
 			ft_prio->refcount--;
@@ -3056,8 +3053,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
 	return 0;
 
 destroy_res:
-	--ft_prio->anchor.rule_goto_table_ref;
-	mlx5_steering_anchor_destroy_res(ft_prio);
+	if (!--ft_prio->anchor.rule_goto_table_ref)
+		mlx5_steering_anchor_destroy_res(ft_prio);
 put_flow_table:
 	put_flow_table(dev, ft_prio, true);
 free_obj:
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index a457647edacd..4a0b4dcf2830 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3370,7 +3370,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_ib_resources *devr = &dev->devr;
 	struct ib_srq_init_attr attr;
-	struct ib_srq *s0, *s1;
+	struct ib_srq *s0 = NULL, *s1;
 	int ret = 0;
 
 	/*
@@ -3388,19 +3388,27 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
 	if (ret)
 		goto unlock;
 
-	memset(&attr, 0, sizeof(attr));
-	attr.attr.max_sge = 1;
-	attr.attr.max_wr = 1;
-	attr.srq_type = IB_SRQT_XRC;
-	attr.ext.cq = devr->c0;
-
-	s0 = ib_create_srq(devr->p0, &attr);
-	if (IS_ERR(s0)) {
-		ret = PTR_ERR(s0);
-		mlx5_ib_err(dev,
-			    "Couldn't create SRQ 0 for res init, err=%pe\n",
-			    s0);
-		goto unlock;
+	/*
+	 * s0 is an XRC-type placeholder SRQ used as the default XRQN for
+	 * XRC QPs.  Skip it when XRC is absent; all devr->s0 accesses in
+	 * qp.c are inside XRC QP paths that the verbs layer blocks before
+	 * reaching this driver when xrc=0, so NULL is safe.
+	 */
+	if (MLX5_CAP_GEN(dev->mdev, xrc)) {
+		memset(&attr, 0, sizeof(attr));
+		attr.attr.max_sge = 1;
+		attr.attr.max_wr = 1;
+		attr.srq_type = IB_SRQT_XRC;
+		attr.ext.cq = devr->c0;
+
+		s0 = ib_create_srq(devr->p0, &attr);
+		if (IS_ERR(s0)) {
+			ret = PTR_ERR(s0);
+			mlx5_ib_err(dev,
+				    "Couldn't create SRQ 0 for res init, err=%pe\n",
+				    s0);
+			goto unlock;
+		}
 	}
 
 	memset(&attr, 0, sizeof(attr));
@@ -3414,7 +3422,8 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
 		mlx5_ib_err(dev,
 			    "Couldn't create SRQ 1 for res init, err=%pe\n",
 			    s1);
-		ib_destroy_srq(s0);
+		if (s0)
+			ib_destroy_srq(s0);
 		goto unlock;
 	}
 
@@ -3431,8 +3440,11 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
 	struct mlx5_ib_resources *devr = &dev->devr;
 	int ret;
 
+	mutex_init(&devr->cq_lock);
+	mutex_init(&devr->srq_lock);
+
 	if (!MLX5_CAP_GEN(dev->mdev, xrc))
-		return -EOPNOTSUPP;
+		return 0;
 
 	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
 	if (ret)
@@ -3444,9 +3456,6 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
 		return ret;
 	}
 
-	mutex_init(&devr->cq_lock);
-	mutex_init(&devr->srq_lock);
-
 	return 0;
 }
 
@@ -3457,10 +3466,13 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
 	/* After s0/s1 init, they are not unset during the device lifetime. */
 	if (devr->s1) {
 		ib_destroy_srq(devr->s1);
-		ib_destroy_srq(devr->s0);
+		if (devr->s0)
+			ib_destroy_srq(devr->s0);
+	}
+	if (MLX5_CAP_GEN(dev->mdev, xrc)) {
+		mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
+		mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
 	}
-	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
-	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
 	/* After p0/c0 init, they are not unset during the device lifetime. */
 	if (devr->c0) {
 		ib_destroy_cq(devr->c0);
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index b8618610737a..8dcbb0e5ccd7 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -300,7 +300,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni,
 	/*
 	 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
 	 * while we are doing the invalidation, no page fault will attempt to
-	 * overwrite the same MTTs.  Concurent invalidations might race us,
+	 * overwrite the same MTTs.  Concurrent invalidations might race us,
 	 * but they will write 0s as well, so no difference in the end result.
 	 */
 	for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 0d4f8b109ad2..627dd17804c9 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -1624,6 +1624,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		if (err)
 			return err;
 
+		sq->base.container_mibqp = qp;
+		sq->base.mqp.event = mlx5_ib_qp_event;
 		err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd,
 					      to_mcq(init_attr->send_cq));
 		if (err)
@@ -1635,9 +1637,6 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			resp->sqn = sq->base.mqp.qpn;
 			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
 		}
-
-		sq->base.container_mibqp = qp;
-		sq->base.mqp.event = mlx5_ib_qp_event;
 	}
 
 	if (qp->rq.wqe_cnt) {
@@ -1647,6 +1646,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		}
 
 		rq->base.container_mibqp = qp;
+		rq->base.mqp.event = mlx5_ib_qp_event;
 
 		if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
 			rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
@@ -2092,13 +2092,13 @@ static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	}
 
 	base = &qp->trans_qp.base;
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 	kvfree(in);
 	if (err)
 		return err;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
 	if (MLX5_CAP_GEN(mdev, ece_support))
 		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
@@ -2236,14 +2236,14 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
 	}
 
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 
 	kvfree(in);
 	if (err)
 		goto err_create;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
 	if (MLX5_CAP_GEN(mdev, ece_support))
 		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
@@ -2432,6 +2432,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
 	}
 
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
 	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
@@ -2446,8 +2448,6 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	if (err)
 		goto err_create;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
 	if (MLX5_CAP_GEN(mdev, ece_support))
 		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
@@ -2577,14 +2577,13 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	    MLX5_CAP_GEN(mdev, go_back_n))
 		MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
 
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 	kvfree(in);
 	if (err)
 		goto err_create;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
-
 	get_cqs(qp->type, attr->send_cq, attr->recv_cq,
 		&send_cq, &recv_cq);
 	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
@@ -3186,7 +3185,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 out:
 	if (err) {
-		mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
+		mlx5_ib_err(dev, "Create QP type %d failed, err = %d\n", qp->type, err);
 		return err;
 	}
 
@@ -5344,11 +5343,14 @@ static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
 			break;
 		default:
 			mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
-			return;
+			goto out;
 		}
 
 		rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
 	}
+
+out:
+	mlx5_core_res_put(&core_qp->common);
 }
 
 static int set_delay_drop(struct mlx5_ib_dev *dev)
@@ -5616,6 +5618,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 		rwq = kzalloc_obj(*rwq);
 		if (!rwq)
 			return ERR_PTR(-ENOMEM);
+		rwq->core_qp.event = mlx5_ib_wq_event;
 		err = prepare_user_rq(pd, init_attr, udata, rwq);
 		if (err)
 			goto err;
@@ -5639,7 +5642,6 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 			goto err_copy;
 	}
 
-	rwq->core_qp.event = mlx5_ib_wq_event;
 	rwq->ibwq.event_handler = init_attr->event_handler;
 	return &rwq->ibwq;
 
diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c
index a7a4f9420271..ce2ef8828c09 100644
--- a/drivers/infiniband/hw/mlx5/qpc.c
+++ b/drivers/infiniband/hw/mlx5/qpc.c
@@ -128,7 +128,7 @@ static int rsc_event_notifier(struct notifier_block *nb,
 	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 		rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
-		rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
+		rsn |= ((u32)eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
 		break;
 	default:
 		return NOTIFY_DONE;
@@ -146,6 +146,8 @@ static int rsc_event_notifier(struct notifier_block *nb,
 	case MLX5_RES_RQ:
 	case MLX5_RES_SQ:
 		qp = (struct mlx5_core_qp *)common;
+		if (WARN_ON_ONCE(!qp->event))
+			goto out;
 		qp->event(qp, event_type);
 		/* Need to put resource in event handler */
 		return NOTIFY_OK;
@@ -165,6 +167,10 @@ static int create_resource_common(struct mlx5_ib_dev *dev,
 	int err;
 
 	qp->common.res = rsc_type;
+	refcount_set(&qp->common.refcount, 1);
+	init_completion(&qp->common.free);
+	qp->pid = current->pid;
+
 	spin_lock_irq(&table->lock);
 	err = radix_tree_insert(&table->tree,
 				qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN),
@@ -173,10 +179,6 @@ static int create_resource_common(struct mlx5_ib_dev *dev,
 	if (err)
 		return err;
 
-	refcount_set(&qp->common.refcount, 1);
-	init_completion(&qp->common.free);
-	qp->pid = current->pid;
-
 	return 0;
 }
 
@@ -258,6 +260,14 @@ int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
 
 	qp->uid = MLX5_GET(create_qp_in, in, uid);
 	qp->qpn = MLX5_GET(create_qp_out, out, qpn);
+	/* Set ibqp.qp_num before create_resource_common() inserts the QP into
+	 * the radix tree and makes it visible to EQE processing.  The
+	 * assignment in create_qp() happens only after this function returns,
+	 * leaving a window where an arriving EQE would observe qp_num == 0.
+	 * create_qp() still overrides this with 0/1 for QP0/QP1, and sets it
+	 * independently for DCT (which does not go through this function).
+	 */
+	to_mibqp(qp)->ibqp.qp_num = qp->qpn;
 
 	err = create_resource_common(dev, qp, MLX5_RES_QP);
 	if (err)
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index a973c1b7515f..6a4a937d97ce 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -284,6 +284,7 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,
 
 	in.pd = to_mpd(ib_srq->pd)->pdn;
 	in.db_record = srq->db.dma;
+	srq->msrq.event = mlx5_ib_srq_event;
 	err = mlx5_cmd_create_srq(dev, &srq->msrq, &in);
 	kvfree(in.pas);
 	if (err) {
@@ -292,8 +293,6 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,
 	}
 
 	mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn);
-
-	srq->msrq.event = mlx5_ib_srq_event;
 	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
 
 	if (udata) {
diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c
index fca9e1d9d5e9..bb2c60380fe2 100644
--- a/drivers/infiniband/hw/mlx5/wr.c
+++ b/drivers/infiniband/hw/mlx5/wr.c
@@ -927,6 +927,7 @@ static noinline_for_stack int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
 	qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
 
 out:
+	mr->pi_mr = pi_mr;
 	return err;
 }
 
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 26c3408dcaca..39d4ff15849e 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -300,7 +300,8 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn,
 	 * Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
-	while ((int) --prod_index - (int) cq->cons_index >= 0) {
+	while (prod_index != cq->cons_index) {
+		--prod_index;
 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 		if (cqe->my_qpn == cpu_to_be32(qpn)) {
 			if (srq && is_recv_cqe(cqe))
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index cfe3d19b73b3..97c2e18e67ad 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -1392,9 +1392,9 @@ int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	qp = get_ocrdma_qp(ibqp);
 	dev = get_ocrdma_dev(ibqp->device);
 
-	/* syncronize with multiple context trying to change, retrive qps */
+	/* synchronize with multiple context trying to change, retrieve qps */
 	mutex_lock(&dev->dev_lock);
-	/* syncronize with wqe, rqe posting and cqe processing contexts */
+	/* synchronize with wqe, rqe posting and cqe processing contexts */
 	spin_lock_irqsave(&qp->q_lock, flags);
 	old_qps = get_ibqp_state(qp->state);
 	if (attr_mask & IB_QP_STATE)
@@ -1599,7 +1599,7 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
 	 */
 
 	cur_getp = cq->getp;
-	/* find upto when do we reap the cq. */
+	/* find up to when do we reap the cq. */
 	stop_getp = cur_getp;
 	do {
 		if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 012a0ab98d6b..acab3cb98caa 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -775,16 +775,23 @@ static inline int qedr_init_user_queue(struct ib_udata *udata,
 				       struct qedr_userq *q, u64 buf_addr,
 				       size_t buf_len, bool requires_db_rec,
 				       int access,
-				       int alloc_and_init)
+				       int alloc_and_init, bool is_cq)
 {
 	u32 fw_pages;
 	int rc;
 
 	q->buf_addr = buf_addr;
 	q->buf_len = buf_len;
-	q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access);
+	if (is_cq)
+		q->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
+						   q->buf_addr, q->buf_len,
+						   access);
+	else
+		q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len,
+					 access);
 	if (IS_ERR(q->umem)) {
-		DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n",
+		DP_ERR(dev, "create user queue: failed %s, got %ld\n",
+		       is_cq ? "ib_umem_get_cq_buf_or_va" : "ib_umem_get_va",
 		       PTR_ERR(q->umem));
 		return PTR_ERR(q->umem);
 	}
@@ -946,7 +953,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 					  ureq.len, true, IB_ACCESS_LOCAL_WRITE,
-					  1);
+					  1, true);
 		if (rc)
 			goto err0;
 
@@ -1075,7 +1082,7 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	/* We don't want the IRQ handler to handle a non-existing CQ so we
 	 * wait until all CNQ interrupts, if any, are received. This will always
 	 * happen and will always happen very fast. If not, then a serious error
-	 * has occured. That is why we can use a long delay.
+	 * has occurred. That is why we can use a long delay.
 	 * We spin for a short time so we don’t lose time on context switching
 	 * in case all the completions are handled in that span. Otherwise
 	 * we sleep for a while and check again. Since the CNQ may be
@@ -1438,7 +1445,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata,
 	int rc;
 
 	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
-				  ureq->srq_len, false, access, 1);
+				  ureq->srq_len, false, access, 1, false);
 	if (rc)
 		return rc;
 
@@ -1831,7 +1838,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
 	if (qedr_qp_has_sq(qp)) {
 		/* SQ - read access only (0) */
 		rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
-					  ureq.sq_len, true, 0, alloc_and_init);
+					  ureq.sq_len, true, 0, alloc_and_init,
+					  false);
 		if (rc)
 			return rc;
 	}
@@ -1839,7 +1847,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
 	if (qedr_qp_has_rq(qp)) {
 		/* RQ - read access only (0) */
 		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
-					  ureq.rq_len, true, 0, alloc_and_init);
+					  ureq.rq_len, true, 0, alloc_and_init,
+					  false);
 		if (rc) {
 			ib_umem_release(qp->usq.umem);
 			qp->usq.umem = NULL;
diff --git a/drivers/infiniband/hw/usnic/usnic_abi.h b/drivers/infiniband/hw/usnic/usnic_abi.h
index 86a82a4da0aa..a684bc54f3d4 100644
--- a/drivers/infiniband/hw/usnic/usnic_abi.h
+++ b/drivers/infiniband/hw/usnic/usnic_abi.h
@@ -72,7 +72,7 @@ struct usnic_ib_create_qp_resp {
 	u64				bar_bus_addr;
 	u32				bar_len;
 /*
- * WQ, RQ, CQ are explicitly specified bc exposing a generic resources inteface
+ * WQ, RQ, CQ are explicitly specified bc exposing a generic resources interface
  * expands the scope of ABI to many files.
  */
 	u32				wq_cnt;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index 1a1647d0e345..ebba3027cf48 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -181,7 +181,7 @@ find_free_vf_and_create_qp_grp(struct ib_qp *qp,
 	}
 
 	if (usnic_ib_share_vf) {
-		/* Try to find resouces on a used vf which is in pd */
+		/* Try to find resources on a used vf which is in pd */
 		dev_list = usnic_uiom_get_dev_list(pd->umem_pd);
 		if (IS_ERR(dev_list))
 			return PTR_ERR(dev_list);
diff --git a/drivers/infiniband/hw/usnic/usnic_transport.h b/drivers/infiniband/hw/usnic/usnic_transport.h
index 9a7a2d9755c0..c963c92218d3 100644
--- a/drivers/infiniband/hw/usnic/usnic_transport.h
+++ b/drivers/infiniband/hw/usnic/usnic_transport.h
@@ -50,7 +50,7 @@ int usnic_transport_sock_to_str(char *buf, int buf_sz,
 u16 usnic_transport_rsrv_port(enum usnic_transport_type type, u16 port_num);
 void usnic_transport_unrsrv_port(enum usnic_transport_type type, u16 port_num);
 /*
- * Do a fget on the socket refered to by sock_fd and returns the socket.
+ * Do a fget on the socket referred to by sock_fd and returns the socket.
  * Socket will not be destroyed before usnic_transport_put_socket has
  * been called.
  */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
index e939cd5ce40b..53cc49f2b8c9 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -270,7 +270,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 				/* set qp->sq.wqe_cnt, shift, buf_size.. */
 				qp->rumem = ib_umem_get_va(ibqp->device,
 							   ucmd.rbuf_addr,
-							   ucmd.rbuf_size, 0);
+							   ucmd.rbuf_size,
+							   IB_ACCESS_LOCAL_WRITE);
 				if (IS_ERR(qp->rumem)) {
 					ret = PTR_ERR(qp->rumem);
 					goto err_qp;
@@ -282,7 +283,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 			}
 
 			qp->sumem = ib_umem_get_va(ibqp->device, ucmd.sbuf_addr,
-						   ucmd.sbuf_size, 0);
+						   ucmd.sbuf_size,
+						   IB_ACCESS_LOCAL_WRITE);
 			if (IS_ERR(qp->sumem)) {
 				if (!is_srq)
 					ib_umem_release(qp->rumem);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
index 345ec486a223..3252c2eb405a 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
@@ -146,7 +146,8 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
 	if (ret)
 		goto err_srq;
 
-	srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0);
+	srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size,
+				   IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(srq->umem)) {
 		ret = PTR_ERR(srq->umem);
 		goto err_srq;
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index c40cce69e945..d8ee485c49a6 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -814,7 +814,7 @@ bail:
  *
  * This function is called from both rvt_create_qp() and
  * rvt_reset_qp().   The difference is that the reset
- * patch the necessary locks to protect against concurent
+ * patch the necessary locks to protect against concurrent
  * access.
  */
 static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index d022aa56c5bf..8122b46ed4ab 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -286,7 +286,7 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 }
 
 /**
- * rvt_destroy_srq - destory an srq
+ * rvt_destroy_srq - destroy an srq
  * @ibsrq: srq object to destroy
  * @udata: user data for libibverbs.so
  */
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 64d636bf80fd..7c3cc48e845c 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -92,6 +92,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem);
 /* rxe_net.c */
 struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 				int paylen, struct rxe_pkt_info *pkt);
+void rxe_put_skb(struct sk_buff *skb);
 int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt,
 		struct sk_buff *skb);
 int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 71d9ea477289..615da4bb9a38 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -796,6 +796,12 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 		return -EINVAL;
 	}
 
+	/* an MR with no umem is never an ODP MR */
+	if (unlikely(access & IB_ACCESS_ON_DEMAND)) {
+		rxe_dbg_mr(mr, "access = 0x%x requests ODP\n", access);
+		return -EINVAL;
+	}
+
 	mr->access = access;
 	mr->lkey = key;
 	mr->rkey = key;
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 53daaf4c1eb2..9f1c7f21d88d 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -429,6 +429,29 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt,
 	return err;
 }
 
+/*
+ * skb->dev may be rewritten on the TX path (VLAN/bond/tunnel, etc.).
+ * The netdev we held in rxe_init_packet() is kept in destructor_arg so
+ * that put always matches hold. skb->cb cannot be used: it is already
+ * occupied by struct rxe_pkt_info and rewritten by IP control blocks.
+ */
+static void rxe_skb_set_held_ndev(struct sk_buff *skb, struct net_device *ndev)
+{
+	dev_hold(ndev);
+	skb->dev = ndev;
+	skb_shinfo(skb)->destructor_arg = ndev;
+}
+
+static void rxe_skb_put_held_ndev(struct sk_buff *skb)
+{
+	struct net_device *ndev = skb_shinfo(skb)->destructor_arg;
+
+	if (ndev) {
+		skb_shinfo(skb)->destructor_arg = NULL;
+		dev_put(ndev);
+	}
+}
+
 static void rxe_skb_tx_dtor(struct sk_buff *skb)
 {
 	struct rxe_qp *qp = skb->sk->sk_user_data;
@@ -441,6 +464,18 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb)
 
 	rxe_put(qp);
 	sock_put(skb->sk);
+	rxe_skb_put_held_ndev(skb);
+}
+
+/*
+ * Free an skb that still holds a netdev reference from rxe_init_packet()
+ * and does not yet have rxe_skb_tx_dtor() installed. Once the TX
+ * destructor is set, callers must use kfree_skb() instead.
+ */
+void rxe_put_skb(struct sk_buff *skb)
+{
+	rxe_skb_put_held_ndev(skb);
+	kfree_skb(skb);
 }
 
 static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt)
@@ -529,7 +564,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	goto done;
 
 drop:
-	kfree_skb(skb);
+	rxe_put_skb(skb);
 	err = 0;
 done:
 	return err;
@@ -574,8 +609,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 
 	skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev));
 
-	/* FIXME: hold reference to this netdev until life of this skb. */
-	skb->dev	= ndev;
+	rxe_skb_set_held_ndev(skb, ndev);
 	rcu_read_unlock();
 
 	if (av->network_type == RXE_NETWORK_TYPE_IPV4)
@@ -638,9 +672,8 @@ static void rxe_sock_put(struct sock *sk,
 	if (refcount_read(&sk->sk_refcnt) > SK_REF_FOR_TUNNEL) {
 		__sock_put(sk);
 	} else {
+		set_sk(net, NULL);
 		rxe_release_udp_tunnel(sk);
-		sk = NULL;
-		set_sk(net, sk);
 	}
 }
 
@@ -710,6 +743,28 @@ void rxe_set_port_state(struct rxe_dev *rxe)
 	dev_put(ndev);
 }
 
+/*
+ * Move all QPs to the error state so pending send/recv work is drained and
+ * in-flight TX skbs (which hold a netdev reference) can be released. Called
+ * from the netdev notifier so unregister cannot stall on held skbs.
+ */
+static void rxe_flush_qps(struct rxe_dev *rxe)
+{
+	struct rxe_pool_elem *elem;
+	struct rxe_qp *qp;
+	unsigned long index;
+
+	rcu_read_lock();
+	xa_for_each(&rxe->qp_pool.xa, index, elem) {
+		if (!elem || !kref_get_unless_zero(&elem->ref_cnt))
+			continue;
+		qp = elem->obj;
+		rxe_qp_error(qp);
+		rxe_put(qp);
+	}
+	rcu_read_unlock();
+}
+
 static int rxe_notify(struct notifier_block *not_blk,
 		      unsigned long event,
 		      void *arg)
@@ -721,7 +776,12 @@ static int rxe_notify(struct notifier_block *not_blk,
 		return NOTIFY_OK;
 
 	switch (event) {
+	case NETDEV_GOING_DOWN:
+		/* Start draining TX queues before the netdev disappears. */
+		rxe_flush_qps(rxe);
+		break;
 	case NETDEV_UNREGISTER:
+		rxe_flush_qps(rxe);
 		ib_unregister_device_queued(&rxe->ib_dev);
 		rxe_net_del(&rxe->ib_dev);
 		break;
@@ -735,7 +795,6 @@ static int rxe_notify(struct notifier_block *not_blk,
 			rxe_counter_inc(rxe, RXE_CNT_LINK_DOWNED);
 		break;
 	case NETDEV_REBOOT:
-	case NETDEV_GOING_DOWN:
 	case NETDEV_CHANGEADDR:
 	case NETDEV_CHANGENAME:
 	case NETDEV_FEAT_CHANGE:
diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c
index ab21b620e94c..c375d3efd999 100644
--- a/drivers/infiniband/sw/rxe/rxe_odp.c
+++ b/drivers/infiniband/sw/rxe/rxe_odp.c
@@ -469,7 +469,7 @@ static int rxe_ib_prefetch_sg_list(struct ib_pd *ibpd,
 		struct rxe_mr *mr;
 		struct ib_umem_odp *umem_odp;
 
-		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE,
+		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND,
 			       sg_list[i].lkey, RXE_LOOKUP_LOCAL);
 
 		if (!mr) {
@@ -535,7 +535,7 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd,
 
 	for (i = 0; i < num_sge; ++i) {
 		/* Takes a reference, which will be released in the queued work */
-		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE,
+		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND,
 			       sg_list[i].lkey, RXE_LOOKUP_LOCAL);
 		if (!mr) {
 			mr = ERR_PTR(-EINVAL);
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 24f5c044363f..53f7c1e7fe55 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -811,7 +811,7 @@ int rxe_requester(struct rxe_qp *qp)
 			wqe->status = IB_WC_LOC_PROT_ERR;
 		else
 			wqe->status = IB_WC_LOC_QP_OP_ERR;
-		kfree_skb(skb);
+		rxe_put_skb(skb);
 		if (ah)
 			rxe_put(ah);
 		goto err;
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 02b16e2b49b8..ba3b2d23b36b 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -288,6 +288,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
 	}
 	size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge);
 	memcpy(&qp->resp.srq_wqe, wqe, size);
+	qp->resp.srq_wqe.wqe.dma.num_sge = num_sge;
 
 	qp->resp.wqe = &qp->resp.srq_wqe.wqe;
 	queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT);
@@ -328,6 +329,7 @@ static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp)
 	}
 	size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge);
 	memcpy(&qp->resp.srq_wqe, wqe, size);
+	qp->resp.srq_wqe.wqe.dma.num_sge = num_sge;
 
 	qp->resp.wqe = &qp->resp.srq_wqe.wqe;
 	return RESPST_CHK_LENGTH;
@@ -866,7 +868,7 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
 
 	err = rxe_prepare(&qp->pri_av, ack, skb);
 	if (err) {
-		kfree_skb(skb);
+		rxe_put_skb(skb);
 		return NULL;
 	}
 
@@ -994,7 +996,7 @@ static enum resp_states read_reply(struct rxe_qp *qp,
 	err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
 			  payload, RXE_FROM_MR_OBJ);
 	if (err) {
-		kfree_skb(skb);
+		rxe_put_skb(skb);
 		state = RESPST_ERR_RKEY_VIOLATION;
 		goto err_out;
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 3864284522eb..46d0810ea2a7 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1337,6 +1337,12 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags,
 		return ERR_PTR(-EOPNOTSUPP);
 	}
 
+	if ((flags & IB_MR_REREG_ACCESS) &&
+	    ((access ^ mr->access) & IB_ACCESS_ON_DEMAND)) {
+		rxe_err_mr(mr, "cannot change IB_ACCESS_ON_DEMAND\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
 	if (flags & IB_MR_REREG_PD) {
 		rxe_put(old_pd);
 		rxe_get(pd);
diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c
index 9735b75ac933..c08796e4262a 100644
--- a/drivers/infiniband/sw/siw/siw_main.c
+++ b/drivers/infiniband/sw/siw/siw_main.c
@@ -217,8 +217,11 @@ static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id)
 		/*
 		 * siw_qp_id2obj() increments object reference count
 		 */
+		struct ib_qp *base_qp = &qp->base_qp;
+
 		siw_qp_put(qp);
-		return &qp->base_qp;
+
+		return base_qp;
 	}
 	return NULL;
 }
diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c
index f7dd32c6e5ba..56bd009879c2 100644
--- a/drivers/infiniband/sw/siw/siw_qp_tx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_tx.c
@@ -336,7 +336,6 @@ static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset,
 		if (size + offset <= PAGE_SIZE)
 			msg.msg_flags &= ~MSG_MORE;
 
-		tcp_rate_check_app_limited(sk);
 		if (!sendpage_ok(page[i]))
 			msg.msg_flags &= ~MSG_SPLICE_PAGES;
 		bvec_set_page(&bvec, page[i], bytes, offset);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
index ba3b45152907..697cad0d063d 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
@@ -140,7 +140,7 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv,
 		goto out_early;
 	}
 
-	/* RTNL childs don't need proprietary sysfs entries */
+	/* RTNL children don't need proprietary sysfs entries */
 	if (type == IPOIB_LEGACY_CHILD) {
 		if (ipoib_cm_add_mode_attr(ndev))
 			goto sysfs_failed;
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 7df441685780..31e3a8ab6c48 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -498,7 +498,7 @@ out:
  * iscsi_iser_conn_start() - start iscsi-iser connection
  * @cls_conn: iscsi class connection
  *
- * Notes: Here iser intialize (or re-initialize) stop_completion as
+ * Notes: Here iser initialize (or re-initialize) stop_completion as
  *        from this point iscsi must call conn_stop in session/connection
  *        teardown so iser transport must wait for it.
  */
@@ -593,7 +593,7 @@ static inline unsigned int iser_dif_prot_caps(int prot_caps)
  * @qdepth:         session command queue depth
  * @initial_cmdsn:  initiator command sequnce number
  *
- * Allocates and adds a scsi host, expose DIF supprot if
+ * Allocates and adds a scsi host, expose DIF support if
  * exists, and sets up an iscsi session.
  */
 static struct iscsi_cls_session *
@@ -866,7 +866,7 @@ static int iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
  * @ep:    iscsi endpoint handle
  *
  * This routine is not blocked by iser and RDMA termination process
- * completion as we queue a deffered work for iser/RDMA destruction
+ * completion as we queue a deferred work for iser/RDMA destruction
  * and cleanup or actually call it immediately in case we didn't pass
  * iscsi conn bind/start stage, thus it is safe.
  */
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 55fe68e5b837..ab2a27d23478 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -347,7 +347,7 @@ void iser_release_work(struct work_struct *work)
 
 	/* Wait for conn_stop to complete */
 	wait_for_completion(&iser_conn->stop_completion);
-	/* Wait for IB resouces cleanup to complete */
+	/* Wait for IB resources cleanup to complete */
 	wait_for_completion(&iser_conn->ib_completion);
 
 	mutex_lock(&iser_conn->state_mutex);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index e69db43370ff..efa8d7b7844c 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -2583,7 +2583,7 @@ static void
 isert_put_unsol_pending_cmds(struct iscsit_conn *conn)
 {
 	struct iscsit_cmd *cmd, *tmp;
-	static LIST_HEAD(drop_cmd_list);
+	LIST_HEAD(drop_cmd_list);
 
 	spin_lock_bh(&conn->cmd_lock);
 	list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index eac38b57b00d..3f34cf2b7294 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -70,19 +70,24 @@ __rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type)
 {
 	size_t max_depth = clt->queue_depth;
 	struct rtrs_permit *permit;
-	int bit;
+	unsigned long bit = 0;
 
 	/*
-	 * Adapted from null_blk get_tag(). Callers from different cpus may
-	 * grab the same bit, since find_first_zero_bit is not atomic.
-	 * But then the test_and_set_bit_lock will fail for all the
-	 * callers but one, so that they will loop again.
-	 * This way an explicit spinlock is not required.
+	 * Callers from different CPUs may grab the same bit, since the bitmap
+	 * scan is not atomic. But then the test_and_set_bit_lock() will fail
+	 * for all the callers but one, so that they loop again. This way an
+	 * explicit spinlock is not required. find_next_zero_bit() resumes
+	 * from the last position so that a lost race does not rescan the
+	 * already-set low bits; if it reaches the end, wrap to the beginning
+	 * to exhaust the map and still find a permit freed below the cursor.
 	 */
 	do {
-		bit = find_first_zero_bit(clt->permits_map, max_depth);
-		if (bit >= max_depth)
-			return NULL;
+		bit = find_next_zero_bit(clt->permits_map, max_depth, bit);
+		if (bit >= max_depth) {
+			bit = find_first_zero_bit(clt->permits_map, max_depth);
+			if (bit >= max_depth)
+				return NULL;
+		}
 	} while (test_and_set_bit_lock(bit, clt->permits_map));
 
 	permit = get_permit(clt, bit);
@@ -1853,6 +1858,10 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,
 	}
 	if (con->c.cid == 0) {
 		queue_depth = le16_to_cpu(msg->queue_depth);
+		if (!queue_depth) {
+			rtrs_err(clt, "Invalid queue depth %u\n", queue_depth);
+			return -ECONNRESET;
+		}
 
 		if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) {
 			rtrs_err(clt, "Error: queue depth changed\n");
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 7197d95f2216..43c3ff1ad8e7 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -1888,7 +1888,7 @@ retry:
 	 * We divide up our send queue size into half SEND WRs to send the
 	 * completions, and half R/W contexts to actually do the RDMA
 	 * READ/WRITE transfers.  Note that we need to allocate CQ slots for
-	 * both both, as RDMA contexts will also post completions for the
+	 * both, as RDMA contexts will also post completions for the
 	 * RDMA READ case.
 	 */
 	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index 4c46b301eea1..506c11b84359 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -350,7 +350,7 @@ struct srpt_nexus {
  * struct srpt_port_attrib - attributes for SRPT port
  * @srp_max_rdma_size: Maximum size of SRP RDMA transfers for new connections.
  * @srp_max_rsp_size: Maximum size of SRP response messages in bytes.
- * @srp_sq_size: Shared receive queue (SRQ) size.
+ * @srp_sq_size: Send queue size.
  * @use_srq: Whether or not to use SRQ.
  */
 struct srpt_port_attrib {