smb: client: make use of smbdirect_connection_send_{single_iter,immediate_work}()

With this the low level send functions are in common,
we'll have to do some more changes in generic smb code
in order to use smbdirect_connection_send_iter() instead
of looping around smbdirect_connection_send_single_iter().

David's cleanups will allow us to use
smbdirect_connection_send_iter().

Cc: Steve French <smfrench@gmail.com>
Cc: Tom Talpey <tom@talpey.com>
Cc: Long Li <longli@microsoft.com>
Cc: Namjae Jeon <linkinjeon@kernel.org>
Cc: David Howells <dhowells@redhat.com>
Cc: linux-cifs@vger.kernel.org
Cc: samba-technical@lists.samba.org
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Acked-by: Namjae Jeon <linkinjeon@kernel.org>
Signed-off-by: Steve French <stfrench@microsoft.com>

1 files changed, 20 insertions, 369 deletions

diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c
index ffc6a4b6de39..ee3347289c87 100644
--- a/fs/smb/client/smbdirect.c
+++ b/fs/smb/client/smbdirect.c
@@ -817,349 +817,6 @@ static int smbd_post_send(struct smbdirect_socket *sc,
 	return smbd_ib_post_send(sc, &request->wr);
 }
 
-static void smbd_send_batch_init(struct smbdirect_send_batch *batch,
-				 bool need_invalidate_rkey,
-				 unsigned int remote_key)
-{
-	INIT_LIST_HEAD(&batch->msg_list);
-	batch->wr_cnt = 0;
-	batch->need_invalidate_rkey = need_invalidate_rkey;
-	batch->remote_key = remote_key;
-	batch->credit = 0;
-}
-
-static int smbd_send_batch_flush(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch,
-				 bool is_last)
-{
-	struct smbdirect_send_io *first, *last;
-	int ret = 0;
-
-	if (list_empty(&batch->msg_list))
-		goto release_credit;
-
-	first = list_first_entry(&batch->msg_list,
-				 struct smbdirect_send_io,
-				 sibling_list);
-	last = list_last_entry(&batch->msg_list,
-			       struct smbdirect_send_io,
-			       sibling_list);
-
-	if (batch->need_invalidate_rkey) {
-		first->wr.opcode = IB_WR_SEND_WITH_INV;
-		first->wr.ex.invalidate_rkey = batch->remote_key;
-		batch->need_invalidate_rkey = false;
-		batch->remote_key = 0;
-	}
-
-	last->wr.send_flags = IB_SEND_SIGNALED;
-	last->wr.wr_cqe = &last->cqe;
-
-	/*
-	 * Remove last from batch->msg_list
-	 * and splice the rest of batch->msg_list
-	 * to last->sibling_list.
-	 *
-	 * batch->msg_list is a valid empty list
-	 * at the end.
-	 */
-	list_del_init(&last->sibling_list);
-	list_splice_tail_init(&batch->msg_list, &last->sibling_list);
-	batch->wr_cnt = 0;
-
-	ret = smbd_ib_post_send(sc, &first->wr);
-	if (ret) {
-		struct smbdirect_send_io *sibling, *next;
-
-		list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) {
-			list_del_init(&sibling->sibling_list);
-			smbdirect_connection_free_send_io(sibling);
-		}
-		smbdirect_connection_free_send_io(last);
-	}
-
-release_credit:
-	if (is_last && !ret && batch->credit) {
-		atomic_add(batch->credit, &sc->send_io.bcredits.count);
-		batch->credit = 0;
-		wake_up(&sc->send_io.bcredits.wait_queue);
-	}
-
-	return ret;
-}
-
-static int wait_for_credits(struct smbdirect_socket *sc,
-			    wait_queue_head_t *waitq, atomic_t *total_credits,
-			    int needed)
-{
-	int ret;
-
-	do {
-		if (atomic_sub_return(needed, total_credits) >= 0)
-			return 0;
-
-		atomic_add(needed, total_credits);
-		ret = wait_event_interruptible(*waitq,
-					       atomic_read(total_credits) >= needed ||
-					       sc->status != SMBDIRECT_SOCKET_CONNECTED);
-
-		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-			return -ENOTCONN;
-		else if (ret < 0)
-			return ret;
-	} while (true);
-}
-
-static int wait_for_send_bcredit(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch)
-{
-	int ret;
-
-	if (batch->credit)
-		return 0;
-
-	ret = wait_for_credits(sc,
-			       &sc->send_io.bcredits.wait_queue,
-			       &sc->send_io.bcredits.count,
-			       1);
-	if (ret)
-		return ret;
-
-	batch->credit = 1;
-	return 0;
-}
-
-static int wait_for_send_lcredit(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch)
-{
-	if (batch && (atomic_read(&sc->send_io.lcredits.count) <= 1)) {
-		int ret;
-
-		ret = smbd_send_batch_flush(sc, batch, false);
-		if (ret)
-			return ret;
-	}
-
-	return wait_for_credits(sc,
-				&sc->send_io.lcredits.wait_queue,
-				&sc->send_io.lcredits.count,
-				1);
-}
-
-static int wait_for_send_credits(struct smbdirect_socket *sc,
-				 struct smbdirect_send_batch *batch)
-{
-	if (batch &&
-	    (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {
-		int ret;
-
-		ret = smbd_send_batch_flush(sc, batch, false);
-		if (ret)
-			return ret;
-	}
-
-	return wait_for_credits(sc,
-				&sc->send_io.credits.wait_queue,
-				&sc->send_io.credits.count,
-				1);
-}
-
-static int smbd_post_send_iter(struct smbdirect_socket *sc,
-			       struct smbdirect_send_batch *batch,
-			       struct iov_iter *iter,
-			       u32 remaining_data_length)
-{
-	struct smbdirect_socket_parameters *sp = &sc->parameters;
-	int rc;
-	int header_length;
-	int data_length;
-	struct smbdirect_send_io *request;
-	struct smbdirect_data_transfer *packet;
-	u16 new_credits = 0;
-	struct smbdirect_send_batch _batch;
-
-	if (iter) {
-		header_length = sizeof(struct smbdirect_data_transfer);
-		if (WARN_ON_ONCE(remaining_data_length == 0 ||
-				 iov_iter_count(iter) > remaining_data_length))
-			return -EINVAL;
-	} else {
-		/* If this is a packet without payload, don't send padding */
-		header_length = offsetof(struct smbdirect_data_transfer, padding);
-		if (WARN_ON_ONCE(remaining_data_length))
-			return -EINVAL;
-	}
-
-	if (!batch) {
-		smbd_send_batch_init(&_batch, false, 0);
-		batch = &_batch;
-	}
-
-	rc = wait_for_send_bcredit(sc, batch);
-	if (rc) {
-		log_outgoing(ERR, "disconnected not sending on wait_bcredit\n");
-		rc = -EAGAIN;
-		goto err_wait_bcredit;
-	}
-
-	rc = wait_for_send_lcredit(sc, batch);
-	if (rc) {
-		log_outgoing(ERR, "disconnected not sending on wait_lcredit\n");
-		rc = -EAGAIN;
-		goto err_wait_lcredit;
-	}
-
-	rc = wait_for_send_credits(sc, batch);
-	if (rc) {
-		log_outgoing(ERR, "disconnected not sending on wait_credit\n");
-		rc = -EAGAIN;
-		goto err_wait_credit;
-	}
-
-	new_credits = smbdirect_connection_grant_recv_credits(sc);
-	if (new_credits == 0 &&
-	    atomic_read(&sc->send_io.credits.count) == 0 &&
-	    atomic_read(&sc->recv_io.credits.count) == 0) {
-		queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
-		rc = wait_event_interruptible(sc->send_io.credits.wait_queue,
-					      atomic_read(&sc->send_io.credits.count) >= 1 ||
-					      atomic_read(&sc->recv_io.credits.available) >= 1 ||
-					      sc->status != SMBDIRECT_SOCKET_CONNECTED);
-		if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-			rc = -ENOTCONN;
-		if (rc < 0) {
-			log_outgoing(ERR, "disconnected not sending on last credit\n");
-			rc = -EAGAIN;
-			goto err_wait_credit;
-		}
-
-		new_credits = smbdirect_connection_grant_recv_credits(sc);
-	}
-
-	request = smbdirect_connection_alloc_send_io(sc);
-	if (IS_ERR(request)) {
-		rc = PTR_ERR(request);
-		goto err_alloc;
-	}
-
-	memset(request->sge, 0, sizeof(request->sge));
-
-	packet = smbdirect_send_io_payload(request);
-	request->sge[0].addr = ib_dma_map_single(sc->ib.dev,
-						 (void *)packet,
-						 header_length,
-						 DMA_TO_DEVICE);
-	if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) {
-		rc = -EIO;
-		goto err_dma;
-	}
-
-	request->sge[0].length = header_length;
-	request->sge[0].lkey = sc->ib.pd->local_dma_lkey;
-	request->num_sge = 1;
-
-	/* Fill in the data payload to find out how much data we can add */
-	if (iter) {
-		struct smbdirect_map_sges extract = {
-			.num_sge	= request->num_sge,
-			.max_sge	= ARRAY_SIZE(request->sge),
-			.sge		= request->sge,
-			.device		= sc->ib.dev,
-			.local_dma_lkey	= sc->ib.pd->local_dma_lkey,
-			.direction	= DMA_TO_DEVICE,
-		};
-		size_t payload_len = umin(iov_iter_count(iter),
-					  sp->max_send_size - sizeof(*packet));
-
-		rc = smbdirect_map_sges_from_iter(iter, payload_len, &extract);
-		if (rc < 0)
-			goto err_dma;
-		data_length = rc;
-		request->num_sge = extract.num_sge;
-		remaining_data_length -= data_length;
-	} else {
-		data_length = 0;
-	}
-
-	/* Fill in the packet header */
-	packet->credits_requested = cpu_to_le16(sp->send_credit_target);
-	packet->credits_granted = cpu_to_le16(new_credits);
-
-	packet->flags = 0;
-	if (smbdirect_connection_request_keep_alive(sc))
-		packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED);
-
-	packet->reserved = 0;
-	if (!data_length)
-		packet->data_offset = 0;
-	else
-		packet->data_offset = cpu_to_le32(24);
-	packet->data_length = cpu_to_le32(data_length);
-	packet->remaining_data_length = cpu_to_le32(remaining_data_length);
-	packet->padding = 0;
-
-	log_outgoing(INFO, "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
-		     le16_to_cpu(packet->credits_requested),
-		     le16_to_cpu(packet->credits_granted),
-		     le32_to_cpu(packet->data_offset),
-		     le32_to_cpu(packet->data_length),
-		     le32_to_cpu(packet->remaining_data_length));
-
-	rc = smbd_post_send(sc, batch, request);
-	if (!rc) {
-		/*
-		 * From here request is moved to batch
-		 * and we should not free it explicitly.
-		 */
-
-		if (batch != &_batch)
-			return data_length;
-
-		rc = smbd_send_batch_flush(sc, batch, true);
-		if (!rc)
-			return data_length;
-
-		goto err_flush;
-	}
-
-err_dma:
-	smbdirect_connection_free_send_io(request);
-
-err_flush:
-err_alloc:
-	atomic_inc(&sc->send_io.credits.count);
-	wake_up(&sc->send_io.credits.wait_queue);
-
-err_wait_credit:
-	atomic_inc(&sc->send_io.lcredits.count);
-	wake_up(&sc->send_io.lcredits.wait_queue);
-
-err_wait_lcredit:
-	atomic_add(batch->credit, &sc->send_io.bcredits.count);
-	batch->credit = 0;
-	wake_up(&sc->send_io.bcredits.wait_queue);
-
-err_wait_bcredit:
-	return rc;
-}
-
-/*
- * Send an empty message
- * Empty message is used to extend credits to peer to for keep live
- * while there is no upper layer payload to send at the time
- */
-static void smbd_post_send_empty(struct smbdirect_socket *sc)
-{
-	int ret;
-
-	sc->statistics.send_empty++;
-	ret = smbd_post_send_iter(sc, NULL, NULL, 0);
-	if (ret < 0) {
-		log_rdma_send(ERR, "smbd_post_send_iter failed ret=%d\n", ret);
-		smbdirect_socket_schedule_cleanup(sc, ret);
-	}
-}
-
 static int smbd_post_send_full_iter(struct smbdirect_socket *sc,
 				    struct smbdirect_send_batch *batch,
 				    struct iov_iter *iter,
@@ -1168,7 +825,7 @@ static int smbd_post_send_full_iter(struct smbdirect_socket *sc,
 	int bytes = 0;
 
 	/*
-	 * smbd_post_send_iter() respects the
+	 * smbdirect_connection_send_single_iter() respects the
 	 * negotiated max_send_size, so we need to
 	 * loop until the full iter is posted
 	 */
@@ -1176,7 +833,11 @@ static int smbd_post_send_full_iter(struct smbdirect_socket *sc,
 	while (iov_iter_count(iter) > 0) {
 		int rc;
 
-		rc = smbd_post_send_iter(sc, batch, iter, remaining_data_length);
+		rc = smbdirect_connection_send_single_iter(sc,
+							   batch,
+							   iter,
+							   0, /* flags */
+							   remaining_data_length);
 		if (rc < 0)
 			return rc;
 		remaining_data_length -= rc;
@@ -1229,18 +890,6 @@ static int smbd_negotiate(struct smbdirect_socket *sc)
 	return rc;
 }
 
-static void send_immediate_empty_message(struct work_struct *work)
-{
-	struct smbdirect_socket *sc =
-		container_of(work, struct smbdirect_socket, idle.immediate_work);
-
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
-		return;
-
-	log_keep_alive(INFO, "send an empty message\n");
-	smbd_post_send_empty(sc);
-}
-
 /*
  * Destroy the transport and related RDMA and memory resources
  * Need to go through all the pending counters and make sure on one is using
@@ -1423,7 +1072,7 @@ static struct smbd_connection *_smbd_get_connection(
 	list_for_each_entry(recv_io, &sc->recv_io.free.list, list)
 		recv_io->cqe.done = recv_done;
 
-	INIT_WORK(&sc->idle.immediate_work, send_immediate_empty_message);
+	INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work);
 	/*
 	 * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING
 	 * so that the timer will cause a disconnect.
@@ -1539,7 +1188,8 @@ int smbd_send(struct TCP_Server_Info *server,
 	struct smbdirect_socket_parameters *sp = &sc->parameters;
 	struct smb_rqst *rqst;
 	struct iov_iter iter;
-	struct smbdirect_send_batch batch;
+	struct smbdirect_send_batch_storage bstorage;
+	struct smbdirect_send_batch *batch;
 	unsigned int remaining_data_length, klen;
 	int rc, i, rqst_idx;
 	int error = 0;
@@ -1567,7 +1217,7 @@ int smbd_send(struct TCP_Server_Info *server,
 			num_rqst, remaining_data_length);
 
 	rqst_idx = 0;
-	smbd_send_batch_init(&batch, false, 0);
+	batch = smbdirect_init_send_batch_storage(&bstorage, false, 0);
 	do {
 		rqst = &rqst_array[rqst_idx];
 
@@ -1586,7 +1236,7 @@ int smbd_send(struct TCP_Server_Info *server,
 			klen += rqst->rq_iov[i].iov_len;
 		iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen);
 
-		rc = smbd_post_send_full_iter(sc, &batch, &iter, remaining_data_length);
+		rc = smbd_post_send_full_iter(sc, batch, &iter, remaining_data_length);
 		if (rc < 0) {
 			error = rc;
 			break;
@@ -1595,7 +1245,7 @@ int smbd_send(struct TCP_Server_Info *server,
 
 		if (iov_iter_count(&rqst->rq_iter) > 0) {
 			/* And then the data pages if there are any */
-			rc = smbd_post_send_full_iter(sc, &batch, &rqst->rq_iter,
+			rc = smbd_post_send_full_iter(sc, batch, &rqst->rq_iter,
 						      remaining_data_length);
 			if (rc < 0) {
 				error = rc;
@@ -1606,7 +1256,7 @@ int smbd_send(struct TCP_Server_Info *server,
 
 	} while (++rqst_idx < num_rqst);
 
-	rc = smbd_send_batch_flush(sc, &batch, true);
+	rc = smbdirect_connection_send_batch_flush(sc, batch, true);
 	if (unlikely(!rc && error))
 		rc = error;
 
@@ -1617,14 +1267,15 @@ int smbd_send(struct TCP_Server_Info *server,
 	 * that means all the I/Os have been out and we are good to return
 	 */
 
-	wait_event(sc->send_io.pending.zero_wait_queue,
-		atomic_read(&sc->send_io.pending.count) == 0 ||
-		sc->status != SMBDIRECT_SOCKET_CONNECTED);
+	error = rc;
+	rc = smbdirect_connection_send_wait_zero_pending(sc);
+	if (unlikely(rc && !error))
+		error = -EAGAIN;
 
-	if (sc->status != SMBDIRECT_SOCKET_CONNECTED && rc == 0)
-		rc = -EAGAIN;
+	if (unlikely(error))
+		return error;
 
-	return rc;
+	return 0;
 }
 
 /*