From 6551300dc452ac16a855a83dbd1e74899542d3b3 Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Tue, 14 Apr 2026 18:54:38 -0400 Subject: smb: server: fix active_num_conn leak on transport allocation failure Commit 77ffbcac4e56 ("smb: server: fix leak of active_num_conn in ksmbd_tcp_new_connection()") addressed the kthread_run() failure path. The earlier alloc_transport() == NULL path in the same function has the same leak, is reachable pre-authentication via any TCP connect to port 445, and was empirically reproduced on UML (ARCH=um, v7.0-rc7): a small number of forced allocation failures were sufficient to put ksmbd into a state where every subsequent connection attempt was rejected for the remainder of the boot. ksmbd_kthread_fn() increments active_num_conn before calling ksmbd_tcp_new_connection() and discards the return value, so when alloc_transport() returns NULL the socket is released and -ENOMEM returned without decrementing the counter. Each such failure permanently consumes one slot from the max_connections pool; once cumulative failures reach the cap, atomic_inc_return() hits the threshold on every subsequent accept and every new connection is rejected. The counter is only reset by module reload. An unauthenticated remote attacker can drive the server toward the memory pressure that makes alloc_transport() fail by holding open connections with large RFC1002 lengths up to MAX_STREAM_PROT_LEN (0x00FFFFFF); natural transient allocation failures on a loaded host produce the same drift more slowly. Mirror the existing rollback pattern in ksmbd_kthread_fn(): on the alloc_transport() failure path, decrement active_num_conn gated on server_conf.max_connections. Repro details: with the patch reverted, forced alloc_transport() NULL returns leaked counter slots and subsequent connection attempts -- including legitimate connects issued after the forced-fail window had closed -- were all rejected with "Limit the maximum number of connections". With this patch applied, the same connect sequence produces no rejections and the counter cycles cleanly between zero and one on every accept. Fixes: 0d0d4680db22 ("ksmbd: add max connections parameter") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Assisted-by: Codex:gpt-5-4 Signed-off-by: Michael Bommarito Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/transport_tcp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 7e29b06820e2..8d7fe71f525c 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -183,6 +183,8 @@ static int ksmbd_tcp_new_connection(struct socket *client_sk) t = alloc_transport(client_sk); if (!t) { sock_release(client_sk); + if (server_conf.max_connections) + atomic_dec(&active_num_conn); return -ENOMEM; } -- cgit v1.2.3 From d6a6aa81eac2c9bff66dc6e191179cb69a14426b Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Wed, 15 Apr 2026 07:25:00 -0400 Subject: ksmbd: validate response sizes in ipc_validate_msg() ipc_validate_msg() computes the expected message size for each response type by adding (or multiplying) attacker-controlled fields from the daemon response to a fixed struct size in unsigned int arithmetic. Three cases can overflow: KSMBD_EVENT_RPC_REQUEST: msg_sz = sizeof(struct ksmbd_rpc_command) + resp->payload_sz; KSMBD_EVENT_SHARE_CONFIG_REQUEST: msg_sz = sizeof(struct ksmbd_share_config_response) + resp->payload_sz; KSMBD_EVENT_LOGIN_REQUEST_EXT: msg_sz = sizeof(struct ksmbd_login_response_ext) + resp->ngroups * sizeof(gid_t); resp->payload_sz is __u32 and resp->ngroups is __s32. Each addition can wrap in unsigned int; the multiplication by sizeof(gid_t) mixes signed and size_t, so a negative ngroups is converted to SIZE_MAX before the multiply. A wrapped value of msg_sz that happens to equal entry->msg_sz bypasses the size check on the next line, and downstream consumers (smb2pdu.c:6742 memcpy using rpc_resp->payload_sz, kmemdup in ksmbd_alloc_user using resp_ext->ngroups) then trust the unverified length. Use check_add_overflow() on the RPC_REQUEST and SHARE_CONFIG_REQUEST paths to detect integer overflow without constraining functional payload size; userspace ksmbd-tools grows NDR responses in 4096-byte chunks for calls like NetShareEnumAll, so a hard transport cap is unworkable on the response side. For LOGIN_REQUEST_EXT, reject resp->ngroups outside the signed [0, NGROUPS_MAX] range up front and report the error from ipc_validate_msg() so it fires at the IPC boundary; with that bound the subsequent multiplication and addition stay well below UINT_MAX. The now-redundant ngroups check and pr_err in ksmbd_alloc_user() are removed. This is the response-side analogue of aab98e2dbd64 ("ksmbd: fix integer overflows on 32 bit systems"), which hardened the request side. Fixes: 0626e6641f6b ("cifsd: add server handler for central processing and tranport layers") Fixes: a77e0e02af1c ("ksmbd: add support for supplementary groups") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Assisted-by: Codex:gpt-5-4 Signed-off-by: Michael Bommarito Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/mgmt/user_config.c | 6 ------ fs/smb/server/transport_ipc.c | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c index a3183fe5c536..cf45841d9d1b 100644 --- a/fs/smb/server/mgmt/user_config.c +++ b/fs/smb/server/mgmt/user_config.c @@ -56,12 +56,6 @@ struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, goto err_free; if (resp_ext) { - if (resp_ext->ngroups > NGROUPS_MAX) { - pr_err("ngroups(%u) from login response exceeds max groups(%d)\n", - resp_ext->ngroups, NGROUPS_MAX); - goto err_free; - } - user->sgid = kmemdup(resp_ext->____payload, resp_ext->ngroups * sizeof(gid_t), KSMBD_DEFAULT_GFP); diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index f7aa427a06fe..0c581b9624d3 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "vfs_cache.h" #include "transport_ipc.h" @@ -496,7 +497,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_rpc_command *resp = entry->response; - msg_sz = sizeof(struct ksmbd_rpc_command) + resp->payload_sz; + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), + resp->payload_sz, &msg_sz)) + return -EINVAL; break; } case KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST: @@ -515,8 +518,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) if (resp->payload_sz < resp->veto_list_sz) return -EINVAL; - msg_sz = sizeof(struct ksmbd_share_config_response) + - resp->payload_sz; + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; } break; } @@ -525,6 +529,12 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) struct ksmbd_login_response_ext *resp = entry->response; if (resp->ngroups) { + if (resp->ngroups < 0 || + resp->ngroups > NGROUPS_MAX) { + pr_err("ngroups(%d) from login response exceeds max groups(%d)\n", + resp->ngroups, NGROUPS_MAX); + return -EINVAL; + } msg_sz = sizeof(struct ksmbd_login_response_ext) + resp->ngroups * sizeof(gid_t); } -- cgit v1.2.3 From d07b26f39246a82399661936dd0c853983cfade7 Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Tue, 14 Apr 2026 15:15:33 -0400 Subject: ksmbd: require minimum ACE size in smb_check_perm_dacl() Both ACE-walk loops in smb_check_perm_dacl() only guard against an under-sized remaining buffer, not against an ACE whose declared `ace->size` is smaller than the struct it claims to describe: if (offsetof(struct smb_ace, access_req) > aces_size) break; ace_size = le16_to_cpu(ace->size); if (ace_size > aces_size) break; The first check only requires the 4-byte ACE header to be in bounds; it does not require access_req (4 bytes at offset 4) to be readable. An attacker who has set a crafted DACL on a file they own can declare ace->size == 4 with aces_size == 4, pass both checks, and then granted |= le32_to_cpu(ace->access_req); /* upper loop */ compare_sids(&sid, &ace->sid); /* lower loop */ reads access_req at offset 4 (OOB by up to 4 bytes) and ace->sid at offset 8 (OOB by up to CIFS_SID_BASE_SIZE + SID_MAX_SUB_AUTHORITIES * 4 bytes). Tighten both loops to require ace_size >= offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE which is the smallest valid on-wire ACE layout (4-byte header + 4-byte access_req + 8-byte sid base with zero sub-auths). Also reject ACEs whose sid.num_subauth exceeds SID_MAX_SUB_AUTHORITIES before letting compare_sids() dereference sub_auth[] entries. parse_sec_desc() already enforces an equivalent check (lines 441-448); smb_check_perm_dacl() simply grew weaker validation over time. Reachability: authenticated SMB client with permission to set an ACL on a file. On a subsequent CREATE against that file, the kernel walks the stored DACL via smb_check_perm_dacl() and triggers the OOB read. Not pre-auth, and the OOB read is not reflected to the attacker, but KASAN reports and kernel state corruption are possible. Fixes: e2f34481b24d ("cifsd: add server-side procedures for SMB3") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Assisted-by: Codex:gpt-5-4 Signed-off-by: Michael Bommarito Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smbacl.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index 061a305bf9c8..bba26a0355bb 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -1342,10 +1342,13 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, access_req) > aces_size) + if (offsetof(struct smb_ace, sid) + + aces_size < CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); - if (ace_size > aces_size) + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; granted |= le32_to_cpu(ace->access_req); @@ -1360,13 +1363,19 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, access_req) > aces_size) + if (offsetof(struct smb_ace, sid) + + aces_size < CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); - if (ace_size > aces_size) + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + break; + if (!compare_sids(&sid, &ace->sid) || !compare_sids(&sid_unix_NFS_mode, &ace->sid)) { found = 1; -- cgit v1.2.3 From ce23158bfe584bd90d1918f279fdf9de57802012 Mon Sep 17 00:00:00 2001 From: DaeMyung Kang Date: Fri, 17 Apr 2026 06:17:35 +0900 Subject: smb: server: fix max_connections off-by-one in tcp accept path The global max_connections check in ksmbd's TCP accept path counts the newly accepted connection with atomic_inc_return(), but then rejects the connection when the result is greater than or equal to server_conf.max_connections. That makes the effective limit one smaller than configured. For example: - max_connections=1 rejects the first connection - max_connections=2 allows only one connection The per-IP limit in the same function uses <= correctly because it counts only pre-existing connections. The global limit instead checks the post-increment total, so it should reject only when that total exceeds the configured maximum. Fix this by changing the comparison from >= to >, so exactly max_connections simultaneous connections are allowed and the next one is rejected. This matches the documented meaning of max_connections in fs/smb/server/ksmbd_netlink.h as the "Number of maximum simultaneous connections". Fixes: 0d0d4680db22 ("ksmbd: add max connections parameter") Cc: stable@vger.kernel.org Signed-off-by: DaeMyung Kang Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/transport_tcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 8d7fe71f525c..13b711ea575d 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -281,7 +281,7 @@ static int ksmbd_kthread_fn(void *p) skip_max_ip_conns_limit: if (server_conf.max_connections && - atomic_inc_return(&active_num_conn) >= server_conf.max_connections) { + atomic_inc_return(&active_num_conn) > server_conf.max_connections) { pr_info_ratelimited("Limit the maximum number of connections(%u)\n", atomic_read(&active_num_conn)); atomic_dec(&active_num_conn); -- cgit v1.2.3 From 3e4e2ea2a781018ed5d75f969e3e5606beb66e48 Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Fri, 17 Apr 2026 14:45:57 -0400 Subject: ksmbd: validate num_aces and harden ACE walk in smb_inherit_dacl() smb_inherit_dacl() trusts the on-disk num_aces value from the parent directory's DACL xattr and uses it to size a heap allocation: aces_base = kmalloc(sizeof(struct smb_ace) * num_aces * 2, ...); num_aces is a u16 read from le16_to_cpu(parent_pdacl->num_aces) without checking that it is consistent with the declared pdacl_size. An authenticated client whose parent directory's security.NTACL is tampered (e.g. via offline xattr corruption or a concurrent path that bypasses parse_dacl()) can present num_aces = 65535 with minimal actual ACE data. This causes a ~8 MB allocation (not kzalloc, so uninitialized) that the subsequent loop only partially populates, and may also overflow the three-way size_t multiply on 32-bit kernels. Additionally, the ACE walk loop uses the weaker offsetof(struct smb_ace, access_req) minimum size check rather than the minimum valid on-wire ACE size, and does not reject ACEs whose declared size is below the minimum. Reproduced on UML + KASAN + LOCKDEP against the real ksmbd code path. A legitimate mount.cifs client creates a parent directory over SMB (ksmbd writes a valid security.NTACL xattr), then the NTACL blob on the backing filesystem is rewritten to set num_aces = 0xFFFF while keeping the posix_acl_hash bytes intact so ksmbd_vfs_get_sd_xattr()'s hash check still passes. A subsequent SMB2 CREATE of a child under that parent drives smb2_open() into smb_inherit_dacl() (share has "vfs objects = acl_xattr" set), which fails the page allocator: WARNING: mm/page_alloc.c:5226 at __alloc_frozen_pages_noprof+0x46c/0x9c0 Workqueue: ksmbd-io handle_ksmbd_work __alloc_frozen_pages_noprof+0x46c/0x9c0 ___kmalloc_large_node+0x68/0x130 __kmalloc_large_node_noprof+0x24/0x70 __kmalloc_noprof+0x4c9/0x690 smb_inherit_dacl+0x394/0x2430 smb2_open+0x595d/0xabe0 handle_ksmbd_work+0x3d3/0x1140 With the patch applied the added guard rejects the tampered value with -EINVAL before any large allocation runs, smb2_open() falls back to smb2_create_sd_buffer(), and the child is created with a default SD. No warning, no splat. Fix by: 1. Validating num_aces against pdacl_size using the same formula applied in parse_dacl(). 2. Replacing the raw kmalloc(sizeof * num_aces * 2) with kmalloc_array(num_aces * 2, sizeof(...)) for overflow-safe allocation. 3. Tightening the per-ACE loop guard to require the minimum valid ACE size (offsetof(smb_ace, sid) + CIFS_SID_BASE_SIZE) and rejecting under-sized ACEs, matching the hardening in smb_check_perm_dacl() and parse_dacl(). v1 -> v2: - Replace the synthetic test-module splat in the changelog with a real-path UML + KASAN reproduction driven through mount.cifs and SMB2 CREATE; Namjae flagged the kcifs3_test_inherit_dacl_old name in v1 since it does not exist in ksmbd. - Drop the commit-hash citation from the code comment per Namjae's review; keep the parse_dacl() pointer. Fixes: e2f34481b24d ("cifsd: add server-side procedures for SMB3") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Signed-off-by: Michael Bommarito Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smbacl.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index bba26a0355bb..a1de89cc09be 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -1106,8 +1106,24 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, goto free_parent_pntsd; } - aces_base = kmalloc(sizeof(struct smb_ace) * num_aces * 2, - KSMBD_DEFAULT_GFP); + aces_size = pdacl_size - sizeof(struct smb_acl); + + /* + * Validate num_aces against the DACL payload before allocating. + * Each ACE must be at least as large as its fixed-size header + * (up to the SID base), so num_aces cannot exceed the payload + * divided by the minimum ACE size. This mirrors the existing + * check in parse_dacl(). + */ + if (num_aces > aces_size / (offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth) + + sizeof(__le16))) { + rc = -EINVAL; + goto free_parent_pntsd; + } + + aces_base = kmalloc_array(num_aces * 2, sizeof(struct smb_ace), + KSMBD_DEFAULT_GFP); if (!aces_base) { rc = -ENOMEM; goto free_parent_pntsd; @@ -1116,7 +1132,6 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, aces = (struct smb_ace *)aces_base; parent_aces = (struct smb_ace *)((char *)parent_pdacl + sizeof(struct smb_acl)); - aces_size = acl_len - sizeof(struct smb_acl); if (pntsd_type & DACL_AUTO_INHERITED) inherited_flags = INHERITED_ACE; @@ -1124,11 +1139,14 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, for (i = 0; i < num_aces; i++) { int pace_size; - if (offsetof(struct smb_ace, access_req) > aces_size) + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; pace_size = le16_to_cpu(parent_aces->size); - if (pace_size > aces_size) + if (pace_size > aces_size || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= pace_size; -- cgit v1.2.3 From 1baff47b81f94f9231c91236aa511420d0e266b9 Mon Sep 17 00:00:00 2001 From: Akif Date: Fri, 17 Apr 2026 23:57:09 +0530 Subject: ksmbd: fix use-after-free in smb2_open during durable reconnect In smb2_open, the call to ksmbd_put_durable_fd(fp) drops the reference to the durable file descriptor early during the durable reconnect process. If an error occurs subsequently (eg, ksmbd_iov_pin_rsp fails) or a scavenger accesses the file, it leads to a use-after-free when accessing fp properties (eg fp->create_time). Move the single put to the end of the function below err_out2 so fp stays valid until smb2_open returns. Fixes: c8efcc786146 ("ksmbd: add support for durable handles v1/v2") Signed-off-by: Akif Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smb2pdu.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index ee32e61b6d3c..395007c82831 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -3014,29 +3014,23 @@ int smb2_open(struct ksmbd_work *work) if (dh_info.reconnected == true) { rc = smb2_check_durable_oplock(conn, share, dh_info.fp, lc, sess->user, name); - if (rc) { - ksmbd_put_durable_fd(dh_info.fp); + if (rc) goto err_out2; - } rc = ksmbd_reopen_durable_fd(work, dh_info.fp); - if (rc) { - ksmbd_put_durable_fd(dh_info.fp); + if (rc) goto err_out2; - } fp = dh_info.fp; if (ksmbd_override_fsids(work)) { rc = -ENOMEM; - ksmbd_put_durable_fd(dh_info.fp); goto err_out2; } file_info = FILE_OPENED; rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); - ksmbd_put_durable_fd(fp); if (rc) goto err_out2; @@ -3806,6 +3800,9 @@ err_out2: ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status); } + if (dh_info.reconnected) + ksmbd_put_durable_fd(dh_info.fp); + kfree(name); kfree(lc); -- cgit v1.2.3 From 299f962c0b02d048fb45d248b4da493d03f3175d Mon Sep 17 00:00:00 2001 From: Tristan Madani Date: Fri, 17 Apr 2026 19:54:57 +0000 Subject: ksmbd: use check_add_overflow() to prevent u16 DACL size overflow set_posix_acl_entries_dacl() and set_ntacl_dacl() accumulate ACE sizes in u16 variables. When a file has many POSIX ACL entries, the accumulated size can wrap past 65535, causing the pointer arithmetic (char *)pndace + *size to land within already-written ACEs. Subsequent writes then overwrite earlier entries, and pndacl->size gets a truncated value. Use check_add_overflow() at each accumulation point to detect the wrap before it corrupts the buffer, consistent with existing check_mul_overflow() usage elsewhere in smbacl.c. Cc: stable@vger.kernel.org Fixes: e2f34481b24d ("cifsd: add server-side procedures for SMB3") Signed-off-by: Tristan Madani Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smbacl.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index a1de89cc09be..4bbc2c27e680 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -596,6 +596,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, struct smb_sid *sid; struct smb_ace *ntace; int i, j; + u16 ace_sz; if (!fattr->cf_acls) goto posix_default_acl; @@ -640,8 +641,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, flags = 0x03; ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -650,8 +653,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, if (S_ISDIR(fattr->cf_mode) && (pace->e_tag == ACL_USER || pace->e_tag == ACL_GROUP)) { ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x03, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -691,8 +696,10 @@ posix_default_acl: } ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -728,7 +735,8 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, break; memcpy((char *)pndace + size, ntace, nt_ace_size); - size += nt_ace_size; + if (check_add_overflow(size, nt_ace_size, &size)) + break; aces_size -= nt_ace_size; ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); num_aces++; -- cgit v1.2.3 From 30010c952077a1c89ecdd71fc4d574c75a8f5617 Mon Sep 17 00:00:00 2001 From: Tristan Madani Date: Fri, 17 Apr 2026 19:33:17 +0000 Subject: ksmbd: fix out-of-bounds write in smb2_get_ea() EA alignment smb2_get_ea() applies 4-byte alignment padding via memset() after writing each EA entry. The bounds check on buf_free_len is performed before the value memcpy, but the alignment memset fires unconditionally afterward with no check on remaining space. When the EA value exactly fills the remaining buffer (buf_free_len == 0 after value subtraction), the alignment memset writes 1-3 NUL bytes past the buf_free_len boundary. In compound requests where the response buffer is shared across commands, the first command (e.g., READ) can consume most of the buffer, leaving a tight remainder for the QUERY_INFO EA response. The alignment memset then overwrites past the physical kvmalloc allocation into adjacent kernel heap memory. Add a bounds check before the alignment memset to ensure buf_free_len can accommodate the padding bytes. This is the same bug pattern fixed by commit beef2634f81f ("ksmbd: fix potencial OOB in get_file_all_info() for compound requests") and commit fda9522ed6af ("ksmbd: fix OOB write in QUERY_INFO for compound requests"), both of which added bounds checks before unconditional writes in QUERY_INFO response handlers. Cc: stable@vger.kernel.org Fixes: e2b76ab8b5c9 ("ksmbd: add support for read compound") Signed-off-by: Tristan Madani Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smb2pdu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 395007c82831..652b6771ccaf 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -4818,6 +4818,8 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, /* align next xattr entry at 4 byte bundary */ alignment_bytes = ((next_offset + 3) & ~3) - next_offset; if (alignment_bytes) { + if (buf_free_len < alignment_bytes) + break; memset(ptr, '\0', alignment_bytes); ptr += alignment_bytes; next_offset += alignment_bytes; -- cgit v1.2.3 From def036ef87f8641c1c525d5ae17438d7a1006491 Mon Sep 17 00:00:00 2001 From: DaeMyung Kang Date: Sun, 19 Apr 2026 02:28:44 +0900 Subject: ksmbd: reset rcount per connection in ksmbd_conn_wait_idle_sess_id() rcount is intended to be connection-specific: 2 for curr_conn, 1 for every other connection sharing the same session. However, it is initialised only once before the hash iteration and is never reset. After the loop visits curr_conn, later sibling connections are also checked against rcount == 2, so a sibling with req_running == 1 is incorrectly treated as idle. This makes the outcome depend on the hash iteration order: whether a given sibling is checked against the loose (< 2) or the strict (< 1) threshold is decided by whether it happens to be visited before or after curr_conn. The function's contract is "wait until every connection sharing this session is idle" so that destroy_previous_session() can safely tear the session down. The latched rcount violates that contract and reopens the teardown race window the wait logic was meant to close: destroy_previous_session() may proceed before sibling channels have actually quiesced, overlapping session teardown with in-flight work on those connections. Recompute rcount inside the loop so each connection is compared against its own threshold regardless of iteration order. This is a code-inspection fix for an iteration-order-dependent logic error; a targeted reproducer would require SMB3 multichannel with in-flight work on a sibling channel landing after curr_conn in hash order, which is not something that can be triggered reliably. Fixes: 76e98a158b20 ("ksmbd: fix race condition between destroy_previous_session() and smb2 operations()") Cc: stable@vger.kernel.org Signed-off-by: DaeMyung Kang Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/connection.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index a26899d12df1..b5e077f272cf 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -237,7 +237,7 @@ int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) { struct ksmbd_conn *conn; int rc, retry_count = 0, max_timeout = 120; - int rcount = 1, bkt; + int rcount, bkt; retry_idle: if (retry_count >= max_timeout) @@ -246,8 +246,7 @@ retry_idle: down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { if (conn->binding || xa_load(&conn->sessions, sess_id)) { - if (conn == curr_conn) - rcount = 2; + rcount = (conn == curr_conn) ? 2 : 1; if (atomic_read(&conn->req_running) >= rcount) { rc = wait_event_timeout(conn->req_running_q, atomic_read(&conn->req_running) < rcount, -- cgit v1.2.3 From cc92b479b6ed1d7d1a6eb13aba472badb836a832 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 18 Apr 2026 15:17:07 -0700 Subject: ksmbd: Use AES-CMAC library for SMB3 signature calculation Now that AES-CMAC has a library API, convert ksmbd_sign_smb3_pdu() to use it instead of a "cmac(aes)" crypto_shash. The result is simpler and faster code. With the library there's no need to dynamically allocate memory, no need to handle errors, and the AES-CMAC code is accessed directly without inefficient indirect calls and other unnecessary API overhead. Acked-by: Namjae Jeon Reviewed-by: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Steve French --- fs/smb/server/Kconfig | 2 +- fs/smb/server/auth.c | 51 +++++++++++----------------------------- fs/smb/server/auth.h | 4 ++-- fs/smb/server/crypto_ctx.c | 58 ---------------------------------------------- fs/smb/server/crypto_ctx.h | 12 ---------- fs/smb/server/server.c | 1 - fs/smb/server/smb2pdu.c | 8 +++---- 7 files changed, 19 insertions(+), 117 deletions(-) diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 37387410e5bb..8827b3653786 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -7,13 +7,13 @@ config SMB_SERVER select NLS_UTF8 select NLS_UCS2_UTILS select CRYPTO + select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_ARC4 select CRYPTO_LIB_DES select CRYPTO_LIB_MD5 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 select CRYPTO_LIB_UTILS - select CRYPTO_CMAC select CRYPTO_AEAD2 select CRYPTO_CCM select CRYPTO_GCM diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index 7d0691f7263f..e99409fa721c 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -11,8 +11,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -490,46 +490,21 @@ void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, * @sig: signature value generated for client request packet * */ -int ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, - int n_vec, char *sig) +void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, + int n_vec, char *sig) { - struct ksmbd_crypto_ctx *ctx; - int rc, i; - - ctx = ksmbd_crypto_ctx_find_cmacaes(); - if (!ctx) { - ksmbd_debug(AUTH, "could not crypto alloc cmac\n"); - return -ENOMEM; - } - - rc = crypto_shash_setkey(CRYPTO_CMACAES_TFM(ctx), - key, - SMB2_CMACAES_SIZE); - if (rc) - goto out; - - rc = crypto_shash_init(CRYPTO_CMACAES(ctx)); - if (rc) { - ksmbd_debug(AUTH, "cmaces init error %d\n", rc); - goto out; - } + struct aes_cmac_key cmac_key; + struct aes_cmac_ctx cmac_ctx; + int i; - for (i = 0; i < n_vec; i++) { - rc = crypto_shash_update(CRYPTO_CMACAES(ctx), - iov[i].iov_base, - iov[i].iov_len); - if (rc) { - ksmbd_debug(AUTH, "cmaces update error %d\n", rc); - goto out; - } - } + /* This cannot fail, since we always pass a valid key length. */ + static_assert(SMB2_CMACAES_SIZE == AES_KEYSIZE_128); + aes_cmac_preparekey(&cmac_key, key, SMB2_CMACAES_SIZE); - rc = crypto_shash_final(CRYPTO_CMACAES(ctx), sig); - if (rc) - ksmbd_debug(AUTH, "cmaces generation error %d\n", rc); -out: - ksmbd_release_crypto_ctx(ctx); - return rc; + aes_cmac_init(&cmac_ctx, &cmac_key); + for (i = 0; i < n_vec; i++) + aes_cmac_update(&cmac_ctx, iov[i].iov_base, iov[i].iov_len); + aes_cmac_final(&cmac_ctx, sig); } struct derivation { diff --git a/fs/smb/server/auth.h b/fs/smb/server/auth.h index 6d351d61b0e5..5767aabc63c9 100644 --- a/fs/smb/server/auth.h +++ b/fs/smb/server/auth.h @@ -54,8 +54,8 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, int in_len, char *out_blob, int *out_len); void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, int n_vec, char *sig); -int ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, - int n_vec, char *sig); +void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, + int n_vec, char *sig); int ksmbd_gen_smb30_signingkey(struct ksmbd_session *sess, struct ksmbd_conn *conn); int ksmbd_gen_smb311_signingkey(struct ksmbd_session *sess, diff --git a/fs/smb/server/crypto_ctx.c b/fs/smb/server/crypto_ctx.c index 8fd9713b00b7..2fe7d3300480 100644 --- a/fs/smb/server/crypto_ctx.c +++ b/fs/smb/server/crypto_ctx.c @@ -28,14 +28,6 @@ static inline void free_aead(struct crypto_aead *aead) crypto_free_aead(aead); } -static void free_shash(struct shash_desc *shash) -{ - if (shash) { - crypto_free_shash(shash->tfm); - kfree(shash); - } -} - static struct crypto_aead *alloc_aead(int id) { struct crypto_aead *tfm = NULL; @@ -60,37 +52,10 @@ static struct crypto_aead *alloc_aead(int id) return tfm; } -static struct shash_desc *alloc_shash_desc(int id) -{ - struct crypto_shash *tfm = NULL; - struct shash_desc *shash; - - switch (id) { - case CRYPTO_SHASH_CMACAES: - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - break; - default: - return NULL; - } - - if (IS_ERR(tfm)) - return NULL; - - shash = kzalloc(sizeof(*shash) + crypto_shash_descsize(tfm), - KSMBD_DEFAULT_GFP); - if (!shash) - crypto_free_shash(tfm); - else - shash->tfm = tfm; - return shash; -} - static void ctx_free(struct ksmbd_crypto_ctx *ctx) { int i; - for (i = 0; i < CRYPTO_SHASH_MAX; i++) - free_shash(ctx->desc[i]); for (i = 0; i < CRYPTO_AEAD_MAX; i++) free_aead(ctx->ccmaes[i]); kfree(ctx); @@ -153,29 +118,6 @@ void ksmbd_release_crypto_ctx(struct ksmbd_crypto_ctx *ctx) ctx_free(ctx); } -static struct ksmbd_crypto_ctx *____crypto_shash_ctx_find(int id) -{ - struct ksmbd_crypto_ctx *ctx; - - if (id >= CRYPTO_SHASH_MAX) - return NULL; - - ctx = ksmbd_find_crypto_ctx(); - if (ctx->desc[id]) - return ctx; - - ctx->desc[id] = alloc_shash_desc(id); - if (ctx->desc[id]) - return ctx; - ksmbd_release_crypto_ctx(ctx); - return NULL; -} - -struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_cmacaes(void) -{ - return ____crypto_shash_ctx_find(CRYPTO_SHASH_CMACAES); -} - static struct ksmbd_crypto_ctx *____crypto_aead_ctx_find(int id) { struct ksmbd_crypto_ctx *ctx; diff --git a/fs/smb/server/crypto_ctx.h b/fs/smb/server/crypto_ctx.h index 27fd553d10aa..b22c6e086f03 100644 --- a/fs/smb/server/crypto_ctx.h +++ b/fs/smb/server/crypto_ctx.h @@ -6,14 +6,8 @@ #ifndef __CRYPTO_CTX_H__ #define __CRYPTO_CTX_H__ -#include #include -enum { - CRYPTO_SHASH_CMACAES = 0, - CRYPTO_SHASH_MAX, -}; - enum { CRYPTO_AEAD_AES_GCM = 16, CRYPTO_AEAD_AES_CCM, @@ -23,19 +17,13 @@ enum { struct ksmbd_crypto_ctx { struct list_head list; - struct shash_desc *desc[CRYPTO_SHASH_MAX]; struct crypto_aead *ccmaes[CRYPTO_AEAD_MAX]; }; -#define CRYPTO_CMACAES(c) ((c)->desc[CRYPTO_SHASH_CMACAES]) - -#define CRYPTO_CMACAES_TFM(c) ((c)->desc[CRYPTO_SHASH_CMACAES]->tfm) - #define CRYPTO_GCM(c) ((c)->ccmaes[CRYPTO_AEAD_AES_GCM]) #define CRYPTO_CCM(c) ((c)->ccmaes[CRYPTO_AEAD_AES_CCM]) void ksmbd_release_crypto_ctx(struct ksmbd_crypto_ctx *ctx); -struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_cmacaes(void); struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_gcm(void); struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_ccm(void); void ksmbd_crypto_destroy(void); diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index d8893079abdb..58ef02c423fc 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -631,7 +631,6 @@ MODULE_DESCRIPTION("Linux kernel CIFS/SMB SERVER"); MODULE_LICENSE("GPL"); MODULE_SOFTDEP("pre: nls"); MODULE_SOFTDEP("pre: aes"); -MODULE_SOFTDEP("pre: cmac"); MODULE_SOFTDEP("pre: aead2"); MODULE_SOFTDEP("pre: ccm"); MODULE_SOFTDEP("pre: gcm"); diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 652b6771ccaf..a5d9a56cdee8 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -9066,8 +9066,7 @@ int smb3_check_sign_req(struct ksmbd_work *work) iov[0].iov_base = (char *)&hdr->ProtocolId; iov[0].iov_len = len; - if (ksmbd_sign_smb3_pdu(conn, signing_key, iov, 1, signature)) - return 0; + ksmbd_sign_smb3_pdu(conn, signing_key, iov, 1, signature); if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { pr_err("bad smb2 signature\n"); @@ -9118,9 +9117,8 @@ void smb3_set_sign_rsp(struct ksmbd_work *work) iov = &work->iov[work->iov_idx]; } - if (!ksmbd_sign_smb3_pdu(conn, signing_key, iov, n_vec, - signature)) - memcpy(hdr->Signature, signature, SMB2_SIGNATURE_SIZE); + ksmbd_sign_smb3_pdu(conn, signing_key, iov, n_vec, signature); + memcpy(hdr->Signature, signature, SMB2_SIGNATURE_SIZE); } /** -- cgit v1.2.3 From c049ee14eb4343b69b6f7755563f961f5e153423 Mon Sep 17 00:00:00 2001 From: DaeMyung Kang Date: Sun, 19 Apr 2026 20:02:54 +0900 Subject: ksmbd: destroy tree_conn_ida in ksmbd_session_destroy() When per-session tree_conn_ida was converted from a dynamically allocated ksmbd_ida to an embedded struct ida, ksmbd_ida_free() was removed from ksmbd_session_destroy() but no matching ida_destroy() was added. The session is therefore freed with the IDA's backing xarray still intact. The kernel IDA API expects ida_init() and ida_destroy() to be paired over an object's lifetime, so add the missing cleanup before the enclosing session is freed. Also move ida_init() to right after the session is allocated so that it is always paired with the destroy call even on the early error paths of __session_create() (ksmbd_init_file_table() or __init_smb2_session() failures), both of which jump to the error label and invoke ksmbd_session_destroy() on a partially initialised session. No leak has been observed in testing; this is a pairing fix to match the IDA lifetime rules, not a response to a reproduced regression. Fixes: d40012a83f87 ("cifsd: declare ida statically") Signed-off-by: DaeMyung Kang Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/mgmt/user_session.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index a86589408835..0dd9e6c976ac 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -391,6 +391,7 @@ void ksmbd_session_destroy(struct ksmbd_session *sess) free_channel_list(sess); kfree(sess->Preauth_HashValue); ksmbd_release_id(&session_ida, sess->id); + ida_destroy(&sess->tree_conn_ida); kfree(sess); } @@ -665,6 +666,8 @@ static struct ksmbd_session *__session_create(int protocol) if (!sess) return NULL; + ida_init(&sess->tree_conn_ida); + if (ksmbd_init_file_table(&sess->file_table)) goto error; @@ -684,8 +687,6 @@ static struct ksmbd_session *__session_create(int protocol) if (ret) goto error; - ida_init(&sess->tree_conn_ida); - down_write(&sessions_table_lock); hash_add(sessions_table, &sess->hlist, sess->id); up_write(&sessions_table_lock); -- cgit v1.2.3 From b32c8db48212a34998c36d0bbc05b29d5c407ef5 Mon Sep 17 00:00:00 2001 From: DaeMyung Kang Date: Sun, 19 Apr 2026 20:02:55 +0900 Subject: ksmbd: destroy async_ida in ksmbd_conn_free() When per-connection async_ida was converted from a dynamically allocated ksmbd_ida to an embedded struct ida, ksmbd_ida_free() was removed from the connection teardown path but no matching ida_destroy() was added. The connection is therefore freed with the IDA's backing xarray still intact. The kernel IDA API expects ida_init() and ida_destroy() to be paired over an object's lifetime, so add the missing cleanup before the connection is freed. No leak has been observed in testing; this is a pairing fix to match the IDA lifetime rules, not a response to a reproduced regression. Fixes: d40012a83f87 ("cifsd: declare ida statically") Signed-off-by: DaeMyung Kang Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/connection.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index b5e077f272cf..fbbc0529743f 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -98,6 +98,15 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) kfree(conn->preauth_info); kfree(conn->mechToken); if (atomic_dec_and_test(&conn->refcnt)) { + /* + * async_ida is embedded in struct ksmbd_conn, so pair + * ida_destroy() with the final kfree() rather than with + * the unconditional field teardown above. This keeps + * the IDA valid for the entire lifetime of the struct, + * even while other refcount holders (oplock / vfs + * durable handles) still reference the connection. + */ + ida_destroy(&conn->async_ida); conn->transport->ops->free_transport(conn->transport); kfree(conn); } -- cgit v1.2.3 From bd0a1ca52b6da64b1a163f103b28b488b20497fe Mon Sep 17 00:00:00 2001 From: Akif Sait Date: Mon, 20 Apr 2026 10:58:26 +0900 Subject: ksmbd: fix O(N^2) DoS in smb2_lock via unbounded LockCount smb2_lock() performs O(N^2) conflict detection with no cap on LockCount. Cap lock_count at 64 to prevent CPU exhaustion from a single request. Signed-off-by: Akif Sait Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smb2pdu.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index a5d9a56cdee8..1ed44ed1aaeb 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -7491,7 +7491,12 @@ int smb2_lock(struct ksmbd_work *work) lock_ele = req->locks; ksmbd_debug(SMB, "lock count is %d\n", lock_count); - if (!lock_count) { + /* + * Cap lock_count at 64. The MS-SMB2 spec defines Open.LockSequenceArray + * as exactly 64 entries so 64 is the intended ceiling. No real workload + * comes close to this in a single request. + */ + if (!lock_count || lock_count > 64) { err = -EINVAL; goto out2; } -- cgit v1.2.3 From 804054d19886ac6628883d82410f6ee42a818664 Mon Sep 17 00:00:00 2001 From: DaeMyung Kang Date: Tue, 21 Apr 2026 03:45:11 +0900 Subject: ksmbd: fix durable fd leak on ClientGUID mismatch in durable v2 open ksmbd_lookup_fd_cguid() returns a ksmbd_file with its refcount incremented via ksmbd_fp_get(). parse_durable_handle_context() in the DURABLE_REQ_V2 case properly releases this reference on every path inside the ClientGUID-match branch, either by calling ksmbd_put_durable_fd() or by transferring ownership to dh_info->fp for a successful reconnect. However, when an entry exists in the global file table with the same CreateGuid but a different ClientGUID, the code simply falls through to the new-open path without dropping the reference obtained from ksmbd_lookup_fd_cguid(). Per MS-SMB2 section 3.3.5.9.10 ("Handling the SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 Create Context"), the server MUST locate an Open whose Open.CreateGuid matches the request's CreateGuid AND whose Open.ClientGuid matches the ClientGuid of the connection that received the request. If no such Open is found, the server MUST continue with the normal open execution phase. A CreateGuid hit with a ClientGUID mismatch is therefore the "Open not found" case: proceeding with a new open is correct, but the reference obtained purely as a side effect of the lookup must not be leaked. Repeated requests that hit this mismatch pin global_ft entries, prevent __ksmbd_close_fd() from ever running for the corresponding files, and defeat the durable scavenger, leading to long-lived resource leaks. Release the reference in the mismatch path and clear dh_info->fp so subsequent logic does not mistake a non-matching lookup result for a reconnect target. Fixes: c8efcc786146 ("ksmbd: add support for durable handles v1/v2") Signed-off-by: DaeMyung Kang Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smb2pdu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 1ed44ed1aaeb..a7abef1b208a 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -2844,6 +2844,8 @@ static int parse_durable_handle_context(struct ksmbd_work *work, dh_info->reconnected = true; goto out; } + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; } if ((lc && (lc->req_state & SMB2_LEASE_HANDLE_CACHING_LE)) || -- cgit v1.2.3 From 5d115fa84027e4b999c3d3c7b1294849cf35cdb2 Mon Sep 17 00:00:00 2001 From: DaeMyung Kang Date: Tue, 21 Apr 2026 02:51:25 +0900 Subject: ksmbd: fix CreateOptions sanitization clobbering the whole field smb2_open() attempts to clear conflicting CreateOptions bits (FILE_SEQUENTIAL_ONLY_LE together with FILE_RANDOM_ACCESS_LE, and FILE_NO_COMPRESSION_LE on a directory open), but uses a plain assignment of the bitwise negation of the target flag: req->CreateOptions = ~(FILE_SEQUENTIAL_ONLY_LE); req->CreateOptions = ~(FILE_NO_COMPRESSION_LE); This replaces the entire field with 0xFFFFFFFB / 0xFFFFFFEF rather than clearing a single bit. With the SEQUENTIAL/RANDOM case, the next check for FILE_OPEN_BY_FILE_ID_LE | CREATE_TREE_CONNECTION | FILE_RESERVE_OPFILTER_LE then trivially matches and a legitimate request is rejected with -EOPNOTSUPP. With the NO_COMPRESSION case, every downstream test (FILE_DELETE_ON_CLOSE, etc.) operates on a corrupted CreateOptions value. Use &= ~FLAG to clear only the intended bit in both places. Signed-off-by: DaeMyung Kang Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smb2pdu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index a7abef1b208a..939089304052 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -3057,7 +3057,7 @@ int smb2_open(struct ksmbd_work *work) } else { if (req->CreateOptions & FILE_SEQUENTIAL_ONLY_LE && req->CreateOptions & FILE_RANDOM_ACCESS_LE) - req->CreateOptions = ~(FILE_SEQUENTIAL_ONLY_LE); + req->CreateOptions &= ~FILE_SEQUENTIAL_ONLY_LE; if (req->CreateOptions & (FILE_OPEN_BY_FILE_ID_LE | CREATE_TREE_CONNECTION | @@ -3071,7 +3071,7 @@ int smb2_open(struct ksmbd_work *work) rc = -EINVAL; goto err_out2; } else if (req->CreateOptions & FILE_NO_COMPRESSION_LE) { - req->CreateOptions = ~(FILE_NO_COMPRESSION_LE); + req->CreateOptions &= ~FILE_NO_COMPRESSION_LE; } } } -- cgit v1.2.3 From b0da97c034b6107d14e537e212d4ce8b22109a58 Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Tue, 21 Apr 2026 00:31:47 +0900 Subject: ksmbd: scope conn->binding slowpath to bound sessions only When the binding SESSION_SETUP sets conn->binding = true, the flag stays set after the call so that the global session lookup in ksmbd_session_lookup_all() can find the session, which was not added to conn->sessions. Because the flag is connection-wide, the global lookup path will also resolve any other session by id if asked. Tighten the global lookup so that the returned session must have this connection registered in its channel xarray (sess->ksmbd_chann_list). The channel entry is installed by the existing binding_session path in ntlm_authenticate()/krb5_authenticate() when a SESSION_SETUP completes successfully, so this condition is a strict equivalent of "this connection has been accepted as a channel of this session". Connections that have not bound to a given session cannot reach it via the global table. The existing conn->binding gate for entering the slowpath is preserved so that non-binding connections keep the fast-path-only behavior, and the session->state check is unchanged. Fixes: f5a544e3bab7 ("ksmbd: add support for SMB3 multichannel") Signed-off-by: Hyunwoo Kim Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/mgmt/user_session.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 0dd9e6c976ac..de58aed76cb4 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -548,8 +548,13 @@ struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, struct ksmbd_session *sess; sess = ksmbd_session_lookup(conn, id); - if (!sess && conn->binding) + if (!sess && conn->binding) { sess = ksmbd_session_lookup_slowpath(id); + if (sess && !xa_load(&sess->ksmbd_chann_list, (long)conn)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } if (sess && sess->state != SMB2_SESSION_VALID) { ksmbd_user_session_put(sess); sess = NULL; -- cgit v1.2.3 From 5efb579e0d1ee02b85e3ce2da691c88c93111060 Mon Sep 17 00:00:00 2001 From: Marios Makassikis Date: Wed, 22 Apr 2026 10:14:50 +0900 Subject: smb: server: stop sending fake security descriptors in smb2_get_info_sec, a dummy security descriptor (SD) is returned if the requested information is not supported. the code is currently wrong, as DACL_PROTECTED is set in the type field, but there is no DACL is present. instead of faking a security, report a STATUS_NOT_SUPPORTED error. this seems to fix a "Error 0x80090006: Invalid Signature" on file transfers with Windows 11 clients (25H2, build 26200.8246). capturing traffic shows that the client is sending a GET_INFO/SEC_INFO request, with the additional_info field set to 0x20 (ATTRIBUTE_SECURITY_INFORMATION). Returning an empty SD (with only SELF_RELATIVE set) does not fix the error. Signed-off-by: Marios Makassikis Acked-by: Namjae Jeon Signed-off-by: Steve French --- fs/smb/server/smb2pdu.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 939089304052..21825a69c29a 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -5746,20 +5746,8 @@ static int smb2_get_info_sec(struct ksmbd_work *work, ksmbd_debug(SMB, "Unsupported addition info: 0x%x)\n", addition_info); - pntsd = kzalloc(ALIGN(sizeof(struct smb_ntsd), 8), - KSMBD_DEFAULT_GFP); - if (!pntsd) - return -ENOMEM; - - pntsd->revision = cpu_to_le16(1); - pntsd->type = cpu_to_le16(SELF_RELATIVE | DACL_PROTECTED); - pntsd->osidoffset = 0; - pntsd->gsidoffset = 0; - pntsd->sacloffset = 0; - pntsd->dacloffset = 0; - - secdesclen = sizeof(struct smb_ntsd); - goto iov_pin; + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + return -EINVAL; } if (work->next_smb2_rcv_hdr_off) { @@ -5826,7 +5814,6 @@ release_acl: if (rc) goto err_out; -iov_pin: rsp->OutputBufferLength = cpu_to_le32(secdesclen); rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), rsp, work->response_buf); -- cgit v1.2.3 From 1249c01aa42160e40bc765ba5a3cde751491ff0a Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Sun, 19 Apr 2026 21:20:18 +0200 Subject: smb: smbdirect: move fs/smb/common/smbdirect/ to fs/smb/smbdirect/ This also removes the smbdirect_ prefix from the files. Suggested-by: Linus Torvalds Link: https://lore.kernel.org/linux-cifs/CAHk-=whmue3PVi88K0UZLZO0at22QhQZ-yu+qO2TOKyZpGqecw@mail.gmail.com/ Cc: Steve French Cc: Tom Talpey Cc: Long Li Cc: Linus Torvalds Cc: linux-cifs@vger.kernel.org Cc: samba-technical@lists.samba.org Acked-by: Namjae Jeon Signed-off-by: Stefan Metzmacher Signed-off-by: Steve French --- MAINTAINERS | 2 +- fs/smb/Kconfig | 2 +- fs/smb/Makefile | 1 + fs/smb/client/Kconfig | 2 +- fs/smb/client/smbdirect.c | 2 +- fs/smb/client/smbdirect.h | 2 +- fs/smb/common/Makefile | 1 - fs/smb/common/smbdirect/Kconfig | 9 - fs/smb/common/smbdirect/Makefile | 18 - fs/smb/common/smbdirect/smbdirect.h | 52 - fs/smb/common/smbdirect/smbdirect_accept.c | 857 ---------- fs/smb/common/smbdirect/smbdirect_connect.c | 925 ---------- fs/smb/common/smbdirect/smbdirect_connection.c | 2181 ------------------------ fs/smb/common/smbdirect/smbdirect_debug.c | 88 - fs/smb/common/smbdirect/smbdirect_devices.c | 277 --- fs/smb/common/smbdirect/smbdirect_internal.h | 141 -- fs/smb/common/smbdirect/smbdirect_listen.c | 308 ---- fs/smb/common/smbdirect/smbdirect_main.c | 121 -- fs/smb/common/smbdirect/smbdirect_mr.c | 493 ------ fs/smb/common/smbdirect/smbdirect_pdu.h | 59 - fs/smb/common/smbdirect/smbdirect_public.h | 148 -- fs/smb/common/smbdirect/smbdirect_rw.c | 255 --- fs/smb/common/smbdirect/smbdirect_socket.c | 743 -------- fs/smb/common/smbdirect/smbdirect_socket.h | 832 --------- fs/smb/server/Kconfig | 2 +- fs/smb/server/transport_rdma.c | 2 +- fs/smb/server/transport_rdma.h | 2 +- fs/smb/smbdirect/Kconfig | 9 + fs/smb/smbdirect/Makefile | 18 + fs/smb/smbdirect/accept.c | 857 ++++++++++ fs/smb/smbdirect/connect.c | 925 ++++++++++ fs/smb/smbdirect/connection.c | 2181 ++++++++++++++++++++++++ fs/smb/smbdirect/debug.c | 88 + fs/smb/smbdirect/devices.c | 277 +++ fs/smb/smbdirect/internal.h | 141 ++ fs/smb/smbdirect/listen.c | 308 ++++ fs/smb/smbdirect/main.c | 121 ++ fs/smb/smbdirect/mr.c | 493 ++++++ fs/smb/smbdirect/pdu.h | 59 + fs/smb/smbdirect/public.h | 148 ++ fs/smb/smbdirect/rw.c | 255 +++ fs/smb/smbdirect/smbdirect.h | 52 + fs/smb/smbdirect/socket.c | 743 ++++++++ fs/smb/smbdirect/socket.h | 832 +++++++++ 44 files changed, 7516 insertions(+), 7516 deletions(-) delete mode 100644 fs/smb/common/smbdirect/Kconfig delete mode 100644 fs/smb/common/smbdirect/Makefile delete mode 100644 fs/smb/common/smbdirect/smbdirect.h delete mode 100644 fs/smb/common/smbdirect/smbdirect_accept.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_connect.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_connection.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_debug.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_devices.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_internal.h delete mode 100644 fs/smb/common/smbdirect/smbdirect_listen.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_main.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_mr.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_pdu.h delete mode 100644 fs/smb/common/smbdirect/smbdirect_public.h delete mode 100644 fs/smb/common/smbdirect/smbdirect_rw.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_socket.c delete mode 100644 fs/smb/common/smbdirect/smbdirect_socket.h create mode 100644 fs/smb/smbdirect/Kconfig create mode 100644 fs/smb/smbdirect/Makefile create mode 100644 fs/smb/smbdirect/accept.c create mode 100644 fs/smb/smbdirect/connect.c create mode 100644 fs/smb/smbdirect/connection.c create mode 100644 fs/smb/smbdirect/debug.c create mode 100644 fs/smb/smbdirect/devices.c create mode 100644 fs/smb/smbdirect/internal.h create mode 100644 fs/smb/smbdirect/listen.c create mode 100644 fs/smb/smbdirect/main.c create mode 100644 fs/smb/smbdirect/mr.c create mode 100644 fs/smb/smbdirect/pdu.h create mode 100644 fs/smb/smbdirect/public.h create mode 100644 fs/smb/smbdirect/rw.c create mode 100644 fs/smb/smbdirect/smbdirect.h create mode 100644 fs/smb/smbdirect/socket.c create mode 100644 fs/smb/smbdirect/socket.h diff --git a/MAINTAINERS b/MAINTAINERS index 54b33d43e9fd..839218627f5f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24590,7 +24590,7 @@ L: linux-cifs@vger.kernel.org L: samba-technical@lists.samba.org (moderated for non-subscribers) S: Maintained F: fs/smb/client/smbdirect.* -F: fs/smb/common/smbdirect/ +F: fs/smb/smbdirect/ F: fs/smb/server/transport_rdma.* SMC91x ETHERNET DRIVER diff --git a/fs/smb/Kconfig b/fs/smb/Kconfig index b4b2cfdc2a6b..e549e189ee6a 100644 --- a/fs/smb/Kconfig +++ b/fs/smb/Kconfig @@ -4,7 +4,7 @@ source "fs/smb/client/Kconfig" source "fs/smb/server/Kconfig" -source "fs/smb/common/smbdirect/Kconfig" +source "fs/smb/smbdirect/Kconfig" config SMBFS tristate diff --git a/fs/smb/Makefile b/fs/smb/Makefile index 9a1bf59a1a65..353b1c2eefc4 100644 --- a/fs/smb/Makefile +++ b/fs/smb/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_SMBFS) += common/ +obj-$(CONFIG_SMBDIRECT) += smbdirect/ obj-$(CONFIG_CIFS) += client/ obj-$(CONFIG_SMB_SERVER) += server/ diff --git a/fs/smb/client/Kconfig b/fs/smb/client/Kconfig index 63831242fddf..fed621910876 100644 --- a/fs/smb/client/Kconfig +++ b/fs/smb/client/Kconfig @@ -182,7 +182,7 @@ config CIFS_SMB_DIRECT bool "SMB Direct support" depends on CIFS && INFINIBAND && INFINIBAND_ADDR_TRANS depends on CIFS=m || INFINIBAND=y - select SMB_COMMON_SMBDIRECT + select SMBDIRECT help Enables SMB Direct support for SMB 3.0, 3.02 and 3.1.1. SMB Direct allows transferring SMB packets over RDMA. If unsure, diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index 9e67adcdc7d3..75f9f91a7ec9 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -9,7 +9,7 @@ #include "cifs_debug.h" #include "cifsproto.h" #include "smb2proto.h" -#include "../common/smbdirect/smbdirect_public.h" +#include "../smbdirect/public.h" /* Port numbers for SMBD transport */ #define SMB_PORT 445 diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h index 0017d5b2de44..287ac849213d 100644 --- a/fs/smb/client/smbdirect.h +++ b/fs/smb/client/smbdirect.h @@ -12,7 +12,7 @@ #include "cifsglob.h" -#include "../common/smbdirect/smbdirect.h" +#include "../smbdirect/smbdirect.h" extern int rdma_readwrite_threshold; extern int smbd_max_frmr_depth; diff --git a/fs/smb/common/Makefile b/fs/smb/common/Makefile index e6ee65c31b5d..9e0730a385fb 100644 --- a/fs/smb/common/Makefile +++ b/fs/smb/common/Makefile @@ -4,4 +4,3 @@ # obj-$(CONFIG_SMBFS) += cifs_md4.o -obj-$(CONFIG_SMB_COMMON_SMBDIRECT) += smbdirect/ diff --git a/fs/smb/common/smbdirect/Kconfig b/fs/smb/common/smbdirect/Kconfig deleted file mode 100644 index a46a2e6ec87a..000000000000 --- a/fs/smb/common/smbdirect/Kconfig +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-or-later -# -# smbdirect configuration - -config SMB_COMMON_SMBDIRECT - def_tristate n - depends on INFINIBAND && INFINIBAND_ADDR_TRANS - depends on m || INFINIBAND=y - select SG_POOL diff --git a/fs/smb/common/smbdirect/Makefile b/fs/smb/common/smbdirect/Makefile deleted file mode 100644 index 423f533e1002..000000000000 --- a/fs/smb/common/smbdirect/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-or-later -# -# Makefile for smbdirect support -# - -obj-$(CONFIG_SMB_COMMON_SMBDIRECT) += smbdirect.o - -smbdirect-y := \ - smbdirect_socket.o \ - smbdirect_connection.o \ - smbdirect_mr.o \ - smbdirect_rw.o \ - smbdirect_debug.o \ - smbdirect_connect.o \ - smbdirect_listen.o \ - smbdirect_accept.o \ - smbdirect_devices.o \ - smbdirect_main.o diff --git a/fs/smb/common/smbdirect/smbdirect.h b/fs/smb/common/smbdirect/smbdirect.h deleted file mode 100644 index bbab5f7f7cc9..000000000000 --- a/fs/smb/common/smbdirect/smbdirect.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2025 Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ -#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ - -#include - -/* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */ -struct smbdirect_buffer_descriptor_v1 { - __le64 offset; - __le32 token; - __le32 length; -} __packed; - -/* - * Connection parameters mostly from [MS-SMBD] 3.1.1.1 - * - * These are setup and negotiated at the beginning of a - * connection and remain constant unless explicitly changed. - * - * Some values are important for the upper layer. - */ -struct smbdirect_socket_parameters { - __u64 flags; -#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1) -#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2) - __u32 resolve_addr_timeout_msec; - __u32 resolve_route_timeout_msec; - __u32 rdma_connect_timeout_msec; - __u32 negotiate_timeout_msec; - __u16 initiator_depth; /* limited to U8_MAX */ - __u16 responder_resources; /* limited to U8_MAX */ - __u16 recv_credit_max; - __u16 send_credit_target; - __u32 max_send_size; - __u32 max_fragmented_send_size; - __u32 max_recv_size; - __u32 max_fragmented_recv_size; - __u32 max_read_write_size; - __u32 max_frmr_depth; - __u32 keepalive_interval_msec; - __u32 keepalive_timeout_msec; -} __packed; - -#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \ - SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \ - SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) - -#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ */ diff --git a/fs/smb/common/smbdirect/smbdirect_accept.c b/fs/smb/common/smbdirect/smbdirect_accept.c deleted file mode 100644 index d6d5e6a3f5de..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_accept.c +++ /dev/null @@ -1,857 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (C) 2018, LG Electronics. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" -#include -#include "../../common/smb2status.h" - -static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event); -static int smbdirect_accept_init_params(struct smbdirect_socket *sc); -static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc); -static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc); - -int smbdirect_accept_connect_request(struct smbdirect_socket *sc, - const struct rdma_conn_param *param) -{ - struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_recv_io *recv_io; - u8 peer_initiator_depth; - u8 peer_responder_resources; - struct rdma_conn_param conn_param; - __be32 ird_ord_hdr[2]; - int ret; - - if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED)) - return -EINVAL; - - /* - * First set what the we as server are able to support - */ - sp->initiator_depth = min_t(u8, sp->initiator_depth, - sc->ib.dev->attrs.max_qp_rd_atom); - - peer_initiator_depth = param->initiator_depth; - peer_responder_resources = param->responder_resources; - smbdirect_connection_negotiate_rdma_resources(sc, - peer_initiator_depth, - peer_responder_resources, - param); - - ret = smbdirect_accept_init_params(sc); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_accept_init_params() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto init_params_failed; - } - - ret = smbdirect_connection_create_qp(sc); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_create_qp() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto create_qp_failed; - } - - ret = smbdirect_connection_create_mem_pools(sc); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_create_mem_pools() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto create_mem_failed; - } - - recv_io = smbdirect_connection_get_recv_io(sc); - if (WARN_ON_ONCE(!recv_io)) { - ret = -EINVAL; - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_get_recv_io() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto get_recv_io_failed; - } - recv_io->cqe.done = smbdirect_accept_negotiate_recv_done; - - /* - * Now post the recv_io buffer in order to get - * the negotiate request - */ - sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ; - ret = smbdirect_connection_post_recv_io(recv_io); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_post_recv_io() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto post_recv_io_failed; - } - /* - * From here recv_io is known to the RDMA QP and needs ib_drain_qp and - * smbdirect_accept_negotiate_recv_done to cleanup... - */ - recv_io = NULL; - - /* already checked with SMBDIRECT_CHECK_STATUS_WARN above */ - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); - sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; - - /* - * We already negotiated sp->initiator_depth - * and sp->responder_resources above. - */ - memset(&conn_param, 0, sizeof(conn_param)); - conn_param.initiator_depth = sp->initiator_depth; - conn_param.responder_resources = sp->responder_resources; - - if (sc->rdma.legacy_iwarp) { - ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); - ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); - conn_param.private_data = ird_ord_hdr; - conn_param.private_data_len = sizeof(ird_ord_hdr); - } else { - conn_param.private_data = NULL; - conn_param.private_data_len = 0; - } - conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY; - conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY; - conn_param.flow_control = 0; - - /* explicitly set above */ - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); - sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; - sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED; - sc->rdma.cm_id->event_handler = smbdirect_accept_rdma_event_handler; - ret = rdma_accept(sc->rdma.cm_id, &conn_param); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "rdma_accept() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto rdma_accept_failed; - } - - /* - * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING - * so that the timer will cause a disconnect. - */ - INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work); - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->negotiate_timeout_msec)); - - return 0; - -rdma_accept_failed: - /* - * smbdirect_connection_destroy_qp() calls ib_drain_qp(), - * so that smbdirect_accept_negotiate_recv_done() will - * call smbdirect_connection_put_recv_io() - */ -post_recv_io_failed: - if (recv_io) - smbdirect_connection_put_recv_io(recv_io); -get_recv_io_failed: - smbdirect_connection_destroy_mem_pools(sc); -create_mem_failed: - smbdirect_connection_destroy_qp(sc); -create_qp_failed: -init_params_failed: - return ret; -} - -static int smbdirect_accept_init_params(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - int max_send_sges; - unsigned int maxpages; - - /* need 3 more sge. because a SMB_DIRECT header, SMB2 header, - * SMB2 response could be mapped. - */ - max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3; - if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) { - pr_err("max_send_size %d is too large\n", sp->max_send_size); - return -EINVAL; - } - - /* - * There is only a single batch credit - */ - atomic_set(&sc->send_io.bcredits.count, 1); - - /* - * Initialize the local credits to post - * IB_WR_SEND[_WITH_INV]. - */ - atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target); - - if (sp->max_read_write_size) { - maxpages = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE); - sc->rw_io.credits.max = rdma_rw_mr_factor(sc->ib.dev, - sc->rdma.cm_id->port_num, - maxpages); - sc->rw_io.credits.num_pages = DIV_ROUND_UP(maxpages, sc->rw_io.credits.max); - /* add one extra in order to handle unaligned pages */ - sc->rw_io.credits.max += 1; - } - - sc->recv_io.credits.target = 1; - - atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max); - - return 0; -} - -static void smbdirect_accept_negotiate_recv_work(struct work_struct *work); - -static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_recv_io *recv_io = - container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); - struct smbdirect_socket *sc = recv_io->socket; - unsigned long flags; - - if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - goto error; - } - - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, - "smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - - /* - * This is an internal error! - */ - if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) - goto error; - - /* - * Don't reset timer to the keepalive interval in - * this will be done in smbdirect_accept_direct_negotiate_recv_work. - */ - - ib_dma_sync_single_for_cpu(sc->ib.dev, - recv_io->sge.addr, - recv_io->sge.length, - DMA_FROM_DEVICE); - - /* - * Only remember recv_io if it has enough bytes, - * this gives smbdirect_accept_negotiate_recv_work enough - * information in order to disconnect if it was not - * valid. - */ - sc->recv_io.reassembly.full_packet_received = true; - if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req)) - smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0); - else - smbdirect_connection_put_recv_io(recv_io); - - /* - * Some drivers (at least mlx5_ib and irdma) might post a - * recv completion before RDMA_CM_EVENT_ESTABLISHED, - * we need to adjust our expectation in that case. - * - * So we defer further processing of the negotiation - * to smbdirect_accept_negotiate_recv_work(). - * - * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED - * we queue the work directly otherwise - * smbdirect_accept_rdma_event_handler() will do it, when - * RDMA_CM_EVENT_ESTABLISHED arrived. - */ - spin_lock_irqsave(&sc->connect.lock, flags); - if (!sc->first_error) { - INIT_WORK(&sc->connect.work, smbdirect_accept_negotiate_recv_work); - if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) - queue_work(sc->workqueues.accept, &sc->connect.work); - } - spin_unlock_irqrestore(&sc->connect.lock, flags); - - return; - -error: - /* - * recv_io.posted.refill_work is still disabled, - * so smbdirect_connection_put_recv_io() won't - * start it. - */ - smbdirect_connection_put_recv_io(recv_io); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); -} - -static void smbdirect_accept_negotiate_recv_work(struct work_struct *work) -{ - struct smbdirect_socket *sc = - container_of(work, struct smbdirect_socket, connect.work); - struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_recv_io *recv_io; - struct smbdirect_negotiate_req *nreq; - unsigned long flags; - u16 min_version; - u16 max_version; - u16 credits_requested; - u32 preferred_send_size; - u32 max_receive_size; - u32 max_fragmented_size; - u32 ntstatus; - - if (sc->first_error) - return; - - /* - * make sure we won't start again... - */ - disable_work(work); - - /* - * Reset timer to the keepalive interval in - * order to trigger our next keepalive message. - */ - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->keepalive_interval_msec)); - - /* - * If smbdirect_accept_negotiate_recv_done() detected an - * invalid request we want to disconnect. - */ - recv_io = smbdirect_connection_reassembly_first_recv_io(sc); - if (!recv_io) { - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); - sc->recv_io.reassembly.queue_length--; - list_del(&recv_io->list); - spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); - smbdirect_connection_put_recv_io(recv_io); - - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) - return; - sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; - - /* - * Note recv_io is already part of the free list, - * as we just called smbdirect_connection_put_recv_io(), - * but it won't be reused before we call - * smbdirect_connection_recv_io_refill() below. - */ - - nreq = (struct smbdirect_negotiate_req *)recv_io->packet; - min_version = le16_to_cpu(nreq->min_version); - max_version = le16_to_cpu(nreq->max_version); - credits_requested = le16_to_cpu(nreq->credits_requested); - preferred_send_size = le32_to_cpu(nreq->preferred_send_size); - max_receive_size = le32_to_cpu(nreq->max_receive_size); - max_fragmented_size = le32_to_cpu(nreq->max_fragmented_size); - - smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, - "ReqIn: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n", - "MinVersion=0x", - le16_to_cpu(nreq->min_version), - "MaxVersion=0x", - le16_to_cpu(nreq->max_version), - "CreditsRequested=", - le16_to_cpu(nreq->credits_requested), - "PreferredSendSize=", - le32_to_cpu(nreq->preferred_send_size), - "MaxRecvSize=", - le32_to_cpu(nreq->max_receive_size), - "MaxFragmentedSize=", - le32_to_cpu(nreq->max_fragmented_size)); - - if (!(min_version <= SMBDIRECT_V1 && max_version >= SMBDIRECT_V1)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: min_version=0x%x max_version=0x%x\n", - min_version, max_version); - ntstatus = le32_to_cpu(STATUS_NOT_SUPPORTED); - goto not_supported; - } - - if (credits_requested == 0) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: credits_requested == 0\n"); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: max_receive_size=%u < %u\n", - max_receive_size, - SMBDIRECT_MIN_RECEIVE_SIZE); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: max_fragmented_size=%u < %u\n", - max_fragmented_size, - SMBDIRECT_MIN_FRAGMENTED_SIZE); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - /* - * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used. - */ - sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size); - sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE); - - /* - * The maximum fragmented upper-layer payload receive size supported - * - * Assume max_payload_per_credit is - * smb_direct_receive_credit_max - 24 = 1340 - * - * The maximum number would be - * smb_direct_receive_credit_max * max_payload_per_credit - * - * 1340 * 255 = 341700 (0x536C4) - * - * The minimum value from the spec is 131072 (0x20000) - * - * For now we use the logic we used in ksmbd before: - * (1364 * 255) / 2 = 173910 (0x2A756) - * - * We need to adjust this here in case the peer - * lowered sp->max_recv_size. - * - * TODO: instead of adjusting max_fragmented_recv_size - * we should adjust the number of available buffers, - * but for now we keep the logic as it was used - * in ksmbd before. - */ - sp->max_fragmented_recv_size = (sp->recv_credit_max * sp->max_recv_size) / 2; - - /* - * We take the value from the peer, which is checked to be higher than 0, - * but we limit it to the max value we support in order to have - * the main logic simpler. - */ - sc->recv_io.credits.target = credits_requested; - sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, - sp->recv_credit_max); - - /* - * Note nreq->max_receive_size was already checked against - * SMBDIRECT_MIN_RECEIVE_SIZE above. - */ - sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size); - - /* - * Note nreq->max_fragmented_size was already checked against - * SMBDIRECT_MIN_FRAGMENTED_SIZE above. - */ - sp->max_fragmented_send_size = max_fragmented_size; - - if (sc->accept.listener) { - struct smbdirect_socket *lsc = sc->accept.listener; - unsigned long flags; - - spin_lock_irqsave(&lsc->listen.lock, flags); - list_del(&sc->accept.list); - list_add_tail(&sc->accept.list, &lsc->listen.ready); - wake_up(&lsc->listen.wait_queue); - spin_unlock_irqrestore(&lsc->listen.lock, flags); - - /* - * smbdirect_socket_accept() will call - * smbdirect_accept_negotiate_finish(nsc, 0); - * - * So that we don't send the negotiation - * response that grants credits to the peer - * before the socket is accepted by the - * application. - */ - return; - } - - ntstatus = le32_to_cpu(STATUS_SUCCESS); - -not_supported: - smbdirect_accept_negotiate_finish(sc, ntstatus); -} - -void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_recv_io *recv_io; - struct smbdirect_send_io *send_io; - struct smbdirect_negotiate_resp *nrep; - int posted; - u16 new_credits; - int ret; - - if (ntstatus) - goto not_supported; - - /* - * Prepare for receiving data_transfer messages - */ - sc->recv_io.reassembly.full_packet_received = true; - sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; - list_for_each_entry(recv_io, &sc->recv_io.free.list, list) - recv_io->cqe.done = smbdirect_connection_recv_io_done; - recv_io = NULL; - - /* - * We should at least post 1 smbdirect_recv_io! - */ - posted = smbdirect_connection_recv_io_refill(sc); - if (posted < 1) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_recv_io_refill() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(posted)); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - /* - * The response will grant credits for all posted - * smbdirect_recv_io messages. - */ - new_credits = smbdirect_connection_grant_recv_credits(sc); - -not_supported: - send_io = smbdirect_connection_alloc_send_io(sc); - if (IS_ERR(send_io)) { - ret = PTR_ERR(send_io); - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_alloc_send_io() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_socket_schedule_cleanup(sc, ret); - return; - } - send_io->cqe.done = smbdirect_accept_negotiate_send_done; - - nrep = (struct smbdirect_negotiate_resp *)send_io->packet; - nrep->min_version = cpu_to_le16(SMBDIRECT_V1); - nrep->max_version = cpu_to_le16(SMBDIRECT_V1); - if (ntstatus == 0) { - nrep->negotiated_version = cpu_to_le16(SMBDIRECT_V1); - nrep->reserved = 0; - nrep->credits_requested = cpu_to_le16(sp->send_credit_target); - nrep->credits_granted = cpu_to_le16(new_credits); - nrep->status = cpu_to_le32(ntstatus); - nrep->max_readwrite_size = cpu_to_le32(sp->max_read_write_size); - nrep->preferred_send_size = cpu_to_le32(sp->max_send_size); - nrep->max_receive_size = cpu_to_le32(sp->max_recv_size); - nrep->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); - } else { - nrep->negotiated_version = 0; - nrep->reserved = 0; - nrep->credits_requested = 0; - nrep->credits_granted = 0; - nrep->status = cpu_to_le32(ntstatus); - nrep->max_readwrite_size = 0; - nrep->preferred_send_size = 0; - nrep->max_receive_size = 0; - nrep->max_fragmented_size = 0; - } - - smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, - "RepOut: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n", - "MinVersion=0x", - le16_to_cpu(nrep->min_version), - "MaxVersion=0x", - le16_to_cpu(nrep->max_version), - "NegotiatedVersion=0x", - le16_to_cpu(nrep->negotiated_version), - "CreditsRequested=", - le16_to_cpu(nrep->credits_requested), - "CreditsGranted=", - le16_to_cpu(nrep->credits_granted), - "Status=0x", - le32_to_cpu(nrep->status), - "MaxReadWriteSize=", - le32_to_cpu(nrep->max_readwrite_size), - "PreferredSendSize=", - le32_to_cpu(nrep->preferred_send_size), - "MaxRecvSize=", - le32_to_cpu(nrep->max_receive_size), - "MaxFragmentedSize=", - le32_to_cpu(nrep->max_fragmented_size)); - - send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev, - nrep, - sizeof(*nrep), - DMA_TO_DEVICE); - ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "ib_dma_mapping_error() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_connection_free_send_io(send_io); - smbdirect_socket_schedule_cleanup(sc, ret); - return; - } - - send_io->sge[0].length = sizeof(*nrep); - send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey; - send_io->num_sge = 1; - - ib_dma_sync_single_for_device(sc->ib.dev, - send_io->sge[0].addr, - send_io->sge[0].length, - DMA_TO_DEVICE); - - send_io->wr.next = NULL; - send_io->wr.wr_cqe = &send_io->cqe; - send_io->wr.sg_list = send_io->sge; - send_io->wr.num_sge = send_io->num_sge; - send_io->wr.opcode = IB_WR_SEND; - send_io->wr.send_flags = IB_SEND_SIGNALED; - - ret = smbdirect_connection_post_send_wr(sc, &send_io->wr); - if (ret) { - /* if we reach here, post send failed */ - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_post_send_wr() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - /* - * Note smbdirect_connection_free_send_io() - * does ib_dma_unmap_page() - */ - smbdirect_connection_free_send_io(send_io); - smbdirect_socket_schedule_cleanup(sc, ret); - return; - } - - /* - * smbdirect_accept_negotiate_send_done - * will do all remaining work... - */ -} - -static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_send_io *send_io = - container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); - struct smbdirect_socket *sc = send_io->socket; - struct smbdirect_negotiate_resp *nrep; - u32 ntstatus; - - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, - "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - - nrep = (struct smbdirect_negotiate_resp *)send_io->packet; - ntstatus = le32_to_cpu(nrep->status); - - /* Note this frees wc->wr_cqe, but not wc */ - smbdirect_connection_free_send_io(send_io); - atomic_dec(&sc->send_io.pending.count); - - if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - /* - * If we send a smbdirect_negotiate_resp without NT_STATUS_OK (0) - * we need to disconnect now. - * - * Otherwise smbdirect_connection_negotiation_done() - * will setup all required things and wake up - * the waiter. - */ - if (ntstatus) - smbdirect_socket_schedule_cleanup(sc, -EOPNOTSUPP); - else - smbdirect_connection_negotiation_done(sc); -} - -static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event) -{ - struct smbdirect_socket *sc = id->context; - unsigned long flags; - - /* - * cma_cm_event_handler() has - * lockdep_assert_held(&id_priv->handler_mutex); - * - * Mutexes are not allowed in interrupts, - * and we rely on not being in an interrupt here, - * as we might sleep. - * - * We didn't timeout so we cancel our idle timer, - * it will be scheduled again if needed. - */ - WARN_ON_ONCE(in_interrupt()); - - if (event->status || event->event != sc->rdma.expected_event) { - int ret = -ECONNABORTED; - - if (event->event == RDMA_CM_EVENT_REJECTED) - ret = -ECONNREFUSED; - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ret = -ENETDOWN; - if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) - ret = event->status; - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(sc->rdma.expected_event), - rdma_event_msg(event->event), - event->status, - SMBDIRECT_DEBUG_ERR_PTR(ret)); - - smbdirect_socket_schedule_cleanup(sc, ret); - return 0; - } - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "%s (first_error=%1pe) event=%s\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(event->event)); - - if (sc->first_error) - return 0; - - switch (event->event) { - case RDMA_CM_EVENT_ESTABLISHED: - smbdirect_connection_rdma_established(sc); - - /* - * Some drivers (at least mlx5_ib and irdma) might post a - * recv completion before RDMA_CM_EVENT_ESTABLISHED, - * we need to adjust our expectation in that case. - * - * If smbdirect_accept_negotiate_recv_done was called first - * it initialized sc->connect.work only for us to - * start, so that we turned into - * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before - * smbdirect_accept_negotiate_recv_work() runs. - * - * If smbdirect_accept_negotiate_recv_done didn't happen - * yet. sc->connect.work is still be disabled and - * queue_work() is a no-op. - */ - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) - return 0; - sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; - spin_lock_irqsave(&sc->connect.lock, flags); - if (!sc->first_error) - queue_work(sc->workqueues.accept, &sc->connect.work); - spin_unlock_irqrestore(&sc->connect.lock, flags); - - /* - * wait for smbdirect_accept_negotiate_recv_done() - * to get the negotiate request. - */ - return 0; - - default: - break; - } - - /* - * This is an internal error - */ - WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return 0; -} - -static long smbdirect_socket_wait_for_accept(struct smbdirect_socket *lsc, long timeo) -{ - long ret; - - ret = wait_event_interruptible_timeout(lsc->listen.wait_queue, - !list_empty_careful(&lsc->listen.ready) || - lsc->status != SMBDIRECT_SOCKET_LISTENING || - lsc->first_error, - timeo); - if (lsc->status != SMBDIRECT_SOCKET_LISTENING) - return -EINVAL; - if (lsc->first_error) - return lsc->first_error; - if (!ret) - ret = -ETIMEDOUT; - if (ret < 0) - return ret; - - return 0; -} - -struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, - long timeo, - struct proto_accept_arg *arg) -{ - struct smbdirect_socket *nsc; - unsigned long flags; - - if (lsc->status != SMBDIRECT_SOCKET_LISTENING) { - arg->err = -EINVAL; - return NULL; - } - - if (lsc->first_error) { - arg->err = lsc->first_error; - return NULL; - } - - if (list_empty_careful(&lsc->listen.ready)) { - int ret; - - if (timeo == 0) { - arg->err = -EAGAIN; - return NULL; - } - - ret = smbdirect_socket_wait_for_accept(lsc, timeo); - if (ret) { - arg->err = ret; - return NULL; - } - } - - spin_lock_irqsave(&lsc->listen.lock, flags); - nsc = list_first_entry_or_null(&lsc->listen.ready, - struct smbdirect_socket, - accept.list); - if (nsc) { - nsc->accept.listener = NULL; - list_del_init_careful(&nsc->accept.list); - arg->is_empty = list_empty_careful(&lsc->listen.ready); - } - spin_unlock_irqrestore(&lsc->listen.lock, flags); - if (!nsc) { - arg->err = -EAGAIN; - return NULL; - } - - /* - * We did not send the negotiation response - * yet, so we did not grant any credits to the client, - * so it didn't grant any credits to us. - * - * The caller expects a connected socket - * now as there are no credits anyway. - * - * Then we send the negotiation response in - * order to grant credits to the peer. - */ - nsc->status = SMBDIRECT_SOCKET_CONNECTED; - smbdirect_accept_negotiate_finish(nsc, 0); - - return nsc; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_accept); diff --git a/fs/smb/common/smbdirect/smbdirect_connect.c b/fs/smb/common/smbdirect/smbdirect_connect.c deleted file mode 100644 index 2b54f79dba43..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_connect.c +++ /dev/null @@ -1,925 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2012,2016,2017,2025 Stefan Metzmacher - */ - -#include "smbdirect_internal.h" -#include "../../common/smb2status.h" - -static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc); -static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc, - const struct sockaddr *src, - const struct sockaddr *dst); -static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event); -static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc); -static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc); -static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc); - -int smbdirect_connect(struct smbdirect_socket *sc, const struct sockaddr *dst) -{ - const struct sockaddr *src = NULL; - union { - struct sockaddr sa; - struct sockaddr_storage ss; - } src_addr = { - .sa = { - .sa_family = AF_UNSPEC, - }, - }; - int ret; - - if (sc->first_error) - return -ENOTCONN; - - if (sc->status != SMBDIRECT_SOCKET_CREATED) - return -EALREADY; - - if (WARN_ON_ONCE(!sc->rdma.cm_id)) - return -EINVAL; - - src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; - if (src_addr.sa.sa_family != AF_UNSPEC) - src = &src_addr.sa; - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "connect: src: %pISpsfc dst: %pISpsfc\n", - src, dst); - - ret = smbdirect_connect_setup_connection(sc); - if (ret) - return ret; - - ret = smbdirect_connect_resolve_addr(sc, src, dst); - if (ret) - return ret; - - /* - * The rest happens async via smbdirect_connect_rdma_event_handler() - * the caller will decide to wait or not. - */ - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect); - -static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc) -{ - rdma_lock_handler(sc->rdma.cm_id); - sc->rdma.cm_id->event_handler = smbdirect_connect_rdma_event_handler; - rdma_unlock_handler(sc->rdma.cm_id); - - if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED)) - return -EINVAL; - sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED; - - return 0; -} - -static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc, - const struct sockaddr *src, - const struct sockaddr *dst) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct sockaddr *src_addr = NULL; - struct sockaddr *dst_addr = NULL; - int ret; - - src_addr = (struct sockaddr *)src; - if (src_addr && src_addr->sa_family == AF_UNSPEC) - src_addr = NULL; - dst_addr = (struct sockaddr *)dst; - - if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED)) - return -EINVAL; - sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING; - sc->rdma.expected_event = RDMA_CM_EVENT_ADDR_RESOLVED; - ret = rdma_resolve_addr(sc->rdma.cm_id, src_addr, dst_addr, - sp->resolve_addr_timeout_msec); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "rdma_resolve_addr() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - return 0; -} - -static int smbdirect_connect_resolve_route(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - int ret; - - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED)) - return sc->first_error; - sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING; - sc->rdma.expected_event = RDMA_CM_EVENT_ROUTE_RESOLVED; - ret = rdma_resolve_route(sc->rdma.cm_id, sp->resolve_route_timeout_msec); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "rdma_resolve_route() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - return 0; -} - -static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc) -{ - struct smbdirect_socket_parameters *sp = &sc->parameters; - struct rdma_conn_param conn_param; - __be32 ird_ord_hdr[2]; - int ret; - - sc->ib.dev = sc->rdma.cm_id->device; - - if (!smbdirect_frwr_is_supported(&sc->ib.dev->attrs)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "Fast Registration Work Requests (FRWR) is not supported device %.*s\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name); - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", - sc->ib.dev->attrs.device_cap_flags, - sc->ib.dev->attrs.max_fast_reg_page_list_len); - return -EPROTONOSUPPORT; - } - - if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && - !rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num), - &sc->rdma.cm_id->route.addr.src_addr, - &sc->rdma.cm_id->route.addr.dst_addr); - return -EPROTONOSUPPORT; - } - if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW && - !rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num), - &sc->rdma.cm_id->route.addr.src_addr, - &sc->rdma.cm_id->route.addr.dst_addr); - return -EPROTONOSUPPORT; - } - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "rdma connect: device: %.*s local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - &sc->rdma.cm_id->route.addr.src_addr, - &sc->rdma.cm_id->route.addr.dst_addr); - - sp->max_frmr_depth = min_t(u32, sp->max_frmr_depth, - sc->ib.dev->attrs.max_fast_reg_page_list_len); - sc->mr_io.type = IB_MR_TYPE_MEM_REG; - if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) - sc->mr_io.type = IB_MR_TYPE_SG_GAPS; - - sp->responder_resources = min_t(u8, sp->responder_resources, - sc->ib.dev->attrs.max_qp_rd_atom); - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, - "responder_resources=%d\n", - sp->responder_resources); - - ret = smbdirect_connection_create_qp(sc); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_create_qp() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - memset(&conn_param, 0, sizeof(conn_param)); - conn_param.initiator_depth = sp->initiator_depth; - conn_param.responder_resources = sp->responder_resources; - - /* Need to send IRD/ORD in private data for iWARP */ - if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) { - ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); - ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); - conn_param.private_data = ird_ord_hdr; - conn_param.private_data_len = sizeof(ird_ord_hdr); - } else { - conn_param.private_data = NULL; - conn_param.private_data_len = 0; - } - - conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY; - conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY; - conn_param.flow_control = 0; - - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED)) - return sc->first_error; - sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; - sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED; - ret = rdma_connect_locked(sc->rdma.cm_id, &conn_param); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "rdma_connect_locked() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - /* - * start with the rdma connect timeout and SMBDIRECT_KEEPALIVE_PENDING - * so that the timer will cause a disconnect. - */ - INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work); - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->rdma_connect_timeout_msec)); - - return 0; -} - -static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event) -{ - struct smbdirect_socket *sc = id->context; - u8 peer_initiator_depth; - u8 peer_responder_resources; - int ret; - - /* - * cma_cm_event_handler() has - * lockdep_assert_held(&id_priv->handler_mutex); - * - * Mutexes are not allowed in interrupts, - * and we rely on not being in an interrupt here, - * as we might sleep. - * - * We didn't timeout so we cancel our idle timer, - * it will be scheduled again if needed. - */ - WARN_ON_ONCE(in_interrupt()); - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; - cancel_delayed_work_sync(&sc->idle.timer_work); - - if (event->status || event->event != sc->rdma.expected_event) { - int lvl = SMBDIRECT_LOG_ERR; - - ret = -ECONNABORTED; - - if (event->event == RDMA_CM_EVENT_REJECTED) - ret = -ECONNREFUSED; - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ret = -ENETDOWN; - if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) - ret = event->status; - - if (ret == -ENODEV) - lvl = SMBDIRECT_LOG_INFO; - - smbdirect_log_rdma_event(sc, lvl, - "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(sc->rdma.expected_event), - rdma_event_msg(event->event), - event->status, - SMBDIRECT_DEBUG_ERR_PTR(ret)); - - smbdirect_socket_schedule_cleanup_lvl(sc, - lvl, - ret); - return 0; - } - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "%s (first_error=%1pe) event=%s\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(event->event)); - - if (sc->first_error) - return 0; - - switch (event->event) { - case RDMA_CM_EVENT_ADDR_RESOLVED: - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING)) - return 0; - sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED; - - ret = smbdirect_connect_resolve_route(sc); - if (ret) - smbdirect_socket_schedule_cleanup(sc, ret); - return 0; - - case RDMA_CM_EVENT_ROUTE_RESOLVED: - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING)) - return 0; - sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; - - ret = smbdirect_connect_rdma_connect(sc); - if (ret) - smbdirect_socket_schedule_cleanup(sc, ret); - return 0; - - case RDMA_CM_EVENT_ESTABLISHED: - smbdirect_connection_rdma_established(sc); - - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) - return 0; - sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; - - /* - * Here we work around an inconsistency between - * iWarp and other devices (at least rxe and irdma using RoCEv2) - */ - if (rdma_protocol_iwarp(id->device, id->port_num)) { - /* - * iWarp devices report the peer's values - * with the perspective of the peer here. - * Tested with siw and irdma (in iwarp mode) - * We need to change to our perspective here, - * so we need to switch the values. - */ - peer_initiator_depth = event->param.conn.responder_resources; - peer_responder_resources = event->param.conn.initiator_depth; - } else { - /* - * Non iWarp devices report the peer's values - * already changed to our perspective here. - * Tested with rxe and irdma (in roce mode). - */ - peer_initiator_depth = event->param.conn.initiator_depth; - peer_responder_resources = event->param.conn.responder_resources; - } - smbdirect_connection_negotiate_rdma_resources(sc, - peer_initiator_depth, - peer_responder_resources, - &event->param.conn); - - ret = smbdirect_connect_negotiate_start(sc); - if (ret) - smbdirect_socket_schedule_cleanup(sc, ret); - return 0; - - default: - break; - } - - /* - * This is an internal error - */ - WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return 0; -} - -static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_recv_io *recv_io = NULL; - struct smbdirect_send_io *send_io = NULL; - struct smbdirect_negotiate_req *nreq = NULL; - int ret; - - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) - return sc->first_error; - sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; - - ret = smbdirect_connection_create_mem_pools(sc); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_create_mem_pools() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto create_mem_pools_failed; - } - - /* - * There is only a single batch credit - */ - atomic_set(&sc->send_io.bcredits.count, 1); - - /* - * Initialize the local credits to post - * IB_WR_SEND[_WITH_INV]. - */ - atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target); - - recv_io = smbdirect_connection_get_recv_io(sc); - if (WARN_ON_ONCE(!recv_io)) { - ret = -EINVAL; - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_get_recv_io() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto get_recv_io_failed; - } - recv_io->cqe.done = smbdirect_connect_negotiate_recv_done; - - send_io = smbdirect_connection_alloc_send_io(sc); - if (IS_ERR(send_io)) { - ret = PTR_ERR(send_io); - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_alloc_send_io() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto alloc_send_io_failed; - } - send_io->cqe.done = smbdirect_connect_negotiate_send_done; - - nreq = (struct smbdirect_negotiate_req *)send_io->packet; - nreq->min_version = cpu_to_le16(SMBDIRECT_V1); - nreq->max_version = cpu_to_le16(SMBDIRECT_V1); - nreq->reserved = 0; - nreq->credits_requested = cpu_to_le16(sp->send_credit_target); - nreq->preferred_send_size = cpu_to_le32(sp->max_send_size); - nreq->max_receive_size = cpu_to_le32(sp->max_recv_size); - nreq->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); - - smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, - "ReqOut: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n", - "MinVersion=0x", - le16_to_cpu(nreq->min_version), - "MaxVersion=0x", - le16_to_cpu(nreq->max_version), - "CreditsRequested=", - le16_to_cpu(nreq->credits_requested), - "PreferredSendSize=", - le32_to_cpu(nreq->preferred_send_size), - "MaxRecvSize=", - le32_to_cpu(nreq->max_receive_size), - "MaxFragmentedSize=", - le32_to_cpu(nreq->max_fragmented_size)); - - send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev, - nreq, - sizeof(*nreq), - DMA_TO_DEVICE); - ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "ib_dma_mapping_error() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto dma_mapping_failed; - } - - send_io->sge[0].length = sizeof(*nreq); - send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey; - send_io->num_sge = 1; - - ib_dma_sync_single_for_device(sc->ib.dev, - send_io->sge[0].addr, - send_io->sge[0].length, - DMA_TO_DEVICE); - - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, - "sge addr=0x%llx length=%u lkey=0x%x\n", - send_io->sge[0].addr, - send_io->sge[0].length, - send_io->sge[0].lkey); - - /* - * Now post the recv_io buffer in order to get - * the negotiate response - */ - sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP; - ret = smbdirect_connection_post_recv_io(recv_io); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_post_recv_io() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto post_recv_io_failed; - } - - send_io->wr.next = NULL; - send_io->wr.wr_cqe = &send_io->cqe; - send_io->wr.sg_list = send_io->sge; - send_io->wr.num_sge = send_io->num_sge; - send_io->wr.opcode = IB_WR_SEND; - send_io->wr.send_flags = IB_SEND_SIGNALED; - - ret = smbdirect_connection_post_send_wr(sc, &send_io->wr); - if (ret) { - /* if we reach here, post send failed */ - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_post_send_wr() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto post_send_wr_failed; - } - - /* - * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING - * so that the timer will cause a disconnect. - */ - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->negotiate_timeout_msec)); - - return 0; - -post_send_wr_failed: - /* - * ib_dma_unmap_single is called in - * smbdirect_connection_free_send_io() - */ - smbdirect_connection_free_send_io(send_io); - /* - * recv_io is given to the rdma layer, - * we should not put it even on error - * nor call smbdirect_connection_destroy_mem_pools() - * it will be cleaned up during disconnect. - */ - return ret; - -post_recv_io_failed: - /* - * ib_dma_unmap_single is called in - * smbdirect_connection_free_send_io() - */ -dma_mapping_failed: - smbdirect_connection_free_send_io(send_io); - -alloc_send_io_failed: - smbdirect_connection_put_recv_io(recv_io); - -get_recv_io_failed: - smbdirect_connection_destroy_mem_pools(sc); - -create_mem_pools_failed: - return ret; -} - -static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_send_io *send_io = - container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); - struct smbdirect_socket *sc = send_io->socket; - - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, - "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - - /* Note this frees wc->wr_cqe, but not wc */ - smbdirect_connection_free_send_io(send_io); - atomic_dec(&sc->send_io.pending.count); - - if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } -} - -static void smbdirect_connect_negotiate_recv_work(struct work_struct *work); - -static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_recv_io *recv_io = - container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); - struct smbdirect_socket *sc = recv_io->socket; - unsigned long flags; - - if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - goto error; - } - - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, - "smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - - /* - * This is an internal error! - */ - if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REP)) - goto error; - - /* - * Don't reset timer to the keepalive interval in - * this will be done in smbdirect_accept_direct_negotiate_recv_work. - */ - - ib_dma_sync_single_for_cpu(sc->ib.dev, - recv_io->sge.addr, - recv_io->sge.length, - DMA_FROM_DEVICE); - - /* - * Only remember recv_io if it has enough bytes, - * this gives smbdirect_accept_negotiate_recv_work enough - * information in order to disconnect if it was not - * valid. - */ - sc->recv_io.reassembly.full_packet_received = true; - if (wc->byte_len >= sizeof(struct smbdirect_negotiate_resp)) - smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0); - else - smbdirect_connection_put_recv_io(recv_io); - - /* - * We continue via the workqueue as we may have - * complex work that might sleep. - * - * So we defer further processing of the negotiation - * to smbdirect_connect_negotiate_recv_work(). - */ - spin_lock_irqsave(&sc->connect.lock, flags); - if (!sc->first_error) { - INIT_WORK(&sc->connect.work, smbdirect_connect_negotiate_recv_work); - if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) - queue_work(sc->workqueues.connect, &sc->connect.work); - } - spin_unlock_irqrestore(&sc->connect.lock, flags); - - return; - -error: - /* - * recv_io.posted.refill_work is still disabled, - * so smbdirect_connection_put_recv_io() won't - * start it. - */ - smbdirect_connection_put_recv_io(recv_io); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); -} - -static void smbdirect_connect_negotiate_recv_work(struct work_struct *work) -{ - struct smbdirect_socket *sc = - container_of(work, struct smbdirect_socket, connect.work); - struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_recv_io *recv_io; - struct smbdirect_negotiate_resp *nrep; - unsigned long flags; - u16 negotiated_version; - u16 credits_requested; - u16 credits_granted; - u32 status; - u32 max_readwrite_size; - u32 preferred_send_size; - u32 max_receive_size; - u32 max_fragmented_size; - int posted; - int ret; - - if (sc->first_error) - return; - - /* - * make sure we won't start again... - */ - disable_work(work); - - /* - * Reset timer to the keepalive interval in - * order to trigger our next keepalive message. - */ - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->keepalive_interval_msec)); - - /* - * If smbdirect_connect_negotiate_recv_done() detected an - * invalid request we want to disconnect. - */ - recv_io = smbdirect_connection_reassembly_first_recv_io(sc); - if (!recv_io) { - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); - sc->recv_io.reassembly.queue_length--; - list_del(&recv_io->list); - spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); - smbdirect_connection_put_recv_io(recv_io); - - if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)) - return; - - /* - * Note recv_io is already part of the free list, - * as we just called smbdirect_connection_put_recv_io(), - * but it won't be reused before we call - * smbdirect_connection_recv_io_refill() below. - */ - - nrep = (struct smbdirect_negotiate_resp *)recv_io->packet; - negotiated_version = le16_to_cpu(nrep->negotiated_version); - credits_requested = le16_to_cpu(nrep->credits_requested); - credits_granted = le16_to_cpu(nrep->credits_granted); - status = le32_to_cpu(nrep->status); - max_readwrite_size = le32_to_cpu(nrep->max_readwrite_size); - preferred_send_size = le32_to_cpu(nrep->preferred_send_size); - max_receive_size = le32_to_cpu(nrep->max_receive_size); - max_fragmented_size = le32_to_cpu(nrep->max_fragmented_size); - - smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, - "RepIn: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n", - "MinVersion=0x", - le16_to_cpu(nrep->min_version), - "MaxVersion=0x", - le16_to_cpu(nrep->max_version), - "NegotiatedVersion=0x", - le16_to_cpu(nrep->negotiated_version), - "CreditsRequested=", - le16_to_cpu(nrep->credits_requested), - "CreditsGranted=", - le16_to_cpu(nrep->credits_granted), - "Status=0x", - le32_to_cpu(nrep->status), - "MaxReadWriteSize=", - le32_to_cpu(nrep->max_readwrite_size), - "PreferredSendSize=", - le32_to_cpu(nrep->preferred_send_size), - "MaxRecvSize=", - le32_to_cpu(nrep->max_receive_size), - "MaxFragmentedSize=", - le32_to_cpu(nrep->max_fragmented_size)); - - if (negotiated_version != SMBDIRECT_V1) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: negotiated_version=0x%x\n", - negotiated_version); - smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED); - return; - } - - if (status != le32_to_cpu(STATUS_SUCCESS)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: status=0x%x != 0x0\n", - status); - smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED); - return; - } - - if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: max_receive_size=%u < %u\n", - max_receive_size, - SMBDIRECT_MIN_RECEIVE_SIZE); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: max_fragmented_size=%u < %u\n", - max_fragmented_size, - SMBDIRECT_MIN_FRAGMENTED_SIZE); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - if (credits_granted == 0) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: credits_granted == 0\n"); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - if (credits_requested == 0) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: credits_requested == 0\n"); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - if (preferred_send_size > sp->max_recv_size) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: preferred_send_size=%u > max_recv_size=%u\n", - preferred_send_size, - sp->max_recv_size); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - /* - * We take the value from the peer, which is checked to be higher than 0, - * but we limit it to the max value we support in order to have - * the main logic simpler. - */ - sc->recv_io.credits.target = credits_requested; - sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, - sp->recv_credit_max); - - /* - * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used. - */ - sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size); - sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE); - - /* - * We already sent our sp->max_fragmented_recv_size - * to the peer, so we can't lower it here any more. - * - * TODO: but if the peer lowered sp->max_recv_size - * we will have to adjust our number of buffers. - * - * But for now we keep it as the cifs.ko code - * worked before. - */ - - /* - * Note nrep->max_receive_size was already checked against - * SMBDIRECT_MIN_RECEIVE_SIZE above. - */ - sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size); - - /* - * Make sure the resulting max_frmr_depth is at least 1, - * which means max_read_write_size needs to be at least PAGE_SIZE. - */ - sp->max_read_write_size = min_t(u32, sp->max_frmr_depth * PAGE_SIZE, - max_readwrite_size); - if (sp->max_read_write_size < PAGE_SIZE) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: max_readwrite_size=%u < PAGE_SIZE(%lu)\n", - max_readwrite_size, - PAGE_SIZE); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; - - /* - * Note nrep->credits_granted was already checked against 0 above. - */ - atomic_set(&sc->send_io.credits.count, credits_granted); - - /* - * Note nrep->max_fragmented_size was already checked against - * SMBDIRECT_MIN_FRAGMENTED_SIZE above. - */ - sp->max_fragmented_send_size = max_fragmented_size; - - ret = smbdirect_connection_create_mr_list(sc); - if (ret) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_create_mr_list() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_socket_schedule_cleanup(sc, ret); - return; - } - - /* - * Prepare for receiving data_transfer messages - */ - sc->recv_io.reassembly.full_packet_received = true; - sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; - list_for_each_entry(recv_io, &sc->recv_io.free.list, list) - recv_io->cqe.done = smbdirect_connection_recv_io_done; - recv_io = NULL; - - /* - * We should at least post 1 smbdirect_recv_io! - */ - posted = smbdirect_connection_recv_io_refill(sc); - if (posted < 1) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_recv_io_refill() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - /* - * smbdirect_connection_negotiation_done() - * will setup all required things and wake up - * the waiter. - */ - smbdirect_connection_negotiation_done(sc); -} - -int smbdirect_connect_sync(struct smbdirect_socket *sc, - const struct sockaddr *dst) -{ - int ret; - - ret = smbdirect_connect(sc, dst); - if (ret) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connect(%pISpsfc) failed %1pe\n", - dst, SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - ret = smbdirect_connection_wait_for_connected(sc); - if (ret) { - int lvl = SMBDIRECT_LOG_ERR; - - if (ret == -ENODEV) - lvl = SMBDIRECT_LOG_INFO; - - smbdirect_log_rdma_event(sc, lvl, - "wait for smbdirect_connect(%pISpsfc) failed %1pe\n", - dst, SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect_sync); diff --git a/fs/smb/common/smbdirect/smbdirect_connection.c b/fs/smb/common/smbdirect/smbdirect_connection.c deleted file mode 100644 index 7e4921b9538c..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_connection.c +++ /dev/null @@ -1,2181 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" -#include - -struct smbdirect_map_sges { - struct ib_sge *sge; - size_t num_sge; - size_t max_sge; - struct ib_device *device; - u32 local_dma_lkey; - enum dma_data_direction direction; -}; - -static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, - struct smbdirect_map_sges *state); - -static void smbdirect_connection_recv_io_refill_work(struct work_struct *work); -static void smbdirect_connection_send_immediate_work(struct work_struct *work); - -static void smbdirect_connection_qp_event_handler(struct ib_event *event, void *context) -{ - struct smbdirect_socket *sc = context; - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "%s on device %.*s socket %p (cm_id=%p) status %s first_error %1pe\n", - ib_event_msg(event->event), - IB_DEVICE_NAME_MAX, - event->device->name, - sc, sc->rdma.cm_id, - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - - switch (event->event) { - case IB_EVENT_CQ_ERR: - case IB_EVENT_QP_FATAL: - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - break; - - default: - break; - } -} - -static int smbdirect_connection_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event) -{ - struct smbdirect_socket *sc = id->context; - int ret = -ECONNRESET; - - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ret = -ENETDOWN; - if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) - ret = event->status; - - /* - * cma_cm_event_handler() has - * lockdep_assert_held(&id_priv->handler_mutex); - * - * Mutexes are not allowed in interrupts, - * and we rely on not being in an interrupt here. - */ - WARN_ON_ONCE(in_interrupt()); - - if (event->event != sc->rdma.expected_event) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(sc->rdma.expected_event), - rdma_event_msg(event->event), - event->status, - SMBDIRECT_DEBUG_ERR_PTR(ret)); - - /* - * If we get RDMA_CM_EVENT_DEVICE_REMOVAL, - * we should change to SMBDIRECT_SOCKET_DISCONNECTED, - * so that rdma_disconnect() is avoided later via - * smbdirect_socket_schedule_cleanup[_status]() => - * smbdirect_socket_cleanup_work(). - * - * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING, - * but never ever get RDMA_CM_EVENT_DISCONNECTED and - * never reach SMBDIRECT_SOCKET_DISCONNECTED. - */ - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - smbdirect_socket_schedule_cleanup_status(sc, - SMBDIRECT_LOG_ERR, - ret, - SMBDIRECT_SOCKET_DISCONNECTED); - else - smbdirect_socket_schedule_cleanup(sc, ret); - if (sc->ib.qp) - ib_drain_qp(sc->ib.qp); - return 0; - } - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "%s (first_error=%1pe) event=%s\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(event->event)); - - switch (event->event) { - case RDMA_CM_EVENT_DISCONNECTED: - /* - * We need to change to SMBDIRECT_SOCKET_DISCONNECTED, - * so that rdma_disconnect() is avoided later via - * smbdirect_socket_schedule_cleanup_status() => - * smbdirect_socket_cleanup_work(). - * - * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING, - * but never ever get RDMA_CM_EVENT_DISCONNECTED and - * never reach SMBDIRECT_SOCKET_DISCONNECTED. - * - * This is also a normal disconnect so - * SMBDIRECT_LOG_INFO should be good enough - * and avoids spamming the default logs. - */ - smbdirect_socket_schedule_cleanup_status(sc, - SMBDIRECT_LOG_INFO, - ret, - SMBDIRECT_SOCKET_DISCONNECTED); - if (sc->ib.qp) - ib_drain_qp(sc->ib.qp); - return 0; - - default: - break; - } - - /* - * This is an internal error, should be handled above via - * event->event != sc->rdma.expected_event already. - */ - WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_DISCONNECTED); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return 0; -} - -void smbdirect_connection_rdma_established(struct smbdirect_socket *sc) -{ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "rdma established: device: %.*s local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - &sc->rdma.cm_id->route.addr.src_addr, - &sc->rdma.cm_id->route.addr.dst_addr); - - sc->rdma.cm_id->event_handler = smbdirect_connection_rdma_event_handler; - sc->rdma.expected_event = RDMA_CM_EVENT_DISCONNECTED; -} - -void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc) -{ - if (unlikely(sc->first_error)) - return; - - if (sc->status == SMBDIRECT_SOCKET_CONNECTED) - /* - * This is the accept case where - * smbdirect_socket_accept() already sets - * SMBDIRECT_SOCKET_CONNECTED - */ - goto done; - - if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) { - /* - * Something went wrong... - */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "status=%s first_error=%1pe local: %pISpsfc remote: %pISpsfc\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - &sc->rdma.cm_id->route.addr.src_addr, - &sc->rdma.cm_id->route.addr.dst_addr); - return; - } - - /* - * We are done, so we can wake up the waiter. - */ - WARN_ONCE(sc->status == SMBDIRECT_SOCKET_CONNECTED, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - sc->status = SMBDIRECT_SOCKET_CONNECTED; - - /* - * We need to setup the refill and send immediate work - * in order to get a working connection. - */ -done: - INIT_WORK(&sc->recv_io.posted.refill_work, smbdirect_connection_recv_io_refill_work); - INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work); - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "negotiated: local: %pISpsfc remote: %pISpsfc\n", - &sc->rdma.cm_id->route.addr.src_addr, - &sc->rdma.cm_id->route.addr.dst_addr); - - wake_up(&sc->status_wait); -} - -static u32 smbdirect_rdma_rw_send_wrs(struct ib_device *dev, - const struct ib_qp_init_attr *attr) -{ - /* - * This could be split out of rdma_rw_init_qp() - * and be a helper function next to rdma_rw_mr_factor() - * - * We can't check unlikely(rdma_rw_force_mr) here, - * but that is most likely 0 anyway. - */ - u32 factor; - - WARN_ON_ONCE(attr->port_num == 0); - - /* - * Each context needs at least one RDMA READ or WRITE WR. - * - * For some hardware we might need more, eventually we should ask the - * HCA driver for a multiplier here. - */ - factor = 1; - - /* - * If the device needs MRs to perform RDMA READ or WRITE operations, - * we'll need two additional MRs for the registrations and the - * invalidation. - */ - if (rdma_protocol_iwarp(dev, attr->port_num) || dev->attrs.max_sgl_rd) - factor += 2; /* inv + reg */ - - return factor * attr->cap.max_rdma_ctxs; -} - -int smbdirect_connection_create_qp(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct ib_qp_init_attr qp_attr; - struct ib_qp_cap qp_cap; - u32 rdma_send_wr; - u32 max_send_wr; - int ret; - - /* - * Note that {rdma,ib}_create_qp() will call - * rdma_rw_init_qp() if max_rdma_ctxs is not 0. - * It will adjust max_send_wr to the required - * number of additional WRs for the RDMA RW operations. - * It will cap max_send_wr to the device limit. - * - * We use allocate sp->responder_resources * 2 MRs - * and each MR needs WRs for REG and INV, so - * we use '* 4'. - * - * +1 for ib_drain_qp() - */ - memset(&qp_cap, 0, sizeof(qp_cap)); - qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1; - qp_cap.max_recv_wr = sp->recv_credit_max + 1; - qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; - qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; - qp_cap.max_inline_data = 0; - qp_cap.max_rdma_ctxs = sc->rw_io.credits.max; - - /* - * Find out the number of max_send_wr - * after rdma_rw_init_qp() adjusted it. - * - * We only do it on a temporary variable, - * as rdma_create_qp() will trigger - * rdma_rw_init_qp() again. - */ - memset(&qp_attr, 0, sizeof(qp_attr)); - qp_attr.cap = qp_cap; - qp_attr.port_num = sc->rdma.cm_id->port_num; - rdma_send_wr = smbdirect_rdma_rw_send_wrs(sc->ib.dev, &qp_attr); - max_send_wr = qp_cap.max_send_wr + rdma_send_wr; - - if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe || - qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) { - pr_err("Possible CQE overrun: max_send_wr %d\n", - qp_cap.max_send_wr); - pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - sc->ib.dev->attrs.max_cqe, - sc->ib.dev->attrs.max_qp_wr); - pr_err("consider lowering send_credit_target = %d\n", - sp->send_credit_target); - return -EINVAL; - } - - if (qp_cap.max_rdma_ctxs && - (max_send_wr >= sc->ib.dev->attrs.max_cqe || - max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) { - pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n", - rdma_send_wr, qp_cap.max_send_wr, max_send_wr); - pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - sc->ib.dev->attrs.max_cqe, - sc->ib.dev->attrs.max_qp_wr); - pr_err("consider lowering send_credit_target = %d, max_rdma_ctxs = %d\n", - sp->send_credit_target, qp_cap.max_rdma_ctxs); - return -EINVAL; - } - - if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe || - qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) { - pr_err("Possible CQE overrun: max_recv_wr %d\n", - qp_cap.max_recv_wr); - pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - sc->ib.dev->attrs.max_cqe, - sc->ib.dev->attrs.max_qp_wr); - pr_err("consider lowering receive_credit_max = %d\n", - sp->recv_credit_max); - return -EINVAL; - } - - if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge || - qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) { - pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", - IB_DEVICE_NAME_MAX, - sc->ib.dev->name, - sc->ib.dev->attrs.max_send_sge, - sc->ib.dev->attrs.max_recv_sge); - return -EINVAL; - } - - sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); - if (IS_ERR(sc->ib.pd)) { - pr_err("Can't create RDMA PD: %1pe\n", sc->ib.pd); - ret = PTR_ERR(sc->ib.pd); - sc->ib.pd = NULL; - return ret; - } - - sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc, - max_send_wr, - sc->ib.poll_ctx); - if (IS_ERR(sc->ib.send_cq)) { - pr_err("Can't create RDMA send CQ: %1pe\n", sc->ib.send_cq); - ret = PTR_ERR(sc->ib.send_cq); - sc->ib.send_cq = NULL; - goto err; - } - - sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc, - qp_cap.max_recv_wr, - sc->ib.poll_ctx); - if (IS_ERR(sc->ib.recv_cq)) { - pr_err("Can't create RDMA recv CQ: %1pe\n", sc->ib.recv_cq); - ret = PTR_ERR(sc->ib.recv_cq); - sc->ib.recv_cq = NULL; - goto err; - } - - /* - * We reset completely here! - * As the above use was just temporary - * to calc max_send_wr and rdma_send_wr. - * - * rdma_create_qp() will trigger rdma_rw_init_qp() - * again if max_rdma_ctxs is not 0. - */ - memset(&qp_attr, 0, sizeof(qp_attr)); - qp_attr.event_handler = smbdirect_connection_qp_event_handler; - qp_attr.qp_context = sc; - qp_attr.cap = qp_cap; - qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; - qp_attr.qp_type = IB_QPT_RC; - qp_attr.send_cq = sc->ib.send_cq; - qp_attr.recv_cq = sc->ib.recv_cq; - qp_attr.port_num = ~0; - - ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); - if (ret) { - pr_err("Can't create RDMA QP: %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto err; - } - sc->ib.qp = sc->rdma.cm_id->qp; - - return 0; -err: - smbdirect_connection_destroy_qp(sc); - return ret; -} - -void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc) -{ - if (sc->ib.qp) { - ib_drain_qp(sc->ib.qp); - sc->ib.qp = NULL; - rdma_destroy_qp(sc->rdma.cm_id); - } - if (sc->ib.recv_cq) { - ib_destroy_cq(sc->ib.recv_cq); - sc->ib.recv_cq = NULL; - } - if (sc->ib.send_cq) { - ib_destroy_cq(sc->ib.send_cq); - sc->ib.send_cq = NULL; - } - if (sc->ib.pd) { - ib_dealloc_pd(sc->ib.pd); - sc->ib.pd = NULL; - } -} - -int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - char name[80]; - size_t i; - - /* - * We use sizeof(struct smbdirect_negotiate_resp) for the - * payload size as it is larger as - * sizeof(struct smbdirect_data_transfer). - * - * This will fit client and server usage for now. - */ - snprintf(name, sizeof(name), "smbdirect_send_io_cache_%p", sc); - struct kmem_cache_args send_io_args = { - .align = __alignof__(struct smbdirect_send_io), - }; - sc->send_io.mem.cache = kmem_cache_create(name, - sizeof(struct smbdirect_send_io) + - sizeof(struct smbdirect_negotiate_resp), - &send_io_args, - SLAB_HWCACHE_ALIGN); - if (!sc->send_io.mem.cache) - goto err; - - sc->send_io.mem.pool = mempool_create_slab_pool(sp->send_credit_target, - sc->send_io.mem.cache); - if (!sc->send_io.mem.pool) - goto err; - - /* - * A payload size of sp->max_recv_size should fit - * any message. - * - * For smbdirect_data_transfer messages the whole - * buffer might be exposed to userspace - * (currently on the client side...) - * The documentation says data_offset = 0 would be - * strange but valid. - */ - snprintf(name, sizeof(name), "smbdirect_recv_io_cache_%p", sc); - struct kmem_cache_args recv_io_args = { - .align = __alignof__(struct smbdirect_recv_io), - .useroffset = sizeof(struct smbdirect_recv_io), - .usersize = sp->max_recv_size, - }; - sc->recv_io.mem.cache = kmem_cache_create(name, - sizeof(struct smbdirect_recv_io) + - sp->max_recv_size, - &recv_io_args, - SLAB_HWCACHE_ALIGN); - if (!sc->recv_io.mem.cache) - goto err; - - sc->recv_io.mem.pool = mempool_create_slab_pool(sp->recv_credit_max, - sc->recv_io.mem.cache); - if (!sc->recv_io.mem.pool) - goto err; - - for (i = 0; i < sp->recv_credit_max; i++) { - struct smbdirect_recv_io *recv_io; - - recv_io = mempool_alloc(sc->recv_io.mem.pool, - sc->recv_io.mem.gfp_mask); - if (!recv_io) - goto err; - recv_io->socket = sc; - recv_io->sge.length = 0; - list_add_tail(&recv_io->list, &sc->recv_io.free.list); - } - - return 0; -err: - smbdirect_connection_destroy_mem_pools(sc); - return -ENOMEM; -} - -void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc) -{ - struct smbdirect_recv_io *recv_io, *next_io; - - list_for_each_entry_safe(recv_io, next_io, &sc->recv_io.free.list, list) { - list_del(&recv_io->list); - mempool_free(recv_io, sc->recv_io.mem.pool); - } - - /* - * Note mempool_destroy() and kmem_cache_destroy() - * work fine with a NULL pointer - */ - - mempool_destroy(sc->recv_io.mem.pool); - sc->recv_io.mem.pool = NULL; - - kmem_cache_destroy(sc->recv_io.mem.cache); - sc->recv_io.mem.cache = NULL; - - mempool_destroy(sc->send_io.mem.pool); - sc->send_io.mem.pool = NULL; - - kmem_cache_destroy(sc->send_io.mem.cache); - sc->send_io.mem.cache = NULL; -} - -struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc) -{ - struct smbdirect_send_io *msg; - - msg = mempool_alloc(sc->send_io.mem.pool, sc->send_io.mem.gfp_mask); - if (!msg) - return ERR_PTR(-ENOMEM); - msg->socket = sc; - INIT_LIST_HEAD(&msg->sibling_list); - msg->num_sge = 0; - - return msg; -} - -void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg) -{ - struct smbdirect_socket *sc = msg->socket; - size_t i; - - /* - * The list needs to be empty! - * The caller should take care of it. - */ - WARN_ON_ONCE(!list_empty(&msg->sibling_list)); - - /* - * Note we call ib_dma_unmap_page(), even if some sges are mapped using - * ib_dma_map_single(). - * - * The difference between _single() and _page() only matters for the - * ib_dma_map_*() case. - * - * For the ib_dma_unmap_*() case it does not matter as both take the - * dma_addr_t and dma_unmap_single_attrs() is just an alias to - * dma_unmap_page_attrs(). - */ - for (i = 0; i < msg->num_sge; i++) - ib_dma_unmap_page(sc->ib.dev, - msg->sge[i].addr, - msg->sge[i].length, - DMA_TO_DEVICE); - - mempool_free(msg, sc->send_io.mem.pool); -} - -struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc) -{ - struct smbdirect_recv_io *msg = NULL; - unsigned long flags; - - spin_lock_irqsave(&sc->recv_io.free.lock, flags); - if (likely(!sc->first_error)) - msg = list_first_entry_or_null(&sc->recv_io.free.list, - struct smbdirect_recv_io, - list); - if (likely(msg)) { - list_del(&msg->list); - sc->statistics.get_receive_buffer++; - } - spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); - - return msg; -} - -void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg) -{ - struct smbdirect_socket *sc = msg->socket; - unsigned long flags; - - if (likely(msg->sge.length != 0)) { - ib_dma_unmap_single(sc->ib.dev, - msg->sge.addr, - msg->sge.length, - DMA_FROM_DEVICE); - msg->sge.length = 0; - } - - spin_lock_irqsave(&sc->recv_io.free.lock, flags); - list_add_tail(&msg->list, &sc->recv_io.free.list); - sc->statistics.put_receive_buffer++; - spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); - - queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); -} - -void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc, - struct smbdirect_recv_io *msg, - u32 data_length) -{ - unsigned long flags; - - spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); - list_add_tail(&msg->list, &sc->recv_io.reassembly.list); - sc->recv_io.reassembly.queue_length++; - /* - * Make sure reassembly_data_length is updated after list and - * reassembly_queue_length are updated. On the dequeue side - * reassembly_data_length is checked without a lock to determine - * if reassembly_queue_length and list is up to date - */ - virt_wmb(); - sc->recv_io.reassembly.data_length += data_length; - spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); - sc->statistics.enqueue_reassembly_queue++; -} - -struct smbdirect_recv_io * -smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc) -{ - struct smbdirect_recv_io *msg; - - msg = list_first_entry_or_null(&sc->recv_io.reassembly.list, - struct smbdirect_recv_io, - list); - - return msg; -} - -void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc, - u8 peer_initiator_depth, - u8 peer_responder_resources, - const struct rdma_conn_param *param) -{ - struct smbdirect_socket_parameters *sp = &sc->parameters; - - if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num) && - param->private_data_len == 8) { - /* - * Legacy clients with only iWarp MPA v1 support - * need a private blob in order to negotiate - * the IRD/ORD values. - */ - const __be32 *ird_ord_hdr = param->private_data; - u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); - u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); - - /* - * cifs.ko sends the legacy IRD/ORD negotiation - * event if iWarp MPA v2 was used. - * - * Here we check that the values match and only - * mark the client as legacy if they don't match. - */ - if ((u32)param->initiator_depth != ird32 || - (u32)param->responder_resources != ord32) { - /* - * There are broken clients (old cifs.ko) - * using little endian and also - * struct rdma_conn_param only uses u8 - * for initiator_depth and responder_resources, - * so we truncate the value to U8_MAX. - * - * smb_direct_accept_client() will then - * do the real negotiation in order to - * select the minimum between client and - * server. - */ - ird32 = min_t(u32, ird32, U8_MAX); - ord32 = min_t(u32, ord32, U8_MAX); - - sc->rdma.legacy_iwarp = true; - peer_initiator_depth = (u8)ird32; - peer_responder_resources = (u8)ord32; - } - } - - /* - * negotiate the value by using the minimum - * between client and server if the client provided - * non 0 values. - */ - if (peer_initiator_depth != 0) - sp->initiator_depth = min_t(u8, sp->initiator_depth, - peer_initiator_depth); - if (peer_responder_resources != 0) - sp->responder_resources = min_t(u8, sp->responder_resources, - peer_responder_resources); -} - -bool smbdirect_connection_is_connected(struct smbdirect_socket *sc) -{ - if (unlikely(!sc || sc->first_error || sc->status != SMBDIRECT_SOCKET_CONNECTED)) - return false; - return true; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_is_connected); - -int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - union { - struct sockaddr sa; - struct sockaddr_storage ss; - } src_addr, dst_addr; - const struct sockaddr *src = NULL; - const struct sockaddr *dst = NULL; - char _devname[IB_DEVICE_NAME_MAX] = { 0, }; - const char *devname = NULL; - int ret; - - if (sc->rdma.cm_id) { - src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; - if (src_addr.sa.sa_family != AF_UNSPEC) - src = &src_addr.sa; - dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr; - if (dst_addr.sa.sa_family != AF_UNSPEC) - dst = &dst_addr.sa; - - if (sc->ib.dev) { - memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX); - devname = _devname; - } - } - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "waiting for connection: device: %.*s local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, devname, src, dst); - - ret = wait_event_interruptible_timeout(sc->status_wait, - sc->status == SMBDIRECT_SOCKET_CONNECTED || - sc->first_error, - msecs_to_jiffies(sp->negotiate_timeout_msec)); - if (sc->rdma.cm_id) { - /* - * Maybe src and dev are updated in the meantime. - */ - src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; - if (src_addr.sa.sa_family != AF_UNSPEC) - src = &src_addr.sa; - dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr; - if (dst_addr.sa.sa_family != AF_UNSPEC) - dst = &dst_addr.sa; - - if (sc->ib.dev) { - memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX); - devname = _devname; - } - } - if (ret == 0) - ret = -ETIMEDOUT; - if (ret < 0) - smbdirect_socket_schedule_cleanup(sc, ret); - if (sc->first_error) { - int lvl = SMBDIRECT_LOG_ERR; - - ret = sc->first_error; - if (ret == -ENODEV) - lvl = SMBDIRECT_LOG_INFO; - - smbdirect_log_rdma_event(sc, lvl, - "connection failed %1pe device: %.*s local: %pISpsfc remote: %pISpsfc\n", - SMBDIRECT_DEBUG_ERR_PTR(ret), - IB_DEVICE_NAME_MAX, devname, src, dst); - return ret; - } - - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_wait_for_connected); - -void smbdirect_connection_idle_timer_work(struct work_struct *work) -{ - struct smbdirect_socket *sc = - container_of(work, struct smbdirect_socket, idle.timer_work.work); - const struct smbdirect_socket_parameters *sp = &sc->parameters; - - if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { - smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_ERR, - "%s => timeout sc->idle.keepalive=%s\n", - smbdirect_socket_status_string(sc->status), - sc->idle.keepalive == SMBDIRECT_KEEPALIVE_SENT ? - "SENT" : "PENDING"); - smbdirect_socket_schedule_cleanup(sc, -ETIMEDOUT); - return; - } - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) - return; - - /* - * Now use the keepalive timeout (instead of keepalive interval) - * in order to wait for a response - */ - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->keepalive_timeout_msec)); - smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, - "schedule send of empty idle message\n"); - queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); -} - -u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc) -{ - int missing; - int available; - int new_credits; - - if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) - return 0; - - missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); - available = atomic_xchg(&sc->recv_io.credits.available, 0); - new_credits = min3((int)U16_MAX, missing, available); - if (new_credits <= 0) { - /* - * If credits are available, but not granted - * we need to re-add them again. - */ - if (available) - atomic_add(available, &sc->recv_io.credits.available); - return 0; - } - - if (new_credits < available) { - /* - * Readd the remaining available again. - */ - available -= new_credits; - atomic_add(available, &sc->recv_io.credits.available); - } - - /* - * Remember we granted the credits - */ - atomic_add(new_credits, &sc->recv_io.credits.count); - return new_credits; -} - -static bool smbdirect_connection_request_keep_alive(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - - if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; - /* - * Now use the keepalive timeout (instead of keepalive interval) - * in order to wait for a response - */ - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->keepalive_timeout_msec)); - return true; - } - - return false; -} - -int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc, - struct ib_send_wr *wr) -{ - int ret; - - if (unlikely(sc->first_error)) - return sc->first_error; - - atomic_inc(&sc->send_io.pending.count); - ret = ib_post_send(sc->ib.qp, wr, NULL); - if (ret) { - atomic_dec(&sc->send_io.pending.count); - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "ib_post_send() failed %1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_socket_schedule_cleanup(sc, ret); - } - - return ret; -} - -static void smbdirect_connection_send_batch_init(struct smbdirect_send_batch *batch, - bool need_invalidate_rkey, - unsigned int remote_key) -{ - INIT_LIST_HEAD(&batch->msg_list); - batch->wr_cnt = 0; - batch->need_invalidate_rkey = need_invalidate_rkey; - batch->remote_key = remote_key; - batch->credit = 0; -} - -int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - bool is_last) -{ - struct smbdirect_send_io *first, *last; - int ret = 0; - - if (list_empty(&batch->msg_list)) - goto release_credit; - - first = list_first_entry(&batch->msg_list, - struct smbdirect_send_io, - sibling_list); - last = list_last_entry(&batch->msg_list, - struct smbdirect_send_io, - sibling_list); - - if (batch->need_invalidate_rkey) { - first->wr.opcode = IB_WR_SEND_WITH_INV; - first->wr.ex.invalidate_rkey = batch->remote_key; - batch->need_invalidate_rkey = false; - batch->remote_key = 0; - } - - last->wr.send_flags = IB_SEND_SIGNALED; - last->wr.wr_cqe = &last->cqe; - - /* - * Remove last from send_ctx->msg_list - * and splice the rest of send_ctx->msg_list - * to last->sibling_list. - * - * send_ctx->msg_list is a valid empty list - * at the end. - */ - list_del_init(&last->sibling_list); - list_splice_tail_init(&batch->msg_list, &last->sibling_list); - batch->wr_cnt = 0; - - ret = smbdirect_connection_post_send_wr(sc, &first->wr); - if (ret) { - struct smbdirect_send_io *sibling, *next; - - list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { - list_del_init(&sibling->sibling_list); - smbdirect_connection_free_send_io(sibling); - } - smbdirect_connection_free_send_io(last); - } - -release_credit: - if (is_last && !ret && batch->credit) { - atomic_add(batch->credit, &sc->send_io.bcredits.count); - batch->credit = 0; - wake_up(&sc->send_io.bcredits.wait_queue); - } - - return ret; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_batch_flush); - -struct smbdirect_send_batch * -smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, - bool need_invalidate_rkey, - unsigned int remote_key) -{ - struct smbdirect_send_batch *batch = (struct smbdirect_send_batch *)storage; - - memset(storage, 0, sizeof(*storage)); - BUILD_BUG_ON(sizeof(*batch) > sizeof(*storage)); - - smbdirect_connection_send_batch_init(batch, - need_invalidate_rkey, - remote_key); - - return batch; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_init_send_batch_storage); - -static int smbdirect_connection_wait_for_send_bcredit(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch) -{ - int ret; - - if (batch->credit) - return 0; - - ret = smbdirect_socket_wait_for_credits(sc, - SMBDIRECT_SOCKET_CONNECTED, - -ENOTCONN, - &sc->send_io.bcredits.wait_queue, - &sc->send_io.bcredits.count, - 1); - if (ret) - return ret; - - batch->credit = 1; - return 0; -} - -static int smbdirect_connection_wait_for_send_lcredit(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch) -{ - if (batch && atomic_read(&sc->send_io.lcredits.count) <= 1) { - int ret; - - ret = smbdirect_connection_send_batch_flush(sc, batch, false); - if (ret) - return ret; - } - - return smbdirect_socket_wait_for_credits(sc, - SMBDIRECT_SOCKET_CONNECTED, - -ENOTCONN, - &sc->send_io.lcredits.wait_queue, - &sc->send_io.lcredits.count, - 1); -} - -static int smbdirect_connection_wait_for_send_credits(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch) -{ - if (batch && (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { - int ret; - - ret = smbdirect_connection_send_batch_flush(sc, batch, false); - if (ret) - return ret; - } - - return smbdirect_socket_wait_for_credits(sc, - SMBDIRECT_SOCKET_CONNECTED, - -ENOTCONN, - &sc->send_io.credits.wait_queue, - &sc->send_io.credits.count, - 1); -} - -static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc); - -static int smbdirect_connection_post_send_io(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - struct smbdirect_send_io *msg) -{ - int i; - - for (i = 0; i < msg->num_sge; i++) - ib_dma_sync_single_for_device(sc->ib.dev, - msg->sge[i].addr, msg->sge[i].length, - DMA_TO_DEVICE); - - msg->cqe.done = smbdirect_connection_send_io_done; - msg->wr.wr_cqe = &msg->cqe; - msg->wr.opcode = IB_WR_SEND; - msg->wr.sg_list = &msg->sge[0]; - msg->wr.num_sge = msg->num_sge; - msg->wr.next = NULL; - - if (batch) { - msg->wr.send_flags = 0; - if (!list_empty(&batch->msg_list)) { - struct smbdirect_send_io *last; - - last = list_last_entry(&batch->msg_list, - struct smbdirect_send_io, - sibling_list); - last->wr.next = &msg->wr; - } - list_add_tail(&msg->sibling_list, &batch->msg_list); - batch->wr_cnt++; - return 0; - } - - msg->wr.send_flags = IB_SEND_SIGNALED; - return smbdirect_connection_post_send_wr(sc, &msg->wr); -} - -int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - struct iov_iter *iter, - unsigned int flags, - u32 remaining_data_length) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_send_batch _batch; - struct smbdirect_send_io *msg; - struct smbdirect_data_transfer *packet; - size_t header_length; - u16 new_credits = 0; - u32 data_length = 0; - int ret; - - if (WARN_ON_ONCE(flags)) - return -EINVAL; /* no flags support for now */ - - if (iter) { - if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) - return -EINVAL; /* It's a bug in upper layer to get there */ - - header_length = sizeof(struct smbdirect_data_transfer); - if (WARN_ON_ONCE(remaining_data_length == 0 || - iov_iter_count(iter) > remaining_data_length)) - return -EINVAL; - } else { - /* If this is a packet without payload, don't send padding */ - header_length = offsetof(struct smbdirect_data_transfer, padding); - if (WARN_ON_ONCE(remaining_data_length)) - return -EINVAL; - } - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, - "status=%s first_error=%1pe => %1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); - return -ENOTCONN; - } - - if (!batch) { - smbdirect_connection_send_batch_init(&_batch, false, 0); - batch = &_batch; - } - - ret = smbdirect_connection_wait_for_send_bcredit(sc, batch); - if (ret) - goto bcredit_failed; - - ret = smbdirect_connection_wait_for_send_lcredit(sc, batch); - if (ret) - goto lcredit_failed; - - ret = smbdirect_connection_wait_for_send_credits(sc, batch); - if (ret) - goto credit_failed; - - new_credits = smbdirect_connection_grant_recv_credits(sc); - if (new_credits == 0 && - atomic_read(&sc->send_io.credits.count) == 0 && - atomic_read(&sc->recv_io.credits.count) == 0) { - /* - * queue the refill work in order to - * get some new recv credits we can grant to - * the peer. - */ - queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); - - /* - * wait until either the refill work or the peer - * granted new credits - */ - ret = wait_event_interruptible(sc->send_io.credits.wait_queue, - atomic_read(&sc->send_io.credits.count) >= 1 || - atomic_read(&sc->recv_io.credits.available) >= 1 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) - ret = -ENOTCONN; - if (ret < 0) - goto credit_failed; - - new_credits = smbdirect_connection_grant_recv_credits(sc); - } - - msg = smbdirect_connection_alloc_send_io(sc); - if (IS_ERR(msg)) { - ret = PTR_ERR(msg); - goto alloc_failed; - } - - /* Map the packet to DMA */ - msg->sge[0].addr = ib_dma_map_single(sc->ib.dev, - msg->packet, - header_length, - DMA_TO_DEVICE); - ret = ib_dma_mapping_error(sc->ib.dev, msg->sge[0].addr); - if (ret) - goto err; - - msg->sge[0].length = header_length; - msg->sge[0].lkey = sc->ib.pd->local_dma_lkey; - msg->num_sge = 1; - - if (iter) { - struct smbdirect_map_sges extract = { - .num_sge = msg->num_sge, - .max_sge = ARRAY_SIZE(msg->sge), - .sge = msg->sge, - .device = sc->ib.dev, - .local_dma_lkey = sc->ib.pd->local_dma_lkey, - .direction = DMA_TO_DEVICE, - }; - size_t payload_len = umin(iov_iter_count(iter), - sp->max_send_size - sizeof(*packet)); - - ret = smbdirect_map_sges_from_iter(iter, payload_len, &extract); - if (ret < 0) - goto err; - data_length = ret; - remaining_data_length -= data_length; - msg->num_sge = extract.num_sge; - } - - /* Fill in the packet header */ - packet = (struct smbdirect_data_transfer *)msg->packet; - packet->credits_requested = cpu_to_le16(sp->send_credit_target); - packet->credits_granted = cpu_to_le16(new_credits); - - packet->flags = 0; - if (smbdirect_connection_request_keep_alive(sc)) - packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); - - packet->reserved = 0; - if (!data_length) - packet->data_offset = 0; - else - packet->data_offset = cpu_to_le32(24); - packet->data_length = cpu_to_le32(data_length); - packet->remaining_data_length = cpu_to_le32(remaining_data_length); - packet->padding = 0; - - smbdirect_log_outgoing(sc, SMBDIRECT_LOG_INFO, - "DataOut: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n", - "CreditsRequested", - le16_to_cpu(packet->credits_requested), - "CreditsGranted", - le16_to_cpu(packet->credits_granted), - "Flags", - le16_to_cpu(packet->flags), - "RemainingDataLength", - le32_to_cpu(packet->remaining_data_length), - "DataOffset", - le32_to_cpu(packet->data_offset), - "DataLength", - le32_to_cpu(packet->data_length)); - - ret = smbdirect_connection_post_send_io(sc, batch, msg); - if (ret) - goto err; - - /* - * From here msg is moved to send_ctx - * and we should not free it explicitly. - */ - - if (batch == &_batch) { - ret = smbdirect_connection_send_batch_flush(sc, batch, true); - if (ret) - goto flush_failed; - } - - return data_length; -err: - smbdirect_connection_free_send_io(msg); -flush_failed: -alloc_failed: - atomic_inc(&sc->send_io.credits.count); -credit_failed: - atomic_inc(&sc->send_io.lcredits.count); -lcredit_failed: - atomic_add(batch->credit, &sc->send_io.bcredits.count); - batch->credit = 0; -bcredit_failed: - return ret; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_single_iter); - -int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc) -{ - /* - * As an optimization, we don't wait for individual I/O to finish - * before sending the next one. - * Send them all and wait for pending send count to get to 0 - * that means all the I/Os have been out and we are good to return - */ - - wait_event(sc->send_io.pending.zero_wait_queue, - atomic_read(&sc->send_io.pending.count) == 0 || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, - "status=%s first_error=%1pe => %1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); - return -ENOTCONN; - } - - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_wait_zero_pending); - -int smbdirect_connection_send_iter(struct smbdirect_socket *sc, - struct iov_iter *iter, - unsigned int flags, - bool need_invalidate, - unsigned int remote_key) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_send_batch batch; - int total_count = iov_iter_count(iter); - int ret; - int error = 0; - __be32 hdr; - - if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags)) - return -EINVAL; /* no flags support for now */ - - if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) - return -EINVAL; /* It's a bug in upper layer to get there */ - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - smbdirect_log_write(sc, SMBDIRECT_LOG_INFO, - "status=%s first_error=%1pe => %1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); - return -ENOTCONN; - } - - /* - * For now we expect the iter to have the full - * message, including a 4 byte length header. - */ - if (iov_iter_count(iter) <= 4) - return -EINVAL; - if (!copy_from_iter_full(&hdr, sizeof(hdr), iter)) - return -EFAULT; - if (iov_iter_count(iter) != be32_to_cpu(hdr)) - return -EINVAL; - - /* - * The size must fit into the negotiated - * fragmented send size. - */ - if (iov_iter_count(iter) > sp->max_fragmented_send_size) - return -EMSGSIZE; - - smbdirect_log_write(sc, SMBDIRECT_LOG_INFO, - "Sending (RDMA): length=%zu\n", - iov_iter_count(iter)); - - smbdirect_connection_send_batch_init(&batch, need_invalidate, remote_key); - while (iov_iter_count(iter)) { - ret = smbdirect_connection_send_single_iter(sc, - &batch, - iter, - flags, - iov_iter_count(iter)); - if (unlikely(ret < 0)) { - error = ret; - break; - } - } - - ret = smbdirect_connection_send_batch_flush(sc, &batch, true); - if (unlikely(ret && !error)) - error = ret; - - /* - * As an optimization, we don't wait for individual I/O to finish - * before sending the next one. - * Send them all and wait for pending send count to get to 0 - * that means all the I/Os have been out and we are good to return - */ - - ret = smbdirect_connection_send_wait_zero_pending(sc); - if (unlikely(ret && !error)) - error = ret; - - if (unlikely(error)) - return error; - - return total_count; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_iter); - -static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_send_io *msg = - container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); - struct smbdirect_socket *sc = msg->socket; - struct smbdirect_send_io *sibling, *next; - int lcredits = 0; - - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, - "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - - if (unlikely(!(msg->wr.send_flags & IB_SEND_SIGNALED))) { - /* - * This happens when smbdirect_send_io is a sibling - * before the final message, it is signaled on - * error anyway, so we need to skip - * smbdirect_connection_free_send_io here, - * otherwise is will destroy the memory - * of the siblings too, which will cause - * use after free problems for the others - * triggered from ib_drain_qp(). - */ - if (wc->status != IB_WC_SUCCESS) - goto skip_free; - - /* - * This should not happen! - * But we better just close the - * connection... - */ - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - /* - * Free possible siblings and then the main send_io - */ - list_for_each_entry_safe(sibling, next, &msg->sibling_list, sibling_list) { - list_del_init(&sibling->sibling_list); - smbdirect_connection_free_send_io(sibling); - lcredits += 1; - } - /* Note this frees wc->wr_cqe, but not wc */ - smbdirect_connection_free_send_io(msg); - lcredits += 1; - - if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { -skip_free: - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - return; - } - - atomic_add(lcredits, &sc->send_io.lcredits.count); - wake_up(&sc->send_io.lcredits.wait_queue); - - if (atomic_dec_and_test(&sc->send_io.pending.count)) - wake_up(&sc->send_io.pending.zero_wait_queue); -} - -static void smbdirect_connection_send_immediate_work(struct work_struct *work) -{ - struct smbdirect_socket *sc = - container_of(work, struct smbdirect_socket, idle.immediate_work); - int ret; - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) - return; - - smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, - "send an empty message\n"); - sc->statistics.send_empty++; - ret = smbdirect_connection_send_single_iter(sc, NULL, NULL, 0, 0); - if (ret < 0) { - smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_send_single_iter ret=%1pe\n", - SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_socket_schedule_cleanup(sc, ret); - } -} - -int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg) -{ - struct smbdirect_socket *sc = msg->socket; - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct ib_recv_wr recv_wr = { - .wr_cqe = &msg->cqe, - .sg_list = &msg->sge, - .num_sge = 1, - }; - int ret; - - if (unlikely(sc->first_error)) - return sc->first_error; - - msg->sge.addr = ib_dma_map_single(sc->ib.dev, - msg->packet, - sp->max_recv_size, - DMA_FROM_DEVICE); - ret = ib_dma_mapping_error(sc->ib.dev, msg->sge.addr); - if (ret) - return ret; - - msg->sge.length = sp->max_recv_size; - msg->sge.lkey = sc->ib.pd->local_dma_lkey; - - ret = ib_post_recv(sc->ib.qp, &recv_wr, NULL); - if (ret) { - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, - "ib_post_recv failed ret=%d (%1pe)\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - ib_dma_unmap_single(sc->ib.dev, - msg->sge.addr, - msg->sge.length, - DMA_FROM_DEVICE); - msg->sge.length = 0; - smbdirect_socket_schedule_cleanup(sc, ret); - } - - return ret; -} - -void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_recv_io *recv_io = - container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); - struct smbdirect_socket *sc = recv_io->socket; - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_data_transfer *data_transfer; - int current_recv_credits; - u16 old_recv_credit_target; - u16 credits_requested; - u16 credits_granted; - u16 flags; - u32 data_offset; - u32 data_length; - u32 remaining_data_length; - - if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s (%d) wc->opcode=%d\n", - ib_wc_status_msg(wc->status), wc->status, wc->opcode); - goto error; - } - - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, - "recv_io=0x%p type=%d wc status=%s wc opcode %d byte_len=%d pkey_index=%u\n", - recv_io, sc->recv_io.expected, - ib_wc_status_msg(wc->status), wc->opcode, - wc->byte_len, wc->pkey_index); - - /* - * Reset timer to the keepalive interval in - * order to trigger our next keepalive message. - */ - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; - mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, - msecs_to_jiffies(sp->keepalive_interval_msec)); - - ib_dma_sync_single_for_cpu(sc->ib.dev, - recv_io->sge.addr, - recv_io->sge.length, - DMA_FROM_DEVICE); - - if (unlikely(wc->byte_len < - offsetof(struct smbdirect_data_transfer, padding))) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "wc->byte_len=%u < %zu\n", - wc->byte_len, - offsetof(struct smbdirect_data_transfer, padding)); - goto error; - } - - data_transfer = (struct smbdirect_data_transfer *)recv_io->packet; - credits_requested = le16_to_cpu(data_transfer->credits_requested); - credits_granted = le16_to_cpu(data_transfer->credits_granted); - flags = le16_to_cpu(data_transfer->flags); - remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); - data_offset = le32_to_cpu(data_transfer->data_offset); - data_length = le32_to_cpu(data_transfer->data_length); - - smbdirect_log_incoming(sc, SMBDIRECT_LOG_INFO, - "DataIn: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n", - "CreditsRequested", - credits_requested, - "CreditsGranted", - credits_granted, - "Flags", - flags, - "RemainingDataLength", - remaining_data_length, - "DataOffset", - data_offset, - "DataLength", - data_length); - - if (unlikely(credits_requested == 0)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: credits_requested == 0\n"); - goto error; - } - - if (unlikely(data_offset % 8 != 0)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "invalid: data_offset=%u (0x%x) not aligned to 8\n", - data_offset, data_offset); - goto error; - } - - if (unlikely(wc->byte_len < data_offset || - (u64)wc->byte_len < (u64)data_offset + data_length)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "wc->byte_len=%u < date_offset=%u + data_length=%u\n", - wc->byte_len, data_offset, data_length); - goto error; - } - - if (unlikely(remaining_data_length > sp->max_fragmented_recv_size || - data_length > sp->max_fragmented_recv_size || - (u64)remaining_data_length + (u64)data_length > (u64)sp->max_fragmented_recv_size)) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "remaining_data_length=%u + data_length=%u > max_fragmented=%u\n", - remaining_data_length, data_length, sp->max_fragmented_recv_size); - goto error; - } - - if (data_length) { - if (sc->recv_io.reassembly.full_packet_received) - recv_io->first_segment = true; - - if (remaining_data_length) - sc->recv_io.reassembly.full_packet_received = false; - else - sc->recv_io.reassembly.full_packet_received = true; - } - - atomic_dec(&sc->recv_io.posted.count); - current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); - - /* - * We take the value from the peer, which is checked to be higher than 0, - * but we limit it to the max value we support in order to have - * the main logic simpler. - */ - old_recv_credit_target = sc->recv_io.credits.target; - sc->recv_io.credits.target = credits_requested; - sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, - sp->recv_credit_max); - if (credits_granted) { - atomic_add(credits_granted, &sc->send_io.credits.count); - /* - * We have new send credits granted from remote peer - * If any sender is waiting for credits, unblock it - */ - wake_up(&sc->send_io.credits.wait_queue); - } - - /* Send an immediate response right away if requested */ - if (flags & SMBDIRECT_FLAG_RESPONSE_REQUESTED) { - smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, - "schedule send of immediate response\n"); - queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); - } - - /* - * If this is a packet with data playload place the data in - * reassembly queue and wake up the reading thread - */ - if (data_length) { - if (current_recv_credits <= (sc->recv_io.credits.target / 4) || - sc->recv_io.credits.target > old_recv_credit_target) - queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); - - smbdirect_connection_reassembly_append_recv_io(sc, recv_io, data_length); - wake_up(&sc->recv_io.reassembly.wait_queue); - } else - smbdirect_connection_put_recv_io(recv_io); - - return; - -error: - /* - * Make sure smbdirect_connection_put_recv_io() does not - * start recv_io.posted.refill_work. - */ - disable_work(&sc->recv_io.posted.refill_work); - smbdirect_connection_put_recv_io(recv_io); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); -} - -int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc) -{ - int missing; - int posted = 0; - - if (unlikely(sc->first_error)) - return sc->first_error; - - /* - * Find out how much smbdirect_recv_io buffers we should post. - * - * Note that sc->recv_io.credits.target is the value - * from the peer and it can in theory change over time, - * but it is forced to be at least 1 and at max - * sp->recv_credit_max. - * - * So it can happen that missing will be lower than 0, - * which means the peer has recently lowered its desired - * target, while be already granted a higher number of credits. - * - * Note 'posted' is the number of smbdirect_recv_io buffers - * posted within this function, while sc->recv_io.posted.count - * is the overall value of posted smbdirect_recv_io buffers. - * - * We try to post as much buffers as missing, but - * this is limited if a lot of smbdirect_recv_io buffers - * are still in the sc->recv_io.reassembly.list instead of - * the sc->recv_io.free.list. - * - */ - missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.posted.count); - while (posted < missing) { - struct smbdirect_recv_io *recv_io; - int ret; - - /* - * It's ok if smbdirect_connection_get_recv_io() - * returns NULL, it means smbdirect_recv_io structures - * are still be in the reassembly.list. - */ - recv_io = smbdirect_connection_get_recv_io(sc); - if (!recv_io) - break; - - recv_io->first_segment = false; - - ret = smbdirect_connection_post_recv_io(recv_io); - if (ret) { - smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_post_recv_io failed rc=%d (%1pe)\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_connection_put_recv_io(recv_io); - return ret; - } - - atomic_inc(&sc->recv_io.posted.count); - posted += 1; - } - - /* If nothing was posted we're done */ - if (posted == 0) - return 0; - - atomic_add(posted, &sc->recv_io.credits.available); - - /* - * If the last send credit is waiting for credits - * it can grant we need to wake it up - */ - if (atomic_read(&sc->send_io.bcredits.count) == 0 && - atomic_read(&sc->send_io.credits.count) == 0) - wake_up(&sc->send_io.credits.wait_queue); - - /* - * If we posted at least one smbdirect_recv_io buffer, - * we need to inform the peer about it and grant - * additional credits. - * - * However there is one case where we don't want to - * do that. - * - * If only a single credit was missing before - * reaching the requested target, we should not - * post an immediate send, as that would cause - * endless ping pong once a keep alive exchange - * is started. - * - * However if sc->recv_io.credits.target is only 1, - * the peer has no credit left and we need to - * grant the credit anyway. - */ - if (missing == 1 && sc->recv_io.credits.target != 1) - return 0; - - return posted; -} - -static void smbdirect_connection_recv_io_refill_work(struct work_struct *work) -{ - struct smbdirect_socket *sc = - container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); - int posted; - - posted = smbdirect_connection_recv_io_refill(sc); - if (unlikely(posted < 0)) { - smbdirect_socket_schedule_cleanup(sc, posted); - return; - } - if (posted > 0) { - smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, - "schedule send of an empty message\n"); - queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); - } -} - -int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, - struct msghdr *msg, - unsigned int flags) -{ - struct smbdirect_recv_io *response; - struct smbdirect_data_transfer *data_transfer; - size_t size = iov_iter_count(&msg->msg_iter); - int to_copy, to_read, data_read, offset; - u32 data_length, remaining_data_length, data_offset; - int ret; - - if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags)) - return -EINVAL; /* no flags support for now */ - - if (WARN_ON_ONCE(iov_iter_rw(&msg->msg_iter) != ITER_DEST)) - return -EINVAL; /* It's a bug in upper layer to get there */ - -again: - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "status=%s first_error=%1pe => %1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); - return -ENOTCONN; - } - - /* - * No need to hold the reassembly queue lock all the time as we are - * the only one reading from the front of the queue. The transport - * may add more entries to the back of the queue at the same time - */ - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "size=%zd sc->recv_io.reassembly.data_length=%d\n", - size, sc->recv_io.reassembly.data_length); - if (sc->recv_io.reassembly.data_length >= size) { - int queue_length; - int queue_removed = 0; - unsigned long flags; - - /* - * Need to make sure reassembly_data_length is read before - * reading reassembly_queue_length and calling - * smbdirect_connection_reassembly_first_recv_io. This call is lock free - * as we never read at the end of the queue which are being - * updated in SOFTIRQ as more data is received - */ - virt_rmb(); - queue_length = sc->recv_io.reassembly.queue_length; - data_read = 0; - to_read = size; - offset = sc->recv_io.reassembly.first_entry_offset; - while (data_read < size) { - response = smbdirect_connection_reassembly_first_recv_io(sc); - data_transfer = (void *)response->packet; - data_length = le32_to_cpu(data_transfer->data_length); - remaining_data_length = - le32_to_cpu( - data_transfer->remaining_data_length); - data_offset = le32_to_cpu(data_transfer->data_offset); - - /* - * The upper layer expects RFC1002 length at the - * beginning of the payload. Return it to indicate - * the total length of the packet. This minimize the - * change to upper layer packet processing logic. This - * will be eventually remove when an intermediate - * transport layer is added - */ - if (response->first_segment && size == 4) { - unsigned int rfc1002_len = - data_length + remaining_data_length; - __be32 rfc1002_hdr = cpu_to_be32(rfc1002_len); - - if (copy_to_iter(&rfc1002_hdr, sizeof(rfc1002_hdr), - &msg->msg_iter) != sizeof(rfc1002_hdr)) - return -EFAULT; - data_read = 4; - response->first_segment = false; - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "returning rfc1002 length %d\n", - rfc1002_len); - goto read_rfc1002_done; - } - - to_copy = min_t(int, data_length - offset, to_read); - if (copy_to_iter((u8 *)data_transfer + data_offset + offset, - to_copy, &msg->msg_iter) != to_copy) - return -EFAULT; - - /* move on to the next buffer? */ - if (to_copy == data_length - offset) { - queue_length--; - /* - * No need to lock if we are not at the - * end of the queue - */ - if (queue_length) - list_del(&response->list); - else { - spin_lock_irqsave( - &sc->recv_io.reassembly.lock, flags); - list_del(&response->list); - spin_unlock_irqrestore( - &sc->recv_io.reassembly.lock, flags); - } - queue_removed++; - sc->statistics.dequeue_reassembly_queue++; - smbdirect_connection_put_recv_io(response); - offset = 0; - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "smbdirect_connection_put_recv_io offset=0\n"); - } else - offset += to_copy; - - to_read -= to_copy; - data_read += to_copy; - - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "memcpy %d bytes len-ofs=%u => todo=%u done=%u ofs=%u\n", - to_copy, data_length - offset, - to_read, data_read, offset); - } - - spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); - sc->recv_io.reassembly.data_length -= data_read; - sc->recv_io.reassembly.queue_length -= queue_removed; - spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); - - sc->recv_io.reassembly.first_entry_offset = offset; - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "returning data_read=%d reassembly_length=%d first_ofs=%u\n", - data_read, sc->recv_io.reassembly.data_length, - sc->recv_io.reassembly.first_entry_offset); -read_rfc1002_done: - return data_read; - } - - smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, - "wait_event on more data\n"); - ret = wait_event_interruptible(sc->recv_io.reassembly.wait_queue, - sc->recv_io.reassembly.data_length >= size || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - /* Don't return any data if interrupted */ - if (ret) - return ret; - - goto again; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_recvmsg); - -static bool smbdirect_map_sges_single_page(struct smbdirect_map_sges *state, - struct page *page, size_t off, size_t len) -{ - struct ib_sge *sge; - u64 addr; - - if (state->num_sge >= state->max_sge) - return false; - - addr = ib_dma_map_page(state->device, page, - off, len, state->direction); - if (ib_dma_mapping_error(state->device, addr)) - return false; - - sge = &state->sge[state->num_sge++]; - sge->addr = addr; - sge->length = len; - sge->lkey = state->local_dma_lkey; - - return true; -} - -/* - * Extract page fragments from a BVEC-class iterator and add them to an ib_sge - * list. The pages are not pinned. - */ -static ssize_t smbdirect_map_sges_from_bvec(struct iov_iter *iter, - struct smbdirect_map_sges *state, - ssize_t maxsize) -{ - const struct bio_vec *bv = iter->bvec; - unsigned long start = iter->iov_offset; - unsigned int i; - ssize_t ret = 0; - - for (i = 0; i < iter->nr_segs; i++) { - size_t off, len; - bool ok; - - len = bv[i].bv_len; - if (start >= len) { - start -= len; - continue; - } - - len = min_t(size_t, maxsize, len - start); - off = bv[i].bv_offset + start; - - ok = smbdirect_map_sges_single_page(state, - bv[i].bv_page, - off, - len); - if (!ok) - return -EIO; - - ret += len; - maxsize -= len; - if (state->num_sge >= state->max_sge || maxsize <= 0) - break; - start = 0; - } - - if (ret > 0) - iov_iter_advance(iter, ret); - return ret; -} - -/* - * Extract fragments from a KVEC-class iterator and add them to an ib_sge list. - * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers. - * The pages are not pinned. - */ -static ssize_t smbdirect_map_sges_from_kvec(struct iov_iter *iter, - struct smbdirect_map_sges *state, - ssize_t maxsize) -{ - const struct kvec *kv = iter->kvec; - unsigned long start = iter->iov_offset; - unsigned int i; - ssize_t ret = 0; - - for (i = 0; i < iter->nr_segs; i++) { - struct page *page; - unsigned long kaddr; - size_t off, len, seg; - - len = kv[i].iov_len; - if (start >= len) { - start -= len; - continue; - } - - kaddr = (unsigned long)kv[i].iov_base + start; - off = kaddr & ~PAGE_MASK; - len = min_t(size_t, maxsize, len - start); - kaddr &= PAGE_MASK; - - maxsize -= len; - do { - bool ok; - - seg = min_t(size_t, len, PAGE_SIZE - off); - - if (is_vmalloc_or_module_addr((void *)kaddr)) - page = vmalloc_to_page((void *)kaddr); - else - page = virt_to_page((void *)kaddr); - - ok = smbdirect_map_sges_single_page(state, page, off, seg); - if (!ok) - return -EIO; - - ret += seg; - len -= seg; - kaddr += PAGE_SIZE; - off = 0; - } while (len > 0 && state->num_sge < state->max_sge); - - if (state->num_sge >= state->max_sge || maxsize <= 0) - break; - start = 0; - } - - if (ret > 0) - iov_iter_advance(iter, ret); - return ret; -} - -/* - * Extract folio fragments from a FOLIOQ-class iterator and add them to an - * ib_sge list. The folios are not pinned. - */ -static ssize_t smbdirect_map_sges_from_folioq(struct iov_iter *iter, - struct smbdirect_map_sges *state, - ssize_t maxsize) -{ - const struct folio_queue *folioq = iter->folioq; - unsigned int slot = iter->folioq_slot; - ssize_t ret = 0; - size_t offset = iter->iov_offset; - - if (WARN_ON_ONCE(!folioq)) - return -EIO; - - if (slot >= folioq_nr_slots(folioq)) { - folioq = folioq->next; - if (WARN_ON_ONCE(!folioq)) - return -EIO; - slot = 0; - } - - do { - struct folio *folio = folioq_folio(folioq, slot); - size_t fsize = folioq_folio_size(folioq, slot); - - if (offset < fsize) { - size_t part = umin(maxsize, fsize - offset); - bool ok; - - ok = smbdirect_map_sges_single_page(state, - folio_page(folio, 0), - offset, - part); - if (!ok) - return -EIO; - - offset += part; - ret += part; - maxsize -= part; - } - - if (offset >= fsize) { - offset = 0; - slot++; - if (slot >= folioq_nr_slots(folioq)) { - if (!folioq->next) { - WARN_ON_ONCE(ret < iter->count); - break; - } - folioq = folioq->next; - slot = 0; - } - } - } while (state->num_sge < state->max_sge && maxsize > 0); - - iter->folioq = folioq; - iter->folioq_slot = slot; - iter->iov_offset = offset; - iter->count -= ret; - return ret; -} - -/* - * Extract page fragments from up to the given amount of the source iterator - * and build up an ib_sge list that refers to all of those bits. The ib_sge list - * is appended to, up to the maximum number of elements set in the parameter - * block. - * - * The extracted page fragments are not pinned or ref'd in any way; if an - * IOVEC/UBUF-type iterator is to be used, it should be converted to a - * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some - * way. - */ -static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, - struct smbdirect_map_sges *state) -{ - ssize_t ret; - size_t before = state->num_sge; - - if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) - return -EIO; - - switch (iov_iter_type(iter)) { - case ITER_BVEC: - ret = smbdirect_map_sges_from_bvec(iter, state, len); - break; - case ITER_KVEC: - ret = smbdirect_map_sges_from_kvec(iter, state, len); - break; - case ITER_FOLIOQ: - ret = smbdirect_map_sges_from_folioq(iter, state, len); - break; - default: - WARN_ONCE(1, "iov_iter_type[%u]\n", iov_iter_type(iter)); - return -EIO; - } - - if (ret < 0) { - while (state->num_sge > before) { - struct ib_sge *sge = &state->sge[state->num_sge--]; - - ib_dma_unmap_page(state->device, - sge->addr, - sge->length, - state->direction); - } - } - - return ret; -} diff --git a/fs/smb/common/smbdirect/smbdirect_debug.c b/fs/smb/common/smbdirect/smbdirect_debug.c deleted file mode 100644 index d8664fd7f71a..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_debug.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" -#include - -void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, - unsigned int rdma_readwrite_threshold, - struct seq_file *m) -{ - const struct smbdirect_socket_parameters *sp; - - if (!sc) - return; - sp = &sc->parameters; - - seq_puts(m, "\n"); - seq_printf(m, "SMBDirect protocol version: 0x%x ", - SMBDIRECT_V1); - seq_printf(m, "transport status: %s (%u)", - smbdirect_socket_status_string(sc->status), - sc->status); - - seq_puts(m, "\n"); - seq_printf(m, "Conn receive_credit_max: %u ", - sp->recv_credit_max); - seq_printf(m, "send_credit_target: %u max_send_size: %u", - sp->send_credit_target, - sp->max_send_size); - - seq_puts(m, "\n"); - seq_printf(m, "Conn max_fragmented_recv_size: %u ", - sp->max_fragmented_recv_size); - seq_printf(m, "max_fragmented_send_size: %u max_receive_size:%u", - sp->max_fragmented_send_size, - sp->max_recv_size); - - seq_puts(m, "\n"); - seq_printf(m, "Conn keep_alive_interval: %u ", - sp->keepalive_interval_msec * 1000); - seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u", - sp->max_read_write_size, - rdma_readwrite_threshold); - - seq_puts(m, "\n"); - seq_printf(m, "Debug count_get_receive_buffer: %llu ", - sc->statistics.get_receive_buffer); - seq_printf(m, "count_put_receive_buffer: %llu count_send_empty: %llu", - sc->statistics.put_receive_buffer, - sc->statistics.send_empty); - - seq_puts(m, "\n"); - seq_printf(m, "Read Queue count_enqueue_reassembly_queue: %llu ", - sc->statistics.enqueue_reassembly_queue); - seq_printf(m, "count_dequeue_reassembly_queue: %llu ", - sc->statistics.dequeue_reassembly_queue); - seq_printf(m, "reassembly_data_length: %u ", - sc->recv_io.reassembly.data_length); - seq_printf(m, "reassembly_queue_length: %u", - sc->recv_io.reassembly.queue_length); - - seq_puts(m, "\n"); - seq_printf(m, "Current Credits send_credits: %u ", - atomic_read(&sc->send_io.credits.count)); - seq_printf(m, "receive_credits: %u receive_credit_target: %u", - atomic_read(&sc->recv_io.credits.count), - sc->recv_io.credits.target); - - seq_puts(m, "\n"); - seq_printf(m, "Pending send_pending: %u ", - atomic_read(&sc->send_io.pending.count)); - - seq_puts(m, "\n"); - seq_printf(m, "MR responder_resources: %u ", - sp->responder_resources); - seq_printf(m, "max_frmr_depth: %u mr_type: 0x%x", - sp->max_frmr_depth, - sc->mr_io.type); - - seq_puts(m, "\n"); - seq_printf(m, "MR mr_ready_count: %u mr_used_count: %u", - atomic_read(&sc->mr_io.ready.count), - atomic_read(&sc->mr_io.used.count)); -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_legacy_debug_proc_show); diff --git a/fs/smb/common/smbdirect/smbdirect_devices.c b/fs/smb/common/smbdirect/smbdirect_devices.c deleted file mode 100644 index aaab99e9c045..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_devices.c +++ /dev/null @@ -1,277 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (C) 2018, LG Electronics. - * Copyright (c) 2025 Stefan Metzmacher - */ - -#include "smbdirect_internal.h" - -static u8 smbdirect_ib_device_rdma_capable_node_type(struct ib_device *ib_dev) -{ - if (!smbdirect_frwr_is_supported(&ib_dev->attrs)) - return RDMA_NODE_UNSPECIFIED; - - switch (ib_dev->node_type) { - case RDMA_NODE_IB_CA: /* Infiniband, RoCE v1 and v2 */ - case RDMA_NODE_RNIC: /* iWarp */ - return ib_dev->node_type; - } - - return RDMA_NODE_UNSPECIFIED; -} - -static int smbdirect_ib_client_add(struct ib_device *ib_dev) -{ - u8 node_type = smbdirect_ib_device_rdma_capable_node_type(ib_dev); - struct smbdirect_device *sdev; - const char *node_str; - const char *action; - u32 pidx; - - switch (node_type) { - case RDMA_NODE_IB_CA: - node_str = "IB_CA"; - action = "added"; - break; - case RDMA_NODE_RNIC: - node_str = "RNIC"; - action = "added"; - break; - case RDMA_NODE_UNSPECIFIED: - node_str = "UNSPECIFIED"; - action = "ignored"; - break; - default: - node_str = "UNKNOWN"; - action = "ignored"; - node_type = RDMA_NODE_UNSPECIFIED; - break; - } - - pr_info("ib_dev[%.*s]: %s: %s %s=%u %s=0x%llx %s=0x%llx %s=0x%llx\n", - IB_DEVICE_NAME_MAX, - ib_dev->name, - action, - node_str, - "max_fast_reg_page_list_len", - ib_dev->attrs.max_fast_reg_page_list_len, - "device_cap_flags", - ib_dev->attrs.device_cap_flags, - "kernel_cap_flags", - ib_dev->attrs.kernel_cap_flags, - "page_size_cap", - ib_dev->attrs.page_size_cap); - - if (node_type == RDMA_NODE_UNSPECIFIED) - return 0; - - pr_info("ib_dev[%.*s]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u\n", - IB_DEVICE_NAME_MAX, - ib_dev->name, - "num_ports", - rdma_end_port(ib_dev), - "max_qp_rd_atom", - ib_dev->attrs.max_qp_rd_atom, - "max_qp_init_rd_atom", - ib_dev->attrs.max_qp_init_rd_atom, - "max_sgl_rd", - ib_dev->attrs.max_sgl_rd, - "max_sge_rd", - ib_dev->attrs.max_sge_rd, - "max_cqe", - ib_dev->attrs.max_cqe, - "max_qp_wr", - ib_dev->attrs.max_qp_wr, - "max_send_sge", - ib_dev->attrs.max_send_sge, - "max_recv_sge", - ib_dev->attrs.max_recv_sge); - - rdma_for_each_port(ib_dev, pidx) { - const struct ib_port_immutable *ib_pi = - ib_port_immutable_read(ib_dev, pidx); - u32 core_cap_flags = ib_pi ? ib_pi->core_cap_flags : 0; - - pr_info("ib_dev[%.*s]PORT[%u]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=0x%x\n", - IB_DEVICE_NAME_MAX, - ib_dev->name, - pidx, - "iwarp", - rdma_protocol_iwarp(ib_dev, pidx), - "ib", - rdma_protocol_ib(ib_dev, pidx), - "roce", - rdma_protocol_roce(ib_dev, pidx), - "v1", - rdma_protocol_roce_eth_encap(ib_dev, pidx), - "v2", - rdma_protocol_roce_udp_encap(ib_dev, pidx), - "core_cap_flags", - core_cap_flags); - } - - sdev = kzalloc_obj(*sdev); - if (!sdev) - return -ENOMEM; - sdev->ib_dev = ib_dev; - snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", - IB_DEVICE_NAME_MAX, ib_dev->name); - - write_lock(&smbdirect_globals.devices.lock); - list_add(&sdev->list, &smbdirect_globals.devices.list); - write_unlock(&smbdirect_globals.devices.lock); - - return 0; -} - -static void smbdirect_ib_client_remove(struct ib_device *ib_dev, void *client_data) -{ - struct smbdirect_device *sdev, *tmp; - - write_lock(&smbdirect_globals.devices.lock); - list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { - if (sdev->ib_dev == ib_dev) { - list_del(&sdev->list); - pr_info("ib_dev[%.*s] removed\n", - IB_DEVICE_NAME_MAX, sdev->ib_name); - kfree(sdev); - break; - } - } - write_unlock(&smbdirect_globals.devices.lock); -} - -static void smbdirect_ib_client_rename(struct ib_device *ib_dev, void *client_data) -{ - struct smbdirect_device *sdev; - - write_lock(&smbdirect_globals.devices.lock); - list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { - if (sdev->ib_dev == ib_dev) { - pr_info("ib_dev[%.*s] renamed to [%.*s]\n", - IB_DEVICE_NAME_MAX, sdev->ib_name, - IB_DEVICE_NAME_MAX, ib_dev->name); - snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", - IB_DEVICE_NAME_MAX, ib_dev->name); - break; - } - } - write_unlock(&smbdirect_globals.devices.lock); -} - -static struct ib_client smbdirect_ib_client = { - .name = "smbdirect_ib_client", - .add = smbdirect_ib_client_add, - .remove = smbdirect_ib_client_remove, - .rename = smbdirect_ib_client_rename, -}; - -static u8 smbdirect_netdev_find_rdma_capable_node_type(struct net_device *netdev) -{ - struct smbdirect_device *sdev; - u8 node_type = RDMA_NODE_UNSPECIFIED; - - read_lock(&smbdirect_globals.devices.lock); - list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { - u32 pi; - - rdma_for_each_port(sdev->ib_dev, pi) { - struct net_device *ndev; - - ndev = ib_device_get_netdev(sdev->ib_dev, pi); - if (!ndev) - continue; - - if (ndev == netdev) { - dev_put(ndev); - node_type = sdev->ib_dev->node_type; - goto out; - } - dev_put(ndev); - } - } -out: - read_unlock(&smbdirect_globals.devices.lock); - - if (node_type == RDMA_NODE_UNSPECIFIED) { - struct ib_device *ibdev; - - ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); - if (ibdev) { - node_type = smbdirect_ib_device_rdma_capable_node_type(ibdev); - ib_device_put(ibdev); - } - } - - return node_type; -} - -/* - * Returns RDMA_NODE_UNSPECIFIED when the netdev has - * no support for smbdirect capable rdma. - * - * Otherwise RDMA_NODE_RNIC is returned for iwarp devices - * and RDMA_NODE_IB_CA or Infiniband and RoCE (v1 and v2) - */ -u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev) -{ - struct net_device *lower_dev; - struct list_head *iter; - u8 node_type = RDMA_NODE_UNSPECIFIED; - - node_type = smbdirect_netdev_find_rdma_capable_node_type(netdev); - if (node_type != RDMA_NODE_UNSPECIFIED) - return node_type; - - /* check if netdev is bridge or VLAN */ - if (netif_is_bridge_master(netdev) || netdev->priv_flags & IFF_802_1Q_VLAN) - netdev_for_each_lower_dev(netdev, lower_dev, iter) { - node_type = smbdirect_netdev_find_rdma_capable_node_type(lower_dev); - if (node_type != RDMA_NODE_UNSPECIFIED) - return node_type; - } - - /* check if netdev is IPoIB safely without layer violation */ - if (netdev->type == ARPHRD_INFINIBAND) - return RDMA_NODE_IB_CA; - - return RDMA_NODE_UNSPECIFIED; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_netdev_rdma_capable_node_type); - -__init int smbdirect_devices_init(void) -{ - int ret; - - rwlock_init(&smbdirect_globals.devices.lock); - INIT_LIST_HEAD(&smbdirect_globals.devices.list); - - ret = ib_register_client(&smbdirect_ib_client); - if (ret) { - pr_crit("failed to ib_register_client: %d %1pe\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - return 0; -} - -__exit void smbdirect_devices_exit(void) -{ - struct smbdirect_device *sdev, *tmp; - - /* - * On exist we just cleanup so that - * smbdirect_ib_client_remove() won't - * print removals of devices. - */ - write_lock(&smbdirect_globals.devices.lock); - list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { - list_del(&sdev->list); - kfree(sdev); - } - write_unlock(&smbdirect_globals.devices.lock); - - ib_unregister_client(&smbdirect_ib_client); -} diff --git a/fs/smb/common/smbdirect/smbdirect_internal.h b/fs/smb/common/smbdirect/smbdirect_internal.h deleted file mode 100644 index 30a1b8643657..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_internal.h +++ /dev/null @@ -1,141 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2025, Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ -#define __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include "smbdirect.h" -#include "smbdirect_pdu.h" -#include "smbdirect_public.h" - -#include - -struct smbdirect_module_state { - struct mutex mutex; - - struct { - struct workqueue_struct *accept; - struct workqueue_struct *connect; - struct workqueue_struct *idle; - struct workqueue_struct *refill; - struct workqueue_struct *immediate; - struct workqueue_struct *cleanup; - } workqueues; - - struct { - rwlock_t lock; - struct list_head list; - } devices; -}; - -extern struct smbdirect_module_state smbdirect_globals; - -#include "smbdirect_socket.h" - -struct smbdirect_device { - struct list_head list; - struct ib_device *ib_dev; - /* - * copy of ib_dev->name, - * in order to print renames - */ - char ib_name[IB_DEVICE_NAME_MAX]; -}; - -int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc); - -int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc); - -void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, - const char *macro_name, - unsigned int lvl, - const char *func, - unsigned int line, - int error, - enum smbdirect_socket_status *force_status); -#define smbdirect_socket_schedule_cleanup(__sc, __error) \ - __smbdirect_socket_schedule_cleanup(__sc, \ - "smbdirect_socket_schedule_cleanup", SMBDIRECT_LOG_ERR, \ - __func__, __LINE__, __error, NULL) -#define smbdirect_socket_schedule_cleanup_lvl(__sc, __lvl, __error) \ - __smbdirect_socket_schedule_cleanup(__sc, \ - "smbdirect_socket_schedule_cleanup_lvl", __lvl, \ - __func__, __LINE__, __error, NULL) -#define smbdirect_socket_schedule_cleanup_status(__sc, __lvl, __error, __status) do { \ - enum smbdirect_socket_status __force_status = __status; \ - __smbdirect_socket_schedule_cleanup(__sc, \ - "smbdirect_socket_schedule_cleanup_status", __lvl, \ - __func__, __LINE__, __error, &__force_status); \ -} while (0) - -void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc); - -int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, - enum smbdirect_socket_status expected_status, - int unexpected_errno, - wait_queue_head_t *waitq, - atomic_t *total_credits, - int needed); - -void smbdirect_connection_rdma_established(struct smbdirect_socket *sc); - -void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc); - -int smbdirect_connection_create_qp(struct smbdirect_socket *sc); - -void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc); - -int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc); - -void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc); - -struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc); - -void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg); - -struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc); - -void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg); - -void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc, - struct smbdirect_recv_io *msg, - u32 data_length); - -struct smbdirect_recv_io * -smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc); - -void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc, - u8 peer_initiator_depth, - u8 peer_responder_resources, - const struct rdma_conn_param *param); - -void smbdirect_connection_idle_timer_work(struct work_struct *work); - -u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc); - -int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc, - struct ib_send_wr *wr); - -int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg); - -void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc); - -int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc); - -int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc); - -void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc); - -int smbdirect_accept_connect_request(struct smbdirect_socket *sc, - const struct rdma_conn_param *param); - -void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus); - -__init int smbdirect_devices_init(void); -__exit void smbdirect_devices_exit(void); - -#endif /* __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ */ diff --git a/fs/smb/common/smbdirect/smbdirect_listen.c b/fs/smb/common/smbdirect/smbdirect_listen.c deleted file mode 100644 index 05c7902e7020..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_listen.c +++ /dev/null @@ -1,308 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (C) 2018, LG Electronics. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" - -static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event); - -int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog) -{ - int ret; - - if (backlog < 0) - return -EINVAL; - if (!backlog) - backlog = 1; /* use 1 as default for now */ - - if (sc->first_error) - return -EINVAL; - - if (sc->status != SMBDIRECT_SOCKET_CREATED) - return -EINVAL; - - if (WARN_ON_ONCE(!sc->rdma.cm_id)) - return -EINVAL; - - if (sc->rdma.cm_id->device) - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "try to listen on addr: %pISpsfc dev: %.*s\n", - &sc->rdma.cm_id->route.addr.src_addr, - IB_DEVICE_NAME_MAX, - sc->rdma.cm_id->device->name); - else - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "try to listen on addr: %pISpsfc\n", - &sc->rdma.cm_id->route.addr.src_addr); - - /* already checked above */ - WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); - sc->status = SMBDIRECT_SOCKET_LISTENING; - sc->rdma.expected_event = RDMA_CM_EVENT_CONNECT_REQUEST; - rdma_lock_handler(sc->rdma.cm_id); - sc->rdma.cm_id->event_handler = smbdirect_listen_rdma_event_handler; - rdma_unlock_handler(sc->rdma.cm_id); - - ret = rdma_listen(sc->rdma.cm_id, backlog); - if (ret) { - sc->first_error = ret; - sc->status = SMBDIRECT_SOCKET_DISCONNECTED; - if (sc->rdma.cm_id->device) - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "listening failed %1pe on addr: %pISpsfc dev: %.*s\n", - SMBDIRECT_DEBUG_ERR_PTR(ret), - &sc->rdma.cm_id->route.addr.src_addr, - IB_DEVICE_NAME_MAX, - sc->rdma.cm_id->device->name); - else - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "listening failed %1pe on addr: %pISpsfc\n", - SMBDIRECT_DEBUG_ERR_PTR(ret), - &sc->rdma.cm_id->route.addr.src_addr); - return ret; - } - - /* - * This is a value > 0, checked above, - * so we are able to use sc->listen.backlog == -1, - * as indication that the socket was never - * a listener. - */ - sc->listen.backlog = backlog; - - if (sc->rdma.cm_id->device) - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "listening on addr: %pISpsfc dev: %.*s\n", - &sc->rdma.cm_id->route.addr.src_addr, - IB_DEVICE_NAME_MAX, - sc->rdma.cm_id->device->name); - else - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "listening on addr: %pISpsfc\n", - &sc->rdma.cm_id->route.addr.src_addr); - - /* - * The rest happens async via smbdirect_listen_rdma_event_handler() - */ - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_listen); - -static int smbdirect_new_rdma_event_handler(struct rdma_cm_id *new_id, - struct rdma_cm_event *event) -{ - int ret = -ESTALE; - - /* - * This should be replaced before any real work - * starts! So it should never be called! - */ - - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ret = -ENETDOWN; - if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) - ret = event->status; - WARN_ONCE(1, - "%s should not be called! event=%s status=%d => ret=%1pe\n", - __func__, - rdma_event_msg(event->event), - event->status, - SMBDIRECT_DEBUG_ERR_PTR(ret)); - return -ESTALE; -} - -static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc, - struct rdma_cm_id *new_id, - const struct rdma_cm_event *event); - -static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *new_id, - struct rdma_cm_event *event) -{ - struct smbdirect_socket *lsc = new_id->context; - int ret; - - if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { - new_id->context = NULL; - new_id->event_handler = smbdirect_new_rdma_event_handler; - } else - new_id = NULL; - - /* - * cma_cm_event_handler() has - * lockdep_assert_held(&id_priv->handler_mutex); - * - * Mutexes are not allowed in interrupts, - * and we rely on not being in an interrupt here, - * as we might sleep. - */ - WARN_ON_ONCE(in_interrupt()); - - if (event->status || event->event != lsc->rdma.expected_event) { - ret = -ECONNABORTED; - - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ret = -ENETDOWN; - if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) - ret = event->status; - - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, - "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", - smbdirect_socket_status_string(lsc->status), - SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error), - rdma_event_msg(lsc->rdma.expected_event), - rdma_event_msg(event->event), - event->status, - SMBDIRECT_DEBUG_ERR_PTR(ret)); - - /* - * In case of error return it and let the caller - * destroy new_id - */ - smbdirect_socket_schedule_cleanup(lsc, ret); - return new_id ? ret : 0; - } - - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_INFO, - "%s (first_error=%1pe) event=%s\n", - smbdirect_socket_status_string(lsc->status), - SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error), - rdma_event_msg(event->event)); - - /* - * In case of error return it and let the caller - * destroy new_id - */ - if (lsc->first_error) - return new_id ? lsc->first_error : 0; - - switch (event->event) { - case RDMA_CM_EVENT_CONNECT_REQUEST: - WARN_ON_ONCE(lsc->status != SMBDIRECT_SOCKET_LISTENING); - - /* - * In case of error return it and let the caller - * destroy new_id - */ - ret = smbdirect_listen_connect_request(lsc, new_id, event); - if (ret) - return ret; - return 0; - - default: - break; - } - - /* - * This is an internal error - */ - WARN_ON_ONCE(lsc->rdma.expected_event != RDMA_CM_EVENT_CONNECT_REQUEST); - smbdirect_socket_schedule_cleanup(lsc, -EINVAL); - return 0; -} - -static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc, - struct rdma_cm_id *new_id, - const struct rdma_cm_event *event) -{ - const struct smbdirect_socket_parameters *lsp = &lsc->parameters; - struct smbdirect_socket *nsc; - unsigned long flags; - size_t backlog = max_t(size_t, 1, lsc->listen.backlog); - size_t psockets; - size_t rsockets; - int ret; - - if (!smbdirect_frwr_is_supported(&new_id->device->attrs)) { - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, - "Fast Registration Work Requests (FRWR) is not supported device %.*s\n", - IB_DEVICE_NAME_MAX, - new_id->device->name); - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, - "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", - new_id->device->attrs.device_cap_flags, - new_id->device->attrs.max_fast_reg_page_list_len); - return -EPROTONOSUPPORT; - } - - if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && - !rdma_ib_or_roce(new_id->device, new_id->port_num)) { - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, - "Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, - new_id->device->name, - rdma_protocol_iwarp(new_id->device, new_id->port_num), - &new_id->route.addr.src_addr, - &new_id->route.addr.dst_addr); - return -EPROTONOSUPPORT; - } - if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW && - !rdma_protocol_iwarp(new_id->device, new_id->port_num)) { - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, - "Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n", - IB_DEVICE_NAME_MAX, - new_id->device->name, - rdma_ib_or_roce(new_id->device, new_id->port_num), - &new_id->route.addr.src_addr, - &new_id->route.addr.dst_addr); - return -EPROTONOSUPPORT; - } - - spin_lock_irqsave(&lsc->listen.lock, flags); - psockets = list_count_nodes(&lsc->listen.pending); - rsockets = list_count_nodes(&lsc->listen.ready); - spin_unlock_irqrestore(&lsc->listen.lock, flags); - - if (psockets > backlog || - rsockets > backlog || - (psockets + rsockets) > backlog) { - smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, - "Backlog[%d][%zu] full pending[%zu] ready[%zu]\n", - lsc->listen.backlog, backlog, psockets, rsockets); - return -EBUSY; - } - - ret = smbdirect_socket_create_accepting(new_id, &nsc); - if (ret) - goto socket_init_failed; - - nsc->logging = lsc->logging; - ret = smbdirect_socket_set_initial_parameters(nsc, &lsc->parameters); - if (ret) - goto set_params_failed; - ret = smbdirect_socket_set_kernel_settings(nsc, - lsc->ib.poll_ctx, - lsc->send_io.mem.gfp_mask); - if (ret) - goto set_settings_failed; - - spin_lock_irqsave(&lsc->listen.lock, flags); - list_add_tail(&nsc->accept.list, &lsc->listen.pending); - nsc->accept.listener = lsc; - spin_unlock_irqrestore(&lsc->listen.lock, flags); - - ret = smbdirect_accept_connect_request(nsc, &event->param.conn); - if (ret) - goto accept_connect_failed; - - return 0; - -accept_connect_failed: - spin_lock_irqsave(&lsc->listen.lock, flags); - list_del_init(&nsc->accept.list); - nsc->accept.listener = NULL; - spin_unlock_irqrestore(&lsc->listen.lock, flags); -set_settings_failed: -set_params_failed: - /* - * The caller will destroy new_id - */ - nsc->ib.dev = NULL; - nsc->rdma.cm_id = NULL; - smbdirect_socket_release(nsc); -socket_init_failed: - return ret; -} diff --git a/fs/smb/common/smbdirect/smbdirect_main.c b/fs/smb/common/smbdirect/smbdirect_main.c deleted file mode 100644 index fe6e8d93c34c..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_main.c +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" -#include - -struct smbdirect_module_state smbdirect_globals = { - .mutex = __MUTEX_INITIALIZER(smbdirect_globals.mutex), -}; - -static __init int smbdirect_module_init(void) -{ - int ret = -ENOMEM; - - pr_notice("subsystem loading...\n"); - mutex_lock(&smbdirect_globals.mutex); - - smbdirect_globals.workqueues.accept = alloc_workqueue("smbdirect-accept", - WQ_SYSFS | - WQ_PERCPU | - WQ_POWER_EFFICIENT, - 0); - if (smbdirect_globals.workqueues.accept == NULL) - goto alloc_accept_wq_failed; - - smbdirect_globals.workqueues.connect = alloc_workqueue("smbdirect-connect", - WQ_SYSFS | - WQ_PERCPU | - WQ_POWER_EFFICIENT, - 0); - if (smbdirect_globals.workqueues.connect == NULL) - goto alloc_connect_wq_failed; - - smbdirect_globals.workqueues.idle = alloc_workqueue("smbdirect-idle", - WQ_SYSFS | - WQ_PERCPU | - WQ_POWER_EFFICIENT, - 0); - if (smbdirect_globals.workqueues.idle == NULL) - goto alloc_idle_wq_failed; - - smbdirect_globals.workqueues.refill = alloc_workqueue("smbdirect-refill", - WQ_HIGHPRI | - WQ_SYSFS | - WQ_PERCPU | - WQ_POWER_EFFICIENT, - 0); - if (smbdirect_globals.workqueues.refill == NULL) - goto alloc_refill_wq_failed; - - smbdirect_globals.workqueues.immediate = alloc_workqueue("smbdirect-immediate", - WQ_HIGHPRI | - WQ_SYSFS | - WQ_PERCPU | - WQ_POWER_EFFICIENT, - 0); - if (smbdirect_globals.workqueues.immediate == NULL) - goto alloc_immediate_wq_failed; - - smbdirect_globals.workqueues.cleanup = alloc_workqueue("smbdirect-cleanup", - WQ_MEM_RECLAIM | - WQ_HIGHPRI | - WQ_SYSFS | - WQ_PERCPU | - WQ_POWER_EFFICIENT, - 0); - if (smbdirect_globals.workqueues.cleanup == NULL) - goto alloc_cleanup_wq_failed; - - ret = smbdirect_devices_init(); - if (ret) - goto devices_init_failed; - - mutex_unlock(&smbdirect_globals.mutex); - pr_notice("subsystem loaded\n"); - return 0; - -devices_init_failed: - destroy_workqueue(smbdirect_globals.workqueues.cleanup); -alloc_cleanup_wq_failed: - destroy_workqueue(smbdirect_globals.workqueues.immediate); -alloc_immediate_wq_failed: - destroy_workqueue(smbdirect_globals.workqueues.refill); -alloc_refill_wq_failed: - destroy_workqueue(smbdirect_globals.workqueues.idle); -alloc_idle_wq_failed: - destroy_workqueue(smbdirect_globals.workqueues.connect); -alloc_connect_wq_failed: - destroy_workqueue(smbdirect_globals.workqueues.accept); -alloc_accept_wq_failed: - mutex_unlock(&smbdirect_globals.mutex); - pr_crit("failed to loaded: %d (%1pe)\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; -} - -static __exit void smbdirect_module_exit(void) -{ - pr_notice("subsystem unloading...\n"); - mutex_lock(&smbdirect_globals.mutex); - - smbdirect_devices_exit(); - - destroy_workqueue(smbdirect_globals.workqueues.accept); - destroy_workqueue(smbdirect_globals.workqueues.connect); - destroy_workqueue(smbdirect_globals.workqueues.idle); - destroy_workqueue(smbdirect_globals.workqueues.refill); - destroy_workqueue(smbdirect_globals.workqueues.immediate); - destroy_workqueue(smbdirect_globals.workqueues.cleanup); - - mutex_unlock(&smbdirect_globals.mutex); - pr_notice("subsystem unloaded\n"); -} - -module_init(smbdirect_module_init); -module_exit(smbdirect_module_exit); - -MODULE_DESCRIPTION("smbdirect subsystem"); -MODULE_LICENSE("GPL"); diff --git a/fs/smb/common/smbdirect/smbdirect_mr.c b/fs/smb/common/smbdirect/smbdirect_mr.c deleted file mode 100644 index fa9be8089925..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_mr.c +++ /dev/null @@ -1,493 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" - -/* - * Allocate MRs used for RDMA read/write - * The number of MRs will not exceed hardware capability in responder_resources - * All MRs are kept in mr_list. The MR can be recovered after it's used - * Recovery is done in smbd_mr_recovery_work. The content of list entry changes - * as MRs are used and recovered for I/O, but the list links will not change - */ -int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_mr_io *mr; - int ret; - u32 i; - - if (sp->responder_resources == 0) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "responder_resources negotiated as 0\n"); - return -EINVAL; - } - - /* Allocate more MRs (2x) than hardware responder_resources */ - for (i = 0; i < sp->responder_resources * 2; i++) { - mr = kzalloc_obj(*mr); - if (!mr) { - ret = -ENOMEM; - goto kzalloc_mr_failed; - } - - kref_init(&mr->kref); - mutex_init(&mr->mutex); - - mr->mr = ib_alloc_mr(sc->ib.pd, - sc->mr_io.type, - sp->max_frmr_depth); - if (IS_ERR(mr->mr)) { - ret = PTR_ERR(mr->mr); - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_alloc_mr failed ret=%d (%1pe) type=0x%x max_frmr_depth=%u\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret), - sc->mr_io.type, sp->max_frmr_depth); - goto ib_alloc_mr_failed; - } - mr->sgt.sgl = kzalloc_objs(struct scatterlist, sp->max_frmr_depth); - if (!mr->sgt.sgl) { - ret = -ENOMEM; - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "failed to allocate sgl, max_frmr_depth=%u\n", - sp->max_frmr_depth); - goto kcalloc_sgl_failed; - } - mr->state = SMBDIRECT_MR_READY; - mr->socket = sc; - - list_add_tail(&mr->list, &sc->mr_io.all.list); - atomic_inc(&sc->mr_io.ready.count); - } - - return 0; - -kcalloc_sgl_failed: - ib_dereg_mr(mr->mr); -ib_alloc_mr_failed: - mutex_destroy(&mr->mutex); - kfree(mr); -kzalloc_mr_failed: - smbdirect_connection_destroy_mr_list(sc); - return ret; -} - -static void smbdirect_mr_io_disable_locked(struct smbdirect_mr_io *mr) -{ - struct smbdirect_socket *sc = mr->socket; - - lockdep_assert_held(&mr->mutex); - - if (mr->state == SMBDIRECT_MR_DISABLED) - return; - - if (mr->mr) - ib_dereg_mr(mr->mr); - if (mr->sgt.nents) - ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); - kfree(mr->sgt.sgl); - - mr->mr = NULL; - mr->sgt.sgl = NULL; - mr->sgt.nents = 0; - - mr->state = SMBDIRECT_MR_DISABLED; -} - -static void smbdirect_mr_io_free_locked(struct kref *kref) -{ - struct smbdirect_mr_io *mr = - container_of(kref, struct smbdirect_mr_io, kref); - - lockdep_assert_held(&mr->mutex); - - /* - * smbdirect_mr_io_disable_locked() should already be called! - */ - if (WARN_ON_ONCE(mr->state != SMBDIRECT_MR_DISABLED)) - smbdirect_mr_io_disable_locked(mr); - - mutex_unlock(&mr->mutex); - mutex_destroy(&mr->mutex); - kfree(mr); -} - -void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc) -{ - struct smbdirect_mr_io *mr, *tmp; - LIST_HEAD(all_list); - unsigned long flags; - - spin_lock_irqsave(&sc->mr_io.all.lock, flags); - list_splice_tail_init(&sc->mr_io.all.list, &all_list); - spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); - - list_for_each_entry_safe(mr, tmp, &all_list, list) { - mutex_lock(&mr->mutex); - - smbdirect_mr_io_disable_locked(mr); - list_del(&mr->list); - mr->socket = NULL; - - /* - * No kref_put_mutex() as it's already locked. - * - * If smbdirect_mr_io_free_locked() is called - * and the mutex is unlocked and mr is gone, - * in that case kref_put() returned 1. - * - * If kref_put() returned 0 we know that - * smbdirect_mr_io_free_locked() didn't - * run. Not by us nor by anyone else, as we - * still hold the mutex, so we need to unlock. - * - * If the mr is still registered it will - * be dangling (detached from the connection - * waiting for smbd_deregister_mr() to be - * called in order to free the memory. - */ - if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) - mutex_unlock(&mr->mutex); - } -} - -/* - * Get a MR from mr_list. This function waits until there is at least one MR - * available in the list. There may be several CPUs issuing I/O trying to get MR - * at the same time, mr_list_lock is used to protect this situation. - */ -static struct smbdirect_mr_io * -smbdirect_connection_get_mr_io(struct smbdirect_socket *sc) -{ - struct smbdirect_mr_io *mr; - unsigned long flags; - int ret; - -again: - ret = wait_event_interruptible(sc->mr_io.ready.wait_queue, - atomic_read(&sc->mr_io.ready.count) || - sc->status != SMBDIRECT_SOCKET_CONNECTED); - if (ret) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "wait_event_interruptible ret=%d (%1pe)\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - return NULL; - } - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "sc->status=%s sc->first_error=%1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - return NULL; - } - - spin_lock_irqsave(&sc->mr_io.all.lock, flags); - list_for_each_entry(mr, &sc->mr_io.all.list, list) { - if (mr->state == SMBDIRECT_MR_READY) { - mr->state = SMBDIRECT_MR_REGISTERED; - kref_get(&mr->kref); - spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); - atomic_dec(&sc->mr_io.ready.count); - atomic_inc(&sc->mr_io.used.count); - return mr; - } - } - - spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); - /* - * It is possible that we could fail to get MR because other processes may - * try to acquire a MR at the same time. If this is the case, retry it. - */ - goto again; -} - -static void smbdirect_connection_mr_io_register_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_mr_io *mr = - container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); - struct smbdirect_socket *sc = mr->socket; - - if (wc->status != IB_WC_SUCCESS) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "wc->status=%s opcode=%d\n", - ib_wc_status_msg(wc->status), wc->opcode); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - } -} - -static void smbdirect_connection_mr_io_local_inv_done(struct ib_cq *cq, struct ib_wc *wc) -{ - struct smbdirect_mr_io *mr = - container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); - struct smbdirect_socket *sc = mr->socket; - - mr->state = SMBDIRECT_MR_INVALIDATED; - if (wc->status != IB_WC_SUCCESS) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "invalidate failed status=%s\n", - ib_wc_status_msg(wc->status)); - smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); - } - complete(&mr->invalidate_done); -} - -/* - * Transcribe the pages from an iterator into an MR scatterlist. - */ -static int smbdirect_iter_to_sgt(struct iov_iter *iter, - struct sg_table *sgt, - unsigned int max_sg) -{ - int ret; - - memset(sgt->sgl, 0, max_sg * sizeof(struct scatterlist)); - - ret = extract_iter_to_sg(iter, iov_iter_count(iter), sgt, max_sg, 0); - WARN_ON(ret < 0); - if (sgt->nents > 0) - sg_mark_end(&sgt->sgl[sgt->nents - 1]); - - return ret; -} - -/* - * Register memory for RDMA read/write - * iter: the buffer to register memory with - * writing: true if this is a RDMA write (SMB read), false for RDMA read - * need_invalidate: true if this MR needs to be locally invalidated after I/O - * return value: the MR registered, NULL if failed. - */ -struct smbdirect_mr_io * -smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, - struct iov_iter *iter, - bool writing, - bool need_invalidate) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_mr_io *mr; - int ret, num_pages; - struct ib_reg_wr *reg_wr; - - num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1); - if (num_pages > sp->max_frmr_depth) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "num_pages=%d max_frmr_depth=%d\n", - num_pages, sp->max_frmr_depth); - WARN_ON_ONCE(1); - return NULL; - } - - mr = smbdirect_connection_get_mr_io(sc); - if (!mr) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "smbdirect_connection_get_mr_io returning NULL\n"); - return NULL; - } - - mutex_lock(&mr->mutex); - - mr->dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; - mr->need_invalidate = need_invalidate; - mr->sgt.nents = 0; - mr->sgt.orig_nents = 0; - - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, - "num_pages=%u count=%zu depth=%u\n", - num_pages, iov_iter_count(iter), sp->max_frmr_depth); - smbdirect_iter_to_sgt(iter, &mr->sgt, sp->max_frmr_depth); - - ret = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); - if (!ret) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_dma_map_sg num_pages=%u dir=%x ret=%d (%1pe)\n", - num_pages, mr->dir, ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - goto dma_map_error; - } - - ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE); - if (ret != mr->sgt.nents) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_map_mr_sg failed ret = %d nents = %u\n", - ret, mr->sgt.nents); - goto map_mr_error; - } - - ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); - reg_wr = &mr->wr; - reg_wr->wr.opcode = IB_WR_REG_MR; - mr->cqe.done = smbdirect_connection_mr_io_register_done; - reg_wr->wr.wr_cqe = &mr->cqe; - reg_wr->wr.num_sge = 0; - reg_wr->wr.send_flags = IB_SEND_SIGNALED; - reg_wr->mr = mr->mr; - reg_wr->key = mr->mr->rkey; - reg_wr->access = writing ? - IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : - IB_ACCESS_REMOTE_READ; - - /* - * There is no need for waiting for complemtion on ib_post_send - * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution - * on the next ib_post_send when we actually send I/O to remote peer - */ - ret = ib_post_send(sc->ib.qp, ®_wr->wr, NULL); - if (!ret) { - /* - * smbdirect_connection_get_mr_io() gave us a reference - * via kref_get(&mr->kref), we keep that and let - * the caller use smbdirect_connection_deregister_mr_io() - * to remove it again. - */ - mutex_unlock(&mr->mutex); - return mr; - } - - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_post_send failed ret=%d (%1pe) reg_wr->key=0x%x\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret), reg_wr->key); - -map_mr_error: - ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); - -dma_map_error: - mr->sgt.nents = 0; - mr->state = SMBDIRECT_MR_ERROR; - atomic_dec(&sc->mr_io.used.count); - - smbdirect_socket_schedule_cleanup(sc, ret); - - /* - * smbdirect_connection_get_mr_io() gave us a reference - * via kref_get(&mr->kref), we need to remove it again - * on error. - * - * No kref_put_mutex() as it's already locked. - * - * If smbdirect_mr_io_free_locked() is called - * and the mutex is unlocked and mr is gone, - * in that case kref_put() returned 1. - * - * If kref_put() returned 0 we know that - * smbdirect_mr_io_free_locked() didn't - * run. Not by us nor by anyone else, as we - * still hold the mutex, so we need to unlock. - */ - if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) - mutex_unlock(&mr->mutex); - return NULL; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_register_mr_io); - -void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, - struct smbdirect_buffer_descriptor_v1 *v1) -{ - mutex_lock(&mr->mutex); - if (mr->state == SMBDIRECT_MR_REGISTERED) { - v1->offset = cpu_to_le64(mr->mr->iova); - v1->token = cpu_to_le32(mr->mr->rkey); - v1->length = cpu_to_le32(mr->mr->length); - } else { - v1->offset = cpu_to_le64(U64_MAX); - v1->token = cpu_to_le32(U32_MAX); - v1->length = cpu_to_le32(U32_MAX); - } - mutex_unlock(&mr->mutex); -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_mr_io_fill_buffer_descriptor); - -/* - * Deregister a MR after I/O is done - * This function may wait if remote invalidation is not used - * and we have to locally invalidate the buffer to prevent data is being - * modified by remote peer after upper layer consumes it - */ -void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr) -{ - struct smbdirect_socket *sc = mr->socket; - int ret = 0; - -lock_again: - mutex_lock(&mr->mutex); - if (mr->state == SMBDIRECT_MR_DISABLED) - goto put_kref; - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { - smbdirect_mr_io_disable_locked(mr); - goto put_kref; - } - - if (mr->need_invalidate) { - struct ib_send_wr *wr = &mr->inv_wr; - - /* Need to finish local invalidation before returning */ - wr->opcode = IB_WR_LOCAL_INV; - mr->cqe.done = smbdirect_connection_mr_io_local_inv_done; - wr->wr_cqe = &mr->cqe; - wr->num_sge = 0; - wr->ex.invalidate_rkey = mr->mr->rkey; - wr->send_flags = IB_SEND_SIGNALED; - - init_completion(&mr->invalidate_done); - ret = ib_post_send(sc->ib.qp, wr, NULL); - if (ret) { - smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, - "ib_post_send failed ret=%d (%1pe)\n", - ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); - smbdirect_mr_io_disable_locked(mr); - smbdirect_socket_schedule_cleanup(sc, ret); - goto done; - } - - /* - * We still hold the reference to mr - * so we can unlock while waiting. - */ - mutex_unlock(&mr->mutex); - wait_for_completion(&mr->invalidate_done); - mr->need_invalidate = false; - goto lock_again; - } else - /* - * For remote invalidation, just set it to SMBDIRECT_MR_INVALIDATED - * and defer to mr_recovery_work to recover the MR for next use - */ - mr->state = SMBDIRECT_MR_INVALIDATED; - - if (mr->sgt.nents) { - ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); - mr->sgt.nents = 0; - } - - WARN_ONCE(mr->state != SMBDIRECT_MR_INVALIDATED, - "mr->state[%u] != SMBDIRECT_MR_INVALIDATED[%u]\n", - mr->state, SMBDIRECT_MR_INVALIDATED); - mr->state = SMBDIRECT_MR_READY; - if (atomic_inc_return(&sc->mr_io.ready.count) == 1) - wake_up(&sc->mr_io.ready.wait_queue); - -done: - atomic_dec(&sc->mr_io.used.count); - -put_kref: - /* - * No kref_put_mutex() as it's already locked. - * - * If smbdirect_mr_io_free_locked() is called - * and the mutex is unlocked and mr is gone, - * in that case kref_put() returned 1. - * - * If kref_put() returned 0 we know that - * smbdirect_mr_io_free_locked() didn't - * run. Not by us nor by anyone else, as we - * still hold the mutex, so we need to unlock - * and keep the mr in SMBDIRECT_MR_READY or - * SMBDIRECT_MR_ERROR state. - */ - if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) - mutex_unlock(&mr->mutex); -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_deregister_mr_io); diff --git a/fs/smb/common/smbdirect/smbdirect_pdu.h b/fs/smb/common/smbdirect/smbdirect_pdu.h deleted file mode 100644 index 7693ba337873..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_pdu.h +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2017 Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ -#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ - -#define SMBDIRECT_V1 0x0100 - -/* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */ -#define SMBDIRECT_MIN_RECEIVE_SIZE 128 -#define SMBDIRECT_MIN_FRAGMENTED_SIZE 131072 - -/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */ -struct smbdirect_negotiate_req { - __le16 min_version; - __le16 max_version; - __le16 reserved; - __le16 credits_requested; - __le32 preferred_send_size; - __le32 max_receive_size; - __le32 max_fragmented_size; -} __packed; - -/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */ -struct smbdirect_negotiate_resp { - __le16 min_version; - __le16 max_version; - __le16 negotiated_version; - __le16 reserved; - __le16 credits_requested; - __le16 credits_granted; - __le32 status; - __le32 max_readwrite_size; - __le32 preferred_send_size; - __le32 max_receive_size; - __le32 max_fragmented_size; -} __packed; - -#define SMBDIRECT_DATA_MIN_HDR_SIZE 0x14 -#define SMBDIRECT_DATA_OFFSET 0x18 - -#define SMBDIRECT_FLAG_RESPONSE_REQUESTED 0x0001 - -/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */ -struct smbdirect_data_transfer { - __le16 credits_requested; - __le16 credits_granted; - __le16 flags; - __le16 reserved; - __le32 remaining_data_length; - __le32 data_offset; - __le32 data_length; - __le32 padding; - __u8 buffer[]; -} __packed; - -#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ */ diff --git a/fs/smb/common/smbdirect/smbdirect_public.h b/fs/smb/common/smbdirect/smbdirect_public.h deleted file mode 100644 index 50088155e7c3..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_public.h +++ /dev/null @@ -1,148 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2025, Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ -#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ - -struct smbdirect_buffer_descriptor_v1; -struct smbdirect_socket_parameters; - -struct smbdirect_socket; -struct smbdirect_send_batch; -struct smbdirect_mr_io; - -#define __SMBDIRECT_EXPORT_SYMBOL__(__sym) EXPORT_SYMBOL_FOR_MODULES(__sym, "cifs,ksmbd") - -#include - -u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev); - -bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs); - -int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc); - -int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc); - -int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, - const struct smbdirect_socket_parameters *sp); - -const struct smbdirect_socket_parameters * -smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc); - -int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, - enum ib_poll_context poll_ctx, - gfp_t gfp_mask); - -#define SMBDIRECT_LOG_ERR 0x0 -#define SMBDIRECT_LOG_INFO 0x1 - -#define SMBDIRECT_LOG_OUTGOING 0x1 -#define SMBDIRECT_LOG_INCOMING 0x2 -#define SMBDIRECT_LOG_READ 0x4 -#define SMBDIRECT_LOG_WRITE 0x8 -#define SMBDIRECT_LOG_RDMA_SEND 0x10 -#define SMBDIRECT_LOG_RDMA_RECV 0x20 -#define SMBDIRECT_LOG_KEEP_ALIVE 0x40 -#define SMBDIRECT_LOG_RDMA_EVENT 0x80 -#define SMBDIRECT_LOG_RDMA_MR 0x100 -#define SMBDIRECT_LOG_RDMA_RW 0x200 -#define SMBDIRECT_LOG_NEGOTIATE 0x400 -void smbdirect_socket_set_logging(struct smbdirect_socket *sc, - void *private_ptr, - bool (*needed)(struct smbdirect_socket *sc, - void *private_ptr, - unsigned int lvl, - unsigned int cls), - void (*vaprintf)(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - void *private_ptr, - unsigned int lvl, - unsigned int cls, - struct va_format *vaf)); - -bool smbdirect_connection_is_connected(struct smbdirect_socket *sc); - -int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc); - -int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr); - -void smbdirect_socket_shutdown(struct smbdirect_socket *sc); - -void smbdirect_socket_release(struct smbdirect_socket *sc); - -int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - bool is_last); - -/* - * This is only temporary and only needed - * as long as the client still requires - * to use smbdirect_connection_send_single_iter() - */ -struct smbdirect_send_batch_storage { - union { - struct list_head __msg_list; - __aligned_u64 __space[5]; - }; -}; - -struct smbdirect_send_batch * -smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, - bool need_invalidate_rkey, - unsigned int remote_key); - -int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, - struct smbdirect_send_batch *batch, - struct iov_iter *iter, - unsigned int flags, - u32 remaining_data_length); - -int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc); - -int smbdirect_connection_send_iter(struct smbdirect_socket *sc, - struct iov_iter *iter, - unsigned int flags, - bool need_invalidate, - unsigned int remote_key); - -int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, - struct msghdr *msg, - unsigned int flags); - -int smbdirect_connect(struct smbdirect_socket *sc, - const struct sockaddr *dst); - -int smbdirect_connect_sync(struct smbdirect_socket *sc, - const struct sockaddr *dst); - -int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog); - -struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, - long timeo, - struct proto_accept_arg *arg); - -int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, - void *buf, size_t buf_len, - struct smbdirect_buffer_descriptor_v1 *desc, - size_t desc_len, - bool is_read); - -struct smbdirect_mr_io * -smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, - struct iov_iter *iter, - bool writing, - bool need_invalidate); - -void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, - struct smbdirect_buffer_descriptor_v1 *v1); - -void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr); - -void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, - unsigned int rdma_readwrite_threshold, - struct seq_file *m); - -#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ */ diff --git a/fs/smb/common/smbdirect/smbdirect_rw.c b/fs/smb/common/smbdirect/smbdirect_rw.c deleted file mode 100644 index 3b2eb8c48efc..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_rw.c +++ /dev/null @@ -1,255 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (C) 2018, LG Electronics. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" - -static int smbdirect_connection_wait_for_rw_credits(struct smbdirect_socket *sc, - int credits) -{ - return smbdirect_socket_wait_for_credits(sc, - SMBDIRECT_SOCKET_CONNECTED, - -ENOTCONN, - &sc->rw_io.credits.wait_queue, - &sc->rw_io.credits.count, - credits); -} - -static int smbdirect_connection_calc_rw_credits(struct smbdirect_socket *sc, - const void *buf, - size_t len) -{ - return DIV_ROUND_UP(smbdirect_get_buf_page_count(buf, len), - sc->rw_io.credits.num_pages); -} - -static int smbdirect_connection_rdma_get_sg_list(void *buf, - size_t size, - struct scatterlist *sg_list, - size_t nentries) -{ - bool high = is_vmalloc_addr(buf); - struct page *page; - size_t offset, len; - int i = 0; - - if (size == 0 || nentries < smbdirect_get_buf_page_count(buf, size)) - return -EINVAL; - - offset = offset_in_page(buf); - buf -= offset; - while (size > 0) { - len = min_t(size_t, PAGE_SIZE - offset, size); - if (high) - page = vmalloc_to_page(buf); - else - page = kmap_to_page(buf); - - if (!sg_list) - return -EINVAL; - sg_set_page(sg_list, page, len, offset); - sg_list = sg_next(sg_list); - - buf += PAGE_SIZE; - size -= len; - offset = 0; - i++; - } - - return i; -} - -static void smbdirect_connection_rw_io_free(struct smbdirect_rw_io *msg, - enum dma_data_direction dir) -{ - struct smbdirect_socket *sc = msg->socket; - - rdma_rw_ctx_destroy(&msg->rdma_ctx, - sc->ib.qp, - sc->ib.qp->port, - msg->sgt.sgl, - msg->sgt.nents, - dir); - sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); - kfree(msg); -} - -static void smbdirect_connection_rdma_rw_done(struct ib_cq *cq, struct ib_wc *wc, - enum dma_data_direction dir) -{ - struct smbdirect_rw_io *msg = - container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe); - struct smbdirect_socket *sc = msg->socket; - - if (wc->status != IB_WC_SUCCESS) { - msg->error = -EIO; - pr_err("read/write error. opcode = %d, status = %s(%d)\n", - wc->opcode, ib_wc_status_msg(wc->status), wc->status); - if (wc->status != IB_WC_WR_FLUSH_ERR) - smbdirect_socket_schedule_cleanup(sc, msg->error); - } - - complete(msg->completion); -} - -static void smbdirect_connection_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) -{ - smbdirect_connection_rdma_rw_done(cq, wc, DMA_FROM_DEVICE); -} - -static void smbdirect_connection_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) -{ - smbdirect_connection_rdma_rw_done(cq, wc, DMA_TO_DEVICE); -} - -int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, - void *buf, size_t buf_len, - struct smbdirect_buffer_descriptor_v1 *desc, - size_t desc_len, - bool is_read) -{ - const struct smbdirect_socket_parameters *sp = &sc->parameters; - enum dma_data_direction direction = is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE; - struct smbdirect_rw_io *msg, *next_msg; - size_t i; - int ret; - DECLARE_COMPLETION_ONSTACK(completion); - struct ib_send_wr *first_wr; - LIST_HEAD(msg_list); - u8 *desc_buf; - int credits_needed; - size_t desc_buf_len, desc_num = 0; - - if (sc->status != SMBDIRECT_SOCKET_CONNECTED) - return -ENOTCONN; - - if (buf_len > sp->max_read_write_size) - return -EINVAL; - - /* calculate needed credits */ - credits_needed = 0; - desc_buf = buf; - for (i = 0; i < desc_len / sizeof(*desc); i++) { - if (!buf_len) - break; - - desc_buf_len = le32_to_cpu(desc[i].length); - if (!desc_buf_len) - return -EINVAL; - - if (desc_buf_len > buf_len) { - desc_buf_len = buf_len; - desc[i].length = cpu_to_le32(desc_buf_len); - buf_len = 0; - } - - credits_needed += smbdirect_connection_calc_rw_credits(sc, - desc_buf, - desc_buf_len); - desc_buf += desc_buf_len; - buf_len -= desc_buf_len; - desc_num++; - } - - smbdirect_log_rdma_rw(sc, SMBDIRECT_LOG_INFO, - "RDMA %s, len %zu, needed credits %d\n", - str_read_write(is_read), buf_len, credits_needed); - - ret = smbdirect_connection_wait_for_rw_credits(sc, credits_needed); - if (ret < 0) - return ret; - - /* build rdma_rw_ctx for each descriptor */ - desc_buf = buf; - for (i = 0; i < desc_num; i++) { - size_t page_count; - - msg = kzalloc_flex(*msg, sg_list, SG_CHUNK_SIZE, - sc->rw_io.mem.gfp_mask); - if (!msg) { - ret = -ENOMEM; - goto out; - } - - desc_buf_len = le32_to_cpu(desc[i].length); - page_count = smbdirect_get_buf_page_count(desc_buf, desc_buf_len); - - msg->socket = sc; - msg->cqe.done = is_read ? - smbdirect_connection_rdma_read_done : - smbdirect_connection_rdma_write_done; - msg->completion = &completion; - - msg->sgt.sgl = &msg->sg_list[0]; - ret = sg_alloc_table_chained(&msg->sgt, - page_count, - msg->sg_list, - SG_CHUNK_SIZE); - if (ret) { - ret = -ENOMEM; - goto free_msg; - } - - ret = smbdirect_connection_rdma_get_sg_list(desc_buf, - desc_buf_len, - msg->sgt.sgl, - msg->sgt.orig_nents); - if (ret < 0) - goto free_table; - - ret = rdma_rw_ctx_init(&msg->rdma_ctx, - sc->ib.qp, - sc->ib.qp->port, - msg->sgt.sgl, - page_count, - 0, - le64_to_cpu(desc[i].offset), - le32_to_cpu(desc[i].token), - direction); - if (ret < 0) { - pr_err("failed to init rdma_rw_ctx: %d\n", ret); - goto free_table; - } - - list_add_tail(&msg->list, &msg_list); - desc_buf += desc_buf_len; - } - - /* concatenate work requests of rdma_rw_ctxs */ - first_wr = NULL; - list_for_each_entry_reverse(msg, &msg_list, list) { - first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, - sc->ib.qp, - sc->ib.qp->port, - &msg->cqe, - first_wr); - } - - ret = ib_post_send(sc->ib.qp, first_wr, NULL); - if (ret) { - pr_err("failed to post send wr for RDMA R/W: %d\n", ret); - goto out; - } - - msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list); - wait_for_completion(&completion); - ret = msg->error; -out: - list_for_each_entry_safe(msg, next_msg, &msg_list, list) { - list_del(&msg->list); - smbdirect_connection_rw_io_free(msg, direction); - } - atomic_add(credits_needed, &sc->rw_io.credits.count); - wake_up(&sc->rw_io.credits.wait_queue); - return ret; - -free_table: - sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); -free_msg: - kfree(msg); - goto out; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_rdma_xmit); diff --git a/fs/smb/common/smbdirect/smbdirect_socket.c b/fs/smb/common/smbdirect/smbdirect_socket.c deleted file mode 100644 index 9153e1dbf53d..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_socket.c +++ /dev/null @@ -1,743 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2017, Microsoft Corporation. - * Copyright (c) 2025, Stefan Metzmacher - */ - -#include "smbdirect_internal.h" - -bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs) -{ - /* - * Test if FRWR (Fast Registration Work Requests) is supported on the - * device This implementation requires FRWR on RDMA read/write return - * value: true if it is supported - */ - - if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) - return false; - if (attrs->max_fast_reg_page_list_len == 0) - return false; - return true; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_frwr_is_supported); - -static void smbdirect_socket_cleanup_work(struct work_struct *work); - -static int smbdirect_socket_rdma_event_handler(struct rdma_cm_id *id, - struct rdma_cm_event *event) -{ - struct smbdirect_socket *sc = id->context; - int ret = -ESTALE; - - /* - * This should be replaced before any real work - * starts! So it should never be called! - */ - - if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) - ret = -ENETDOWN; - if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) - ret = event->status; - pr_err("%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), - rdma_event_msg(sc->rdma.expected_event), - rdma_event_msg(event->event), - event->status, - SMBDIRECT_DEBUG_ERR_PTR(ret)); - WARN_ONCE(1, "%s should not be called!\n", __func__); - sc->rdma.cm_id = NULL; - return -ESTALE; -} - -int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc) -{ - struct rdma_cm_id *id; - int ret; - - smbdirect_socket_init(sc); - - id = rdma_create_id(net, - smbdirect_socket_rdma_event_handler, - sc, - RDMA_PS_TCP, - IB_QPT_RC); - if (IS_ERR(id)) { - pr_err("%s: rdma_create_id() failed %1pe\n", __func__, id); - return PTR_ERR(id); - } - - ret = rdma_set_afonly(id, 1); - if (ret) { - rdma_destroy_id(id); - pr_err("%s: rdma_set_afonly() failed %1pe\n", - __func__, SMBDIRECT_DEBUG_ERR_PTR(ret)); - return ret; - } - - sc->rdma.cm_id = id; - - INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work); - - return 0; -} - -int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc) -{ - struct smbdirect_socket *sc; - int ret; - - ret = -ENOMEM; - sc = kzalloc_obj(*sc); - if (!sc) - goto alloc_failed; - - ret = smbdirect_socket_init_new(net, sc); - if (ret) - goto init_failed; - - kref_init(&sc->refs.destroy); - - *_sc = sc; - return 0; - -init_failed: - kfree(sc); -alloc_failed: - return ret; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_kern); - -int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc) -{ - smbdirect_socket_init(sc); - - sc->rdma.cm_id = id; - sc->rdma.cm_id->context = sc; - sc->rdma.cm_id->event_handler = smbdirect_socket_rdma_event_handler; - - sc->ib.dev = sc->rdma.cm_id->device; - - INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work); - - return 0; -} - -int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc) -{ - struct smbdirect_socket *sc; - int ret; - - ret = -ENOMEM; - sc = kzalloc_obj(*sc); - if (!sc) - goto alloc_failed; - - ret = smbdirect_socket_init_accepting(id, sc); - if (ret) - goto init_failed; - - kref_init(&sc->refs.destroy); - - *_sc = sc; - return 0; - -init_failed: - kfree(sc); -alloc_failed: - return ret; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_accepting); - -int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, - const struct smbdirect_socket_parameters *sp) -{ - /* - * This is only allowed before connect or accept - */ - WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - if (sc->status != SMBDIRECT_SOCKET_CREATED) - return -EINVAL; - - if (sp->flags & ~SMBDIRECT_FLAG_PORT_RANGE_MASK) - return -EINVAL; - - if (sp->initiator_depth > U8_MAX) - return -EINVAL; - if (sp->responder_resources > U8_MAX) - return -EINVAL; - - if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && - sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) - return -EINVAL; - else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB) - rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_IB_CA); - else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) - rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_RNIC); - - /* - * Make a copy of the callers parameters - * from here we only work on the copy - * - * TODO: do we want consistency checking? - */ - sc->parameters = *sp; - - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_initial_parameters); - -const struct smbdirect_socket_parameters * -smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc) -{ - return &sc->parameters; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_get_current_parameters); - -int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, - enum ib_poll_context poll_ctx, - gfp_t gfp_mask) -{ - /* - * This is only allowed before connect or accept - */ - WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - if (sc->status != SMBDIRECT_SOCKET_CREATED) - return -EINVAL; - - sc->ib.poll_ctx = poll_ctx; - - sc->send_io.mem.gfp_mask = gfp_mask; - sc->recv_io.mem.gfp_mask = gfp_mask; - sc->rw_io.mem.gfp_mask = gfp_mask; - - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_kernel_settings); - -void smbdirect_socket_set_logging(struct smbdirect_socket *sc, - void *private_ptr, - bool (*needed)(struct smbdirect_socket *sc, - void *private_ptr, - unsigned int lvl, - unsigned int cls), - void (*vaprintf)(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - void *private_ptr, - unsigned int lvl, - unsigned int cls, - struct va_format *vaf)) -{ - sc->logging.private_ptr = private_ptr; - sc->logging.needed = needed; - sc->logging.vaprintf = vaprintf; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_logging); - -static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc) -{ - /* - * Wake up all waiters in all wait queues - * in order to notice the broken connection. - */ - wake_up_all(&sc->status_wait); - wake_up_all(&sc->listen.wait_queue); - wake_up_all(&sc->send_io.bcredits.wait_queue); - wake_up_all(&sc->send_io.lcredits.wait_queue); - wake_up_all(&sc->send_io.credits.wait_queue); - wake_up_all(&sc->send_io.pending.zero_wait_queue); - wake_up_all(&sc->recv_io.reassembly.wait_queue); - wake_up_all(&sc->rw_io.credits.wait_queue); - wake_up_all(&sc->mr_io.ready.wait_queue); -} - -void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, - const char *macro_name, - unsigned int lvl, - const char *func, - unsigned int line, - int error, - enum smbdirect_socket_status *force_status) -{ - struct smbdirect_socket *psc, *tsc; - unsigned long flags; - bool was_first = false; - - if (!sc->first_error) { - ___smbdirect_log_generic(sc, func, line, - lvl, - SMBDIRECT_LOG_RDMA_EVENT, - "%s(%1pe%s%s) called from %s in line=%u status=%s\n", - macro_name, - SMBDIRECT_DEBUG_ERR_PTR(error), - force_status ? ", " : "", - force_status ? smbdirect_socket_status_string(*force_status) : "", - func, line, - smbdirect_socket_status_string(sc->status)); - if (error) - sc->first_error = error; - else - sc->first_error = -ECONNABORTED; - was_first = true; - } - - /* - * make sure other work (than disconnect_work) - * is not queued again but here we don't block and avoid - * disable[_delayed]_work_sync() - */ - disable_work(&sc->connect.work); - disable_work(&sc->recv_io.posted.refill_work); - disable_work(&sc->idle.immediate_work); - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; - disable_delayed_work(&sc->idle.timer_work); - - /* - * In case we were a listener we need to - * disconnect all pending and ready sockets - * - * First we move ready sockets to pending again. - */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_init(&sc->listen.ready, &sc->listen.pending); - list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) - smbdirect_socket_schedule_cleanup(psc, sc->first_error); - spin_unlock_irqrestore(&sc->listen.lock, flags); - - switch (sc->status) { - case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: - case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: - case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: - case SMBDIRECT_SOCKET_ERROR: - case SMBDIRECT_SOCKET_DISCONNECTING: - case SMBDIRECT_SOCKET_DISCONNECTED: - case SMBDIRECT_SOCKET_DESTROYED: - /* - * Keep the current error status - */ - break; - - case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: - case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: - sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; - break; - - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: - sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; - break; - - case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: - case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: - sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; - break; - - case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: - case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: - sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; - break; - - case SMBDIRECT_SOCKET_CREATED: - case SMBDIRECT_SOCKET_LISTENING: - sc->status = SMBDIRECT_SOCKET_DISCONNECTED; - break; - - case SMBDIRECT_SOCKET_CONNECTED: - sc->status = SMBDIRECT_SOCKET_ERROR; - break; - } - - if (force_status && (was_first || *force_status > sc->status)) - sc->status = *force_status; - - /* - * Wake up all waiters in all wait queues - * in order to notice the broken connection. - */ - smbdirect_socket_wake_up_all(sc); - - queue_work(sc->workqueues.cleanup, &sc->disconnect_work); -} - -static void smbdirect_socket_cleanup_work(struct work_struct *work) -{ - struct smbdirect_socket *sc = - container_of(work, struct smbdirect_socket, disconnect_work); - struct smbdirect_socket *psc, *tsc; - unsigned long flags; - - /* - * This should not never be called in an interrupt! - */ - WARN_ON_ONCE(in_interrupt()); - - if (!sc->first_error) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, - "%s called with first_error==0\n", - smbdirect_socket_status_string(sc->status)); - - sc->first_error = -ECONNABORTED; - } - - /* - * make sure this and other work is not queued again - * but here we don't block and avoid - * disable[_delayed]_work_sync() - */ - disable_work(&sc->disconnect_work); - disable_work(&sc->connect.work); - disable_work(&sc->recv_io.posted.refill_work); - disable_work(&sc->idle.immediate_work); - sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; - disable_delayed_work(&sc->idle.timer_work); - - /* - * In case we were a listener we need to - * disconnect all pending and ready sockets - * - * First we move ready sockets to pending again. - */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_init(&sc->listen.ready, &sc->listen.pending); - list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) - smbdirect_socket_schedule_cleanup(psc, sc->first_error); - spin_unlock_irqrestore(&sc->listen.lock, flags); - - switch (sc->status) { - case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: - case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: - case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: - case SMBDIRECT_SOCKET_CONNECTED: - case SMBDIRECT_SOCKET_ERROR: - sc->status = SMBDIRECT_SOCKET_DISCONNECTING; - /* - * Make sure we hold the callback lock - * im order to coordinate with the - * rdma_event handlers, typically - * smbdirect_connection_rdma_event_handler(), - * and smbdirect_socket_destroy(). - * - * So that the order of ib_drain_qp() - * and rdma_disconnect() is controlled - * by the mutex. - */ - rdma_lock_handler(sc->rdma.cm_id); - rdma_disconnect(sc->rdma.cm_id); - rdma_unlock_handler(sc->rdma.cm_id); - break; - - case SMBDIRECT_SOCKET_CREATED: - case SMBDIRECT_SOCKET_LISTENING: - case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: - case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: - case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: - case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: - case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: - case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: - /* - * rdma_{accept,connect}() never reached - * RDMA_CM_EVENT_ESTABLISHED - */ - sc->status = SMBDIRECT_SOCKET_DISCONNECTED; - break; - - case SMBDIRECT_SOCKET_DISCONNECTING: - case SMBDIRECT_SOCKET_DISCONNECTED: - case SMBDIRECT_SOCKET_DESTROYED: - break; - } - - /* - * Wake up all waiters in all wait queues - * in order to notice the broken connection. - */ - smbdirect_socket_wake_up_all(sc); -} - -static void smbdirect_socket_destroy(struct smbdirect_socket *sc) -{ - struct smbdirect_socket *psc, *tsc; - size_t psockets; - struct smbdirect_recv_io *recv_io; - struct smbdirect_recv_io *recv_tmp; - LIST_HEAD(all_list); - unsigned long flags; - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - - /* - * This should not never be called in an interrupt! - */ - WARN_ON_ONCE(in_interrupt()); - - if (sc->status == SMBDIRECT_SOCKET_DESTROYED) - return; - - WARN_ONCE(sc->status != SMBDIRECT_SOCKET_DISCONNECTED, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - - /* - * The listener should clear this before we reach this - */ - WARN_ONCE(sc->accept.listener, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - - /* - * Wake up all waiters in all wait queues - * in order to notice the broken connection. - * - * Most likely this was already called via - * smbdirect_socket_cleanup_work(), but call it again... - */ - smbdirect_socket_wake_up_all(sc); - - disable_work_sync(&sc->disconnect_work); - disable_work_sync(&sc->connect.work); - disable_work_sync(&sc->recv_io.posted.refill_work); - disable_work_sync(&sc->idle.immediate_work); - disable_delayed_work_sync(&sc->idle.timer_work); - - if (sc->rdma.cm_id) - rdma_lock_handler(sc->rdma.cm_id); - - if (sc->ib.qp) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "drain qp\n"); - ib_drain_qp(sc->ib.qp); - } - - /* - * In case we were a listener we need to - * disconnect all pending and ready sockets - * - * We move ready sockets to pending again. - */ - spin_lock_irqsave(&sc->listen.lock, flags); - list_splice_tail_init(&sc->listen.ready, &all_list); - list_splice_tail_init(&sc->listen.pending, &all_list); - spin_unlock_irqrestore(&sc->listen.lock, flags); - psockets = list_count_nodes(&all_list); - if (sc->listen.backlog != -1) /* was a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "release %zu pending sockets\n", psockets); - list_for_each_entry_safe(psc, tsc, &all_list, accept.list) { - list_del_init(&psc->accept.list); - psc->accept.listener = NULL; - smbdirect_socket_release(psc); - } - if (sc->listen.backlog != -1) /* was a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "released %zu pending sockets\n", psockets); - INIT_LIST_HEAD(&all_list); - - /* It's not possible for upper layer to get to reassembly */ - if (sc->listen.backlog == -1) /* was not a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "drain the reassembly queue\n"); - spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); - list_splice_tail_init(&sc->recv_io.reassembly.list, &all_list); - spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); - list_for_each_entry_safe(recv_io, recv_tmp, &all_list, list) - smbdirect_connection_put_recv_io(recv_io); - sc->recv_io.reassembly.data_length = 0; - - if (sc->listen.backlog == -1) /* was not a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "freeing mr list\n"); - smbdirect_connection_destroy_mr_list(sc); - - if (sc->listen.backlog == -1) /* was not a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "destroying qp\n"); - smbdirect_connection_destroy_qp(sc); - if (sc->rdma.cm_id) { - rdma_unlock_handler(sc->rdma.cm_id); - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "destroying cm_id\n"); - rdma_destroy_id(sc->rdma.cm_id); - sc->rdma.cm_id = NULL; - } - - if (sc->listen.backlog == -1) /* was not a listener */ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "destroying mem pools\n"); - smbdirect_connection_destroy_mem_pools(sc); - - sc->status = SMBDIRECT_SOCKET_DESTROYED; - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "rdma session destroyed\n"); -} - -void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc) -{ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); - - /* - * This should not never be called in an interrupt! - */ - WARN_ON_ONCE(in_interrupt()); - - /* - * First we try to disable the work - * without disable_work_sync() in a - * non blocking way, if it's already - * running it will be handles by - * disable_work_sync() below. - * - * Here we just want to make sure queue_work() in - * smbdirect_socket_schedule_cleanup_lvl() - * is a no-op. - */ - disable_work(&sc->disconnect_work); - - if (!sc->first_error) - /* - * SMBDIRECT_LOG_INFO is enough here - * as this is the typical case where - * we terminate the connection ourself. - */ - smbdirect_socket_schedule_cleanup_lvl(sc, - SMBDIRECT_LOG_INFO, - -ESHUTDOWN); - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "cancelling and disable disconnect_work\n"); - disable_work_sync(&sc->disconnect_work); - - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "destroying rdma session\n"); - if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) - smbdirect_socket_cleanup_work(&sc->disconnect_work); - if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) { - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "wait for transport being disconnected\n"); - wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "waited for transport being disconnected\n"); - } - - /* - * Once we reached SMBDIRECT_SOCKET_DISCONNECTED, - * we should call smbdirect_socket_destroy() - */ - smbdirect_socket_destroy(sc); - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, - "status=%s first_error=%1pe", - smbdirect_socket_status_string(sc->status), - SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); -} - -int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr) -{ - int ret; - - if (sc->status != SMBDIRECT_SOCKET_CREATED) - return -EINVAL; - - ret = rdma_bind_addr(sc->rdma.cm_id, addr); - if (ret) - return ret; - - return 0; -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_bind); - -void smbdirect_socket_shutdown(struct smbdirect_socket *sc) -{ - smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN); -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_shutdown); - -static void smbdirect_socket_release_disconnect(struct kref *kref) -{ - struct smbdirect_socket *sc = - container_of(kref, struct smbdirect_socket, refs.disconnect); - - /* - * For now do a sync disconnect/destroy - */ - smbdirect_socket_destroy_sync(sc); -} - -static void smbdirect_socket_release_destroy(struct kref *kref) -{ - struct smbdirect_socket *sc = - container_of(kref, struct smbdirect_socket, refs.destroy); - - /* - * Do a sync disconnect/destroy... - * hopefully a no-op, as it should be already - * in DESTROYED state, before we free the memory. - */ - smbdirect_socket_destroy_sync(sc); - kfree(sc); -} - -void smbdirect_socket_release(struct smbdirect_socket *sc) -{ - /* - * We expect only 1 disconnect reference - * and if it is already 0, it's a use after free! - */ - WARN_ON_ONCE(kref_read(&sc->refs.disconnect) != 1); - WARN_ON(!kref_put(&sc->refs.disconnect, smbdirect_socket_release_disconnect)); - - /* - * This may not trigger smbdirect_socket_release_destroy(), - * if struct smbdirect_socket is embedded in another structure - * indicated by REFCOUNT_MAX. - */ - kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy); -} -__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_release); - -int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, - enum smbdirect_socket_status expected_status, - int unexpected_errno, - wait_queue_head_t *waitq, - atomic_t *total_credits, - int needed) -{ - int ret; - - if (WARN_ON_ONCE(needed < 0)) - return -EINVAL; - - do { - if (atomic_sub_return(needed, total_credits) >= 0) - return 0; - - atomic_add(needed, total_credits); - ret = wait_event_interruptible(*waitq, - atomic_read(total_credits) >= needed || - sc->status != expected_status); - - if (sc->status != expected_status) - return unexpected_errno; - else if (ret < 0) - return ret; - } while (true); -} diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h deleted file mode 100644 index c09eddd8ad16..000000000000 --- a/fs/smb/common/smbdirect/smbdirect_socket.h +++ /dev/null @@ -1,832 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2025 Stefan Metzmacher - */ - -#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ -#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ - -#include -#include -#include -#include -#include -#include -#include -#include - -enum smbdirect_socket_status { - SMBDIRECT_SOCKET_CREATED, - SMBDIRECT_SOCKET_LISTENING, - SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED, - SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, - SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED, - SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED, - SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, - SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED, - SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED, - SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, - SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED, - SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, - SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, - SMBDIRECT_SOCKET_NEGOTIATE_FAILED, - SMBDIRECT_SOCKET_CONNECTED, - SMBDIRECT_SOCKET_ERROR, - SMBDIRECT_SOCKET_DISCONNECTING, - SMBDIRECT_SOCKET_DISCONNECTED, - SMBDIRECT_SOCKET_DESTROYED -}; - -static __always_inline -const char *smbdirect_socket_status_string(enum smbdirect_socket_status status) -{ - switch (status) { - case SMBDIRECT_SOCKET_CREATED: - return "CREATED"; - case SMBDIRECT_SOCKET_LISTENING: - return "LISTENING"; - case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: - return "RESOLVE_ADDR_NEEDED"; - case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: - return "RESOLVE_ADDR_RUNNING"; - case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: - return "RESOLVE_ADDR_FAILED"; - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: - return "RESOLVE_ROUTE_NEEDED"; - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: - return "RESOLVE_ROUTE_RUNNING"; - case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: - return "RESOLVE_ROUTE_FAILED"; - case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: - return "RDMA_CONNECT_NEEDED"; - case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: - return "RDMA_CONNECT_RUNNING"; - case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: - return "RDMA_CONNECT_FAILED"; - case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: - return "NEGOTIATE_NEEDED"; - case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: - return "NEGOTIATE_RUNNING"; - case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: - return "NEGOTIATE_FAILED"; - case SMBDIRECT_SOCKET_CONNECTED: - return "CONNECTED"; - case SMBDIRECT_SOCKET_ERROR: - return "ERROR"; - case SMBDIRECT_SOCKET_DISCONNECTING: - return "DISCONNECTING"; - case SMBDIRECT_SOCKET_DISCONNECTED: - return "DISCONNECTED"; - case SMBDIRECT_SOCKET_DESTROYED: - return "DESTROYED"; - } - - return ""; -} - -/* - * This can be used with %1pe to print errors as strings or '0' - * And it avoids warnings like: warn: passing zero to 'ERR_PTR' - * from smatch -p=kernel --pedantic - */ -static __always_inline -const void * __must_check SMBDIRECT_DEBUG_ERR_PTR(long error) -{ - if (error == 0) - return NULL; - return ERR_PTR(error); -} - -enum smbdirect_keepalive_status { - SMBDIRECT_KEEPALIVE_NONE, - SMBDIRECT_KEEPALIVE_PENDING, - SMBDIRECT_KEEPALIVE_SENT -}; - -struct smbdirect_socket { - enum smbdirect_socket_status status; - wait_queue_head_t status_wait; - int first_error; - - /* - * This points to the workqueues to - * be used for this socket. - */ - struct { - struct workqueue_struct *accept; - struct workqueue_struct *connect; - struct workqueue_struct *idle; - struct workqueue_struct *refill; - struct workqueue_struct *immediate; - struct workqueue_struct *cleanup; - } workqueues; - - struct work_struct disconnect_work; - - /* - * The reference counts. - */ - struct { - /* - * This holds the references by the - * frontend, typically the smb layer. - * - * It is typically 1 and a disconnect - * will happen if it reaches 0. - */ - struct kref disconnect; - - /* - * This holds the reference by the - * backend, the code that manages - * the lifetime of the whole - * struct smbdirect_socket, - * if this reaches 0 it can will - * be freed. - * - * Can be REFCOUNT_MAX is part - * of another structure. - * - * This is equal or higher than - * the disconnect refcount. - */ - struct kref destroy; - } refs; - - /* RDMA related */ - struct { - struct rdma_cm_id *cm_id; - /* - * The expected event in our current - * cm_id->event_handler, all other events - * are treated as an error. - */ - enum rdma_cm_event_type expected_event; - /* - * This is for iWarp MPA v1 - */ - bool legacy_iwarp; - } rdma; - - /* IB verbs related */ - struct { - struct ib_pd *pd; - enum ib_poll_context poll_ctx; - struct ib_cq *send_cq; - struct ib_cq *recv_cq; - - /* - * shortcuts for rdma.cm_id->{qp,device}; - */ - struct ib_qp *qp; - struct ib_device *dev; - } ib; - - struct smbdirect_socket_parameters parameters; - - /* - * The state for connect/negotiation - */ - struct { - spinlock_t lock; - struct work_struct work; - } connect; - - /* - * The state for keepalive and timeout handling - */ - struct { - enum smbdirect_keepalive_status keepalive; - struct work_struct immediate_work; - struct delayed_work timer_work; - } idle; - - /* - * The state for listen sockets - */ - struct { - spinlock_t lock; - struct list_head pending; - struct list_head ready; - wait_queue_head_t wait_queue; - /* - * This starts as -1 and a value != -1 - * means this socket was in LISTENING state - * before. Note the valid backlog can - * only be > 0. - */ - int backlog; - } listen; - - /* - * The state for sockets waiting - * for accept, either still waiting - * for the negotiation to finish - * or already ready with a usable - * connection. - */ - struct { - struct smbdirect_socket *listener; - struct list_head list; - } accept; - - /* - * The state for posted send buffers - */ - struct { - /* - * Memory pools for preallocating - * smbdirect_send_io buffers - */ - struct { - struct kmem_cache *cache; - mempool_t *pool; - gfp_t gfp_mask; - } mem; - - /* - * This is a coordination for smbdirect_send_batch. - * - * There's only one possible credit, which means - * only one instance is running at a time. - */ - struct { - atomic_t count; - wait_queue_head_t wait_queue; - } bcredits; - - /* - * The local credit state for ib_post_send() - */ - struct { - atomic_t count; - wait_queue_head_t wait_queue; - } lcredits; - - /* - * The remote credit state for the send side - */ - struct { - atomic_t count; - wait_queue_head_t wait_queue; - } credits; - - /* - * The state about posted/pending sends - */ - struct { - atomic_t count; - /* - * woken when count reached zero - */ - wait_queue_head_t zero_wait_queue; - } pending; - } send_io; - - /* - * The state for posted receive buffers - */ - struct { - /* - * The type of PDU we are expecting - */ - enum { - SMBDIRECT_EXPECT_NEGOTIATE_REQ = 1, - SMBDIRECT_EXPECT_NEGOTIATE_REP = 2, - SMBDIRECT_EXPECT_DATA_TRANSFER = 3, - } expected; - - /* - * Memory pools for preallocating - * smbdirect_recv_io buffers - */ - struct { - struct kmem_cache *cache; - mempool_t *pool; - gfp_t gfp_mask; - } mem; - - /* - * The list of free smbdirect_recv_io - * structures - */ - struct { - struct list_head list; - spinlock_t lock; - } free; - - /* - * The state for posted recv_io messages - * and the refill work struct. - */ - struct { - atomic_t count; - struct work_struct refill_work; - } posted; - - /* - * The credit state for the recv side - */ - struct { - u16 target; - atomic_t available; - atomic_t count; - } credits; - - /* - * The list of arrived non-empty smbdirect_recv_io - * structures - * - * This represents the reassembly queue. - */ - struct { - struct list_head list; - spinlock_t lock; - wait_queue_head_t wait_queue; - /* total data length of reassembly queue */ - int data_length; - int queue_length; - /* the offset to first buffer in reassembly queue */ - int first_entry_offset; - /* - * Indicate if we have received a full packet on the - * connection This is used to identify the first SMBD - * packet of a assembled payload (SMB packet) in - * reassembly queue so we can return a RFC1002 length to - * upper layer to indicate the length of the SMB packet - * received - */ - bool full_packet_received; - } reassembly; - } recv_io; - - /* - * The state for Memory registrations on the client - */ - struct { - enum ib_mr_type type; - - /* - * The list of free smbdirect_mr_io - * structures - */ - struct { - struct list_head list; - spinlock_t lock; - } all; - - /* - * The number of available MRs ready for memory registration - */ - struct { - atomic_t count; - wait_queue_head_t wait_queue; - } ready; - - /* - * The number of used MRs - */ - struct { - atomic_t count; - } used; - } mr_io; - - /* - * The state for RDMA read/write requests on the server - */ - struct { - /* - * Memory hints for - * smbdirect_rw_io structs - */ - struct { - gfp_t gfp_mask; - } mem; - - /* - * The credit state for the send side - */ - struct { - /* - * The maximum number of rw credits - */ - size_t max; - /* - * The number of pages per credit - */ - size_t num_pages; - atomic_t count; - wait_queue_head_t wait_queue; - } credits; - } rw_io; - - /* - * For debug purposes - */ - struct { - u64 get_receive_buffer; - u64 put_receive_buffer; - u64 enqueue_reassembly_queue; - u64 dequeue_reassembly_queue; - u64 send_empty; - } statistics; - - struct { - void *private_ptr; - bool (*needed)(struct smbdirect_socket *sc, - void *private_ptr, - unsigned int lvl, - unsigned int cls); - void (*vaprintf)(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - void *private_ptr, - unsigned int lvl, - unsigned int cls, - struct va_format *vaf); - } logging; -}; - -static void __smbdirect_socket_disabled_work(struct work_struct *work) -{ - /* - * Should never be called as disable_[delayed_]work_sync() was used. - */ - WARN_ON_ONCE(1); -} - -static bool __smbdirect_log_needed(struct smbdirect_socket *sc, - void *private_ptr, - unsigned int lvl, - unsigned int cls) -{ - /* - * Should never be called, the caller should - * set it's own functions. - */ - WARN_ON_ONCE(1); - return false; -} - -static void __smbdirect_log_vaprintf(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - void *private_ptr, - unsigned int lvl, - unsigned int cls, - struct va_format *vaf) -{ - /* - * Should never be called, the caller should - * set it's own functions. - */ - WARN_ON_ONCE(1); -} - -__printf(6, 7) -static void __smbdirect_log_printf(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - unsigned int lvl, - unsigned int cls, - const char *fmt, - ...); -__maybe_unused -static void __smbdirect_log_printf(struct smbdirect_socket *sc, - const char *func, - unsigned int line, - unsigned int lvl, - unsigned int cls, - const char *fmt, - ...) -{ - struct va_format vaf; - va_list args; - - va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - sc->logging.vaprintf(sc, - func, - line, - sc->logging.private_ptr, - lvl, - cls, - &vaf); - va_end(args); -} - -#define ___smbdirect_log_generic(sc, func, line, lvl, cls, fmt, args...) do { \ - if (sc->logging.needed(sc, sc->logging.private_ptr, lvl, cls)) { \ - __smbdirect_log_printf(sc, func, line, lvl, cls, fmt, ##args); \ - } \ -} while (0) -#define __smbdirect_log_generic(sc, lvl, cls, fmt, args...) \ - ___smbdirect_log_generic(sc, __func__, __LINE__, lvl, cls, fmt, ##args) - -#define smbdirect_log_outgoing(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_OUTGOING, fmt, ##args) -#define smbdirect_log_incoming(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_INCOMING, fmt, ##args) -#define smbdirect_log_read(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_READ, fmt, ##args) -#define smbdirect_log_write(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_WRITE, fmt, ##args) -#define smbdirect_log_rdma_send(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_SEND, fmt, ##args) -#define smbdirect_log_rdma_recv(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RECV, fmt, ##args) -#define smbdirect_log_keep_alive(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_KEEP_ALIVE, fmt, ##args) -#define smbdirect_log_rdma_event(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_EVENT, fmt, ##args) -#define smbdirect_log_rdma_mr(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_MR, fmt, ##args) -#define smbdirect_log_rdma_rw(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RW, fmt, ##args) -#define smbdirect_log_negotiate(sc, lvl, fmt, args...) \ - __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_NEGOTIATE, fmt, ##args) - -static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) -{ - /* - * This also sets status = SMBDIRECT_SOCKET_CREATED - */ - BUILD_BUG_ON(SMBDIRECT_SOCKET_CREATED != 0); - memset(sc, 0, sizeof(*sc)); - - init_waitqueue_head(&sc->status_wait); - - sc->workqueues.accept = smbdirect_globals.workqueues.accept; - sc->workqueues.connect = smbdirect_globals.workqueues.connect; - sc->workqueues.idle = smbdirect_globals.workqueues.idle; - sc->workqueues.refill = smbdirect_globals.workqueues.refill; - sc->workqueues.immediate = smbdirect_globals.workqueues.immediate; - sc->workqueues.cleanup = smbdirect_globals.workqueues.cleanup; - - INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work); - disable_work_sync(&sc->disconnect_work); - - kref_init(&sc->refs.disconnect); - sc->refs.destroy = (struct kref) KREF_INIT(REFCOUNT_MAX); - - sc->rdma.expected_event = RDMA_CM_EVENT_INTERNAL; - - sc->ib.poll_ctx = IB_POLL_UNBOUND_WORKQUEUE; - - spin_lock_init(&sc->connect.lock); - INIT_WORK(&sc->connect.work, __smbdirect_socket_disabled_work); - disable_work_sync(&sc->connect.work); - - INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work); - disable_work_sync(&sc->idle.immediate_work); - INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); - disable_delayed_work_sync(&sc->idle.timer_work); - - spin_lock_init(&sc->listen.lock); - INIT_LIST_HEAD(&sc->listen.pending); - INIT_LIST_HEAD(&sc->listen.ready); - sc->listen.backlog = -1; /* not a listener */ - init_waitqueue_head(&sc->listen.wait_queue); - - INIT_LIST_HEAD(&sc->accept.list); - - sc->send_io.mem.gfp_mask = GFP_KERNEL; - - atomic_set(&sc->send_io.bcredits.count, 0); - init_waitqueue_head(&sc->send_io.bcredits.wait_queue); - - atomic_set(&sc->send_io.lcredits.count, 0); - init_waitqueue_head(&sc->send_io.lcredits.wait_queue); - - atomic_set(&sc->send_io.credits.count, 0); - init_waitqueue_head(&sc->send_io.credits.wait_queue); - - atomic_set(&sc->send_io.pending.count, 0); - init_waitqueue_head(&sc->send_io.pending.zero_wait_queue); - - sc->recv_io.mem.gfp_mask = GFP_KERNEL; - - INIT_LIST_HEAD(&sc->recv_io.free.list); - spin_lock_init(&sc->recv_io.free.lock); - - atomic_set(&sc->recv_io.posted.count, 0); - INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); - disable_work_sync(&sc->recv_io.posted.refill_work); - - atomic_set(&sc->recv_io.credits.available, 0); - atomic_set(&sc->recv_io.credits.count, 0); - - INIT_LIST_HEAD(&sc->recv_io.reassembly.list); - spin_lock_init(&sc->recv_io.reassembly.lock); - init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); - - sc->rw_io.mem.gfp_mask = GFP_KERNEL; - atomic_set(&sc->rw_io.credits.count, 0); - init_waitqueue_head(&sc->rw_io.credits.wait_queue); - - spin_lock_init(&sc->mr_io.all.lock); - INIT_LIST_HEAD(&sc->mr_io.all.list); - atomic_set(&sc->mr_io.ready.count, 0); - init_waitqueue_head(&sc->mr_io.ready.wait_queue); - atomic_set(&sc->mr_io.used.count, 0); - - sc->logging.private_ptr = NULL; - sc->logging.needed = __smbdirect_log_needed; - sc->logging.vaprintf = __smbdirect_log_vaprintf; -} - -#define __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, __error_cmd, __unexpected_cmd) ({ \ - bool __failed = false; \ - if (unlikely((__sc)->first_error)) { \ - __failed = true; \ - __error_cmd \ - } else if (unlikely((__sc)->status != (__expected_status))) { \ - __failed = true; \ - __unexpected_cmd \ - } \ - __failed; \ -}) - -#define __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, __unexpected_cmd) \ - __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, \ - { \ - const struct sockaddr_storage *__src = NULL; \ - const struct sockaddr_storage *__dst = NULL; \ - if ((__sc)->rdma.cm_id) { \ - __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ - __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ - } \ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, \ - "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ - smbdirect_socket_status_string(__expected_status), \ - smbdirect_socket_status_string((__sc)->status), \ - SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ - __src, __dst); \ - }, \ - { \ - const struct sockaddr_storage *__src = NULL; \ - const struct sockaddr_storage *__dst = NULL; \ - if ((__sc)->rdma.cm_id) { \ - __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ - __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ - } \ - smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, \ - "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ - smbdirect_socket_status_string(__expected_status), \ - smbdirect_socket_status_string((__sc)->status), \ - SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ - __src, __dst); \ - WARN_ONCE(1, \ - "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ - smbdirect_socket_status_string(__expected_status), \ - smbdirect_socket_status_string((__sc)->status), \ - SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ - __src, __dst); \ - __unexpected_cmd \ - }) - -#define SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status) \ - __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, /* nothing */) - -#ifndef __SMBDIRECT_SOCKET_DISCONNECT -#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) \ - smbdirect_socket_schedule_cleanup(__sc, -ECONNABORTED) -#endif /* ! __SMBDIRECT_SOCKET_DISCONNECT */ - -#define SMBDIRECT_CHECK_STATUS_DISCONNECT(__sc, __expected_status) \ - __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, \ - __SMBDIRECT_SOCKET_DISCONNECT(__sc);) - -struct smbdirect_send_io { - struct smbdirect_socket *socket; - struct ib_cqe cqe; - - /* - * The SGE entries for this work request - * - * The first points to the packet header - */ -#define SMBDIRECT_SEND_IO_MAX_SGE 6 - size_t num_sge; - struct ib_sge sge[SMBDIRECT_SEND_IO_MAX_SGE]; - - /* - * Link to the list of sibling smbdirect_send_io - * messages. - */ - struct list_head sibling_list; - struct ib_send_wr wr; - - /* SMBD packet header follows this structure */ - u8 packet[]; -}; - -struct smbdirect_send_batch { - /* - * List of smbdirect_send_io messages - */ - struct list_head msg_list; - /* - * Number of list entries - */ - size_t wr_cnt; - - /* - * Possible remote key invalidation state - */ - bool need_invalidate_rkey; - u32 remote_key; - - int credit; -}; - -struct smbdirect_recv_io { - struct smbdirect_socket *socket; - struct ib_cqe cqe; - - /* - * For now we only use a single SGE - * as we have just one large buffer - * per posted recv. - */ -#define SMBDIRECT_RECV_IO_MAX_SGE 1 - struct ib_sge sge; - - /* Link to free or reassembly list */ - struct list_head list; - - /* Indicate if this is the 1st packet of a payload */ - bool first_segment; - - /* SMBD packet header and payload follows this structure */ - u8 packet[]; -}; - -enum smbdirect_mr_state { - SMBDIRECT_MR_READY, - SMBDIRECT_MR_REGISTERED, - SMBDIRECT_MR_INVALIDATED, - SMBDIRECT_MR_ERROR, - SMBDIRECT_MR_DISABLED -}; - -struct smbdirect_mr_io { - struct smbdirect_socket *socket; - struct ib_cqe cqe; - - /* - * We can have up to two references: - * 1. by the connection - * 2. by the registration - */ - struct kref kref; - struct mutex mutex; - - struct list_head list; - - enum smbdirect_mr_state state; - struct ib_mr *mr; - struct sg_table sgt; - enum dma_data_direction dir; - union { - struct ib_reg_wr wr; - struct ib_send_wr inv_wr; - }; - - bool need_invalidate; - struct completion invalidate_done; -}; - -struct smbdirect_rw_io { - struct smbdirect_socket *socket; - struct ib_cqe cqe; - - struct list_head list; - - int error; - struct completion *completion; - - struct rdma_rw_ctx rdma_ctx; - struct sg_table sgt; - struct scatterlist sg_list[]; -}; - -static inline size_t smbdirect_get_buf_page_count(const void *buf, size_t size) -{ - return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - - (uintptr_t)buf / PAGE_SIZE; -} - -/* - * Maximum number of retries on data transfer operations - */ -#define SMBDIRECT_RDMA_CM_RETRY 6 -/* - * No need to retry on Receiver Not Ready since SMB_DIRECT manages credits - */ -#define SMBDIRECT_RDMA_CM_RNR_RETRY 0 - -#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */ diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 8827b3653786..08d8b7a965a6 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -49,7 +49,7 @@ config SMB_SERVER_SMBDIRECT bool "Support for SMB Direct protocol" depends on SMB_SERVER && INFINIBAND && INFINIBAND_ADDR_TRANS depends on SMB_SERVER=m || INFINIBAND=y - select SMB_COMMON_SMBDIRECT + select SMBDIRECT default n help diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index 706a2c897948..a8242c00096f 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -18,7 +18,7 @@ #include "smb_common.h" #include "../common/smb2status.h" #include "transport_rdma.h" -#include "../common/smbdirect/smbdirect_public.h" +#include "../smbdirect/public.h" #define SMB_DIRECT_PORT_IWARP 5445 diff --git a/fs/smb/server/transport_rdma.h b/fs/smb/server/transport_rdma.h index 05352dc47f95..bde3d88aecc7 100644 --- a/fs/smb/server/transport_rdma.h +++ b/fs/smb/server/transport_rdma.h @@ -25,6 +25,6 @@ static inline void init_smbd_max_io_size(unsigned int sz) { } static inline unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { return 0; } #endif -#include "../common/smbdirect/smbdirect.h" +#include "../smbdirect/smbdirect.h" #endif /* __KSMBD_TRANSPORT_RDMA_H__ */ diff --git a/fs/smb/smbdirect/Kconfig b/fs/smb/smbdirect/Kconfig new file mode 100644 index 000000000000..2bf1b3350f7d --- /dev/null +++ b/fs/smb/smbdirect/Kconfig @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# smbdirect configuration + +config SMBDIRECT + def_tristate n + depends on INFINIBAND && INFINIBAND_ADDR_TRANS + depends on m || INFINIBAND=y + select SG_POOL diff --git a/fs/smb/smbdirect/Makefile b/fs/smb/smbdirect/Makefile new file mode 100644 index 000000000000..80d6025984d1 --- /dev/null +++ b/fs/smb/smbdirect/Makefile @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Makefile for smbdirect support +# + +obj-$(CONFIG_SMBDIRECT) += smbdirect.o + +smbdirect-y := \ + socket.o \ + connection.o \ + mr.o \ + rw.o \ + debug.o \ + connect.o \ + listen.o \ + accept.o \ + devices.o \ + main.o diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c new file mode 100644 index 000000000000..704b271af3a8 --- /dev/null +++ b/fs/smb/smbdirect/accept.c @@ -0,0 +1,857 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include +#include "../common/smb2status.h" + +static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event); +static int smbdirect_accept_init_params(struct smbdirect_socket *sc); +static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc); +static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc); + +int smbdirect_accept_connect_request(struct smbdirect_socket *sc, + const struct rdma_conn_param *param) +{ + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + u8 peer_initiator_depth; + u8 peer_responder_resources; + struct rdma_conn_param conn_param; + __be32 ird_ord_hdr[2]; + int ret; + + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED)) + return -EINVAL; + + /* + * First set what the we as server are able to support + */ + sp->initiator_depth = min_t(u8, sp->initiator_depth, + sc->ib.dev->attrs.max_qp_rd_atom); + + peer_initiator_depth = param->initiator_depth; + peer_responder_resources = param->responder_resources; + smbdirect_connection_negotiate_rdma_resources(sc, + peer_initiator_depth, + peer_responder_resources, + param); + + ret = smbdirect_accept_init_params(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_accept_init_params() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto init_params_failed; + } + + ret = smbdirect_connection_create_qp(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_qp() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto create_qp_failed; + } + + ret = smbdirect_connection_create_mem_pools(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_mem_pools() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto create_mem_failed; + } + + recv_io = smbdirect_connection_get_recv_io(sc); + if (WARN_ON_ONCE(!recv_io)) { + ret = -EINVAL; + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_get_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto get_recv_io_failed; + } + recv_io->cqe.done = smbdirect_accept_negotiate_recv_done; + + /* + * Now post the recv_io buffer in order to get + * the negotiate request + */ + sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ; + ret = smbdirect_connection_post_recv_io(recv_io); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto post_recv_io_failed; + } + /* + * From here recv_io is known to the RDMA QP and needs ib_drain_qp and + * smbdirect_accept_negotiate_recv_done to cleanup... + */ + recv_io = NULL; + + /* already checked with SMBDIRECT_CHECK_STATUS_WARN above */ + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; + + /* + * We already negotiated sp->initiator_depth + * and sp->responder_resources above. + */ + memset(&conn_param, 0, sizeof(conn_param)); + conn_param.initiator_depth = sp->initiator_depth; + conn_param.responder_resources = sp->responder_resources; + + if (sc->rdma.legacy_iwarp) { + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); + conn_param.private_data = ird_ord_hdr; + conn_param.private_data_len = sizeof(ird_ord_hdr); + } else { + conn_param.private_data = NULL; + conn_param.private_data_len = 0; + } + conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY; + conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY; + conn_param.flow_control = 0; + + /* explicitly set above */ + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED; + sc->rdma.cm_id->event_handler = smbdirect_accept_rdma_event_handler; + ret = rdma_accept(sc->rdma.cm_id, &conn_param); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_accept() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto rdma_accept_failed; + } + + /* + * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING + * so that the timer will cause a disconnect. + */ + INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->negotiate_timeout_msec)); + + return 0; + +rdma_accept_failed: + /* + * smbdirect_connection_destroy_qp() calls ib_drain_qp(), + * so that smbdirect_accept_negotiate_recv_done() will + * call smbdirect_connection_put_recv_io() + */ +post_recv_io_failed: + if (recv_io) + smbdirect_connection_put_recv_io(recv_io); +get_recv_io_failed: + smbdirect_connection_destroy_mem_pools(sc); +create_mem_failed: + smbdirect_connection_destroy_qp(sc); +create_qp_failed: +init_params_failed: + return ret; +} + +static int smbdirect_accept_init_params(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + int max_send_sges; + unsigned int maxpages; + + /* need 3 more sge. because a SMB_DIRECT header, SMB2 header, + * SMB2 response could be mapped. + */ + max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3; + if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) { + pr_err("max_send_size %d is too large\n", sp->max_send_size); + return -EINVAL; + } + + /* + * There is only a single batch credit + */ + atomic_set(&sc->send_io.bcredits.count, 1); + + /* + * Initialize the local credits to post + * IB_WR_SEND[_WITH_INV]. + */ + atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target); + + if (sp->max_read_write_size) { + maxpages = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE); + sc->rw_io.credits.max = rdma_rw_mr_factor(sc->ib.dev, + sc->rdma.cm_id->port_num, + maxpages); + sc->rw_io.credits.num_pages = DIV_ROUND_UP(maxpages, sc->rw_io.credits.max); + /* add one extra in order to handle unaligned pages */ + sc->rw_io.credits.max += 1; + } + + sc->recv_io.credits.target = 1; + + atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max); + + return 0; +} + +static void smbdirect_accept_negotiate_recv_work(struct work_struct *work); + +static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_recv_io *recv_io = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = recv_io->socket; + unsigned long flags; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + goto error; + } + + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, + "smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + /* + * This is an internal error! + */ + if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) + goto error; + + /* + * Don't reset timer to the keepalive interval in + * this will be done in smbdirect_accept_direct_negotiate_recv_work. + */ + + ib_dma_sync_single_for_cpu(sc->ib.dev, + recv_io->sge.addr, + recv_io->sge.length, + DMA_FROM_DEVICE); + + /* + * Only remember recv_io if it has enough bytes, + * this gives smbdirect_accept_negotiate_recv_work enough + * information in order to disconnect if it was not + * valid. + */ + sc->recv_io.reassembly.full_packet_received = true; + if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req)) + smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0); + else + smbdirect_connection_put_recv_io(recv_io); + + /* + * Some drivers (at least mlx5_ib and irdma) might post a + * recv completion before RDMA_CM_EVENT_ESTABLISHED, + * we need to adjust our expectation in that case. + * + * So we defer further processing of the negotiation + * to smbdirect_accept_negotiate_recv_work(). + * + * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED + * we queue the work directly otherwise + * smbdirect_accept_rdma_event_handler() will do it, when + * RDMA_CM_EVENT_ESTABLISHED arrived. + */ + spin_lock_irqsave(&sc->connect.lock, flags); + if (!sc->first_error) { + INIT_WORK(&sc->connect.work, smbdirect_accept_negotiate_recv_work); + if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) + queue_work(sc->workqueues.accept, &sc->connect.work); + } + spin_unlock_irqrestore(&sc->connect.lock, flags); + + return; + +error: + /* + * recv_io.posted.refill_work is still disabled, + * so smbdirect_connection_put_recv_io() won't + * start it. + */ + smbdirect_connection_put_recv_io(recv_io); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); +} + +static void smbdirect_accept_negotiate_recv_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, connect.work); + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + struct smbdirect_negotiate_req *nreq; + unsigned long flags; + u16 min_version; + u16 max_version; + u16 credits_requested; + u32 preferred_send_size; + u32 max_receive_size; + u32 max_fragmented_size; + u32 ntstatus; + + if (sc->first_error) + return; + + /* + * make sure we won't start again... + */ + disable_work(work); + + /* + * Reset timer to the keepalive interval in + * order to trigger our next keepalive message. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_interval_msec)); + + /* + * If smbdirect_accept_negotiate_recv_done() detected an + * invalid request we want to disconnect. + */ + recv_io = smbdirect_connection_reassembly_first_recv_io(sc); + if (!recv_io) { + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + sc->recv_io.reassembly.queue_length--; + list_del(&recv_io->list); + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + smbdirect_connection_put_recv_io(recv_io); + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) + return; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; + + /* + * Note recv_io is already part of the free list, + * as we just called smbdirect_connection_put_recv_io(), + * but it won't be reused before we call + * smbdirect_connection_recv_io_refill() below. + */ + + nreq = (struct smbdirect_negotiate_req *)recv_io->packet; + min_version = le16_to_cpu(nreq->min_version); + max_version = le16_to_cpu(nreq->max_version); + credits_requested = le16_to_cpu(nreq->credits_requested); + preferred_send_size = le32_to_cpu(nreq->preferred_send_size); + max_receive_size = le32_to_cpu(nreq->max_receive_size); + max_fragmented_size = le32_to_cpu(nreq->max_fragmented_size); + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "ReqIn: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nreq->min_version), + "MaxVersion=0x", + le16_to_cpu(nreq->max_version), + "CreditsRequested=", + le16_to_cpu(nreq->credits_requested), + "PreferredSendSize=", + le32_to_cpu(nreq->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nreq->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nreq->max_fragmented_size)); + + if (!(min_version <= SMBDIRECT_V1 && max_version >= SMBDIRECT_V1)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: min_version=0x%x max_version=0x%x\n", + min_version, max_version); + ntstatus = le32_to_cpu(STATUS_NOT_SUPPORTED); + goto not_supported; + } + + if (credits_requested == 0) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_requested == 0\n"); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_receive_size=%u < %u\n", + max_receive_size, + SMBDIRECT_MIN_RECEIVE_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_fragmented_size=%u < %u\n", + max_fragmented_size, + SMBDIRECT_MIN_FRAGMENTED_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used. + */ + sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size); + sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE); + + /* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smb_direct_receive_credit_max - 24 = 1340 + * + * The maximum number would be + * smb_direct_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + * + * We need to adjust this here in case the peer + * lowered sp->max_recv_size. + * + * TODO: instead of adjusting max_fragmented_recv_size + * we should adjust the number of available buffers, + * but for now we keep the logic as it was used + * in ksmbd before. + */ + sp->max_fragmented_recv_size = (sp->recv_credit_max * sp->max_recv_size) / 2; + + /* + * We take the value from the peer, which is checked to be higher than 0, + * but we limit it to the max value we support in order to have + * the main logic simpler. + */ + sc->recv_io.credits.target = credits_requested; + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, + sp->recv_credit_max); + + /* + * Note nreq->max_receive_size was already checked against + * SMBDIRECT_MIN_RECEIVE_SIZE above. + */ + sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size); + + /* + * Note nreq->max_fragmented_size was already checked against + * SMBDIRECT_MIN_FRAGMENTED_SIZE above. + */ + sp->max_fragmented_send_size = max_fragmented_size; + + if (sc->accept.listener) { + struct smbdirect_socket *lsc = sc->accept.listener; + unsigned long flags; + + spin_lock_irqsave(&lsc->listen.lock, flags); + list_del(&sc->accept.list); + list_add_tail(&sc->accept.list, &lsc->listen.ready); + wake_up(&lsc->listen.wait_queue); + spin_unlock_irqrestore(&lsc->listen.lock, flags); + + /* + * smbdirect_socket_accept() will call + * smbdirect_accept_negotiate_finish(nsc, 0); + * + * So that we don't send the negotiation + * response that grants credits to the peer + * before the socket is accepted by the + * application. + */ + return; + } + + ntstatus = le32_to_cpu(STATUS_SUCCESS); + +not_supported: + smbdirect_accept_negotiate_finish(sc, ntstatus); +} + +void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + struct smbdirect_send_io *send_io; + struct smbdirect_negotiate_resp *nrep; + int posted; + u16 new_credits; + int ret; + + if (ntstatus) + goto not_supported; + + /* + * Prepare for receiving data_transfer messages + */ + sc->recv_io.reassembly.full_packet_received = true; + sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; + list_for_each_entry(recv_io, &sc->recv_io.free.list, list) + recv_io->cqe.done = smbdirect_connection_recv_io_done; + recv_io = NULL; + + /* + * We should at least post 1 smbdirect_recv_io! + */ + posted = smbdirect_connection_recv_io_refill(sc); + if (posted < 1) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_recv_io_refill() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(posted)); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * The response will grant credits for all posted + * smbdirect_recv_io messages. + */ + new_credits = smbdirect_connection_grant_recv_credits(sc); + +not_supported: + send_io = smbdirect_connection_alloc_send_io(sc); + if (IS_ERR(send_io)) { + ret = PTR_ERR(send_io); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_alloc_send_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + send_io->cqe.done = smbdirect_accept_negotiate_send_done; + + nrep = (struct smbdirect_negotiate_resp *)send_io->packet; + nrep->min_version = cpu_to_le16(SMBDIRECT_V1); + nrep->max_version = cpu_to_le16(SMBDIRECT_V1); + if (ntstatus == 0) { + nrep->negotiated_version = cpu_to_le16(SMBDIRECT_V1); + nrep->reserved = 0; + nrep->credits_requested = cpu_to_le16(sp->send_credit_target); + nrep->credits_granted = cpu_to_le16(new_credits); + nrep->status = cpu_to_le32(ntstatus); + nrep->max_readwrite_size = cpu_to_le32(sp->max_read_write_size); + nrep->preferred_send_size = cpu_to_le32(sp->max_send_size); + nrep->max_receive_size = cpu_to_le32(sp->max_recv_size); + nrep->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + } else { + nrep->negotiated_version = 0; + nrep->reserved = 0; + nrep->credits_requested = 0; + nrep->credits_granted = 0; + nrep->status = cpu_to_le32(ntstatus); + nrep->max_readwrite_size = 0; + nrep->preferred_send_size = 0; + nrep->max_receive_size = 0; + nrep->max_fragmented_size = 0; + } + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "RepOut: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nrep->min_version), + "MaxVersion=0x", + le16_to_cpu(nrep->max_version), + "NegotiatedVersion=0x", + le16_to_cpu(nrep->negotiated_version), + "CreditsRequested=", + le16_to_cpu(nrep->credits_requested), + "CreditsGranted=", + le16_to_cpu(nrep->credits_granted), + "Status=0x", + le32_to_cpu(nrep->status), + "MaxReadWriteSize=", + le32_to_cpu(nrep->max_readwrite_size), + "PreferredSendSize=", + le32_to_cpu(nrep->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nrep->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nrep->max_fragmented_size)); + + send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev, + nrep, + sizeof(*nrep), + DMA_TO_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "ib_dma_mapping_error() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_connection_free_send_io(send_io); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + + send_io->sge[0].length = sizeof(*nrep); + send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey; + send_io->num_sge = 1; + + ib_dma_sync_single_for_device(sc->ib.dev, + send_io->sge[0].addr, + send_io->sge[0].length, + DMA_TO_DEVICE); + + send_io->wr.next = NULL; + send_io->wr.wr_cqe = &send_io->cqe; + send_io->wr.sg_list = send_io->sge; + send_io->wr.num_sge = send_io->num_sge; + send_io->wr.opcode = IB_WR_SEND; + send_io->wr.send_flags = IB_SEND_SIGNALED; + + ret = smbdirect_connection_post_send_wr(sc, &send_io->wr); + if (ret) { + /* if we reach here, post send failed */ + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_send_wr() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + /* + * Note smbdirect_connection_free_send_io() + * does ib_dma_unmap_page() + */ + smbdirect_connection_free_send_io(send_io); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + + /* + * smbdirect_accept_negotiate_send_done + * will do all remaining work... + */ +} + +static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_send_io *send_io = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = send_io->socket; + struct smbdirect_negotiate_resp *nrep; + u32 ntstatus; + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + nrep = (struct smbdirect_negotiate_resp *)send_io->packet; + ntstatus = le32_to_cpu(nrep->status); + + /* Note this frees wc->wr_cqe, but not wc */ + smbdirect_connection_free_send_io(send_io); + atomic_dec(&sc->send_io.pending.count); + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * If we send a smbdirect_negotiate_resp without NT_STATUS_OK (0) + * we need to disconnect now. + * + * Otherwise smbdirect_connection_negotiation_done() + * will setup all required things and wake up + * the waiter. + */ + if (ntstatus) + smbdirect_socket_schedule_cleanup(sc, -EOPNOTSUPP); + else + smbdirect_connection_negotiation_done(sc); +} + +static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + unsigned long flags; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here, + * as we might sleep. + * + * We didn't timeout so we cancel our idle timer, + * it will be scheduled again if needed. + */ + WARN_ON_ONCE(in_interrupt()); + + if (event->status || event->event != sc->rdma.expected_event) { + int ret = -ECONNABORTED; + + if (event->event == RDMA_CM_EVENT_REJECTED) + ret = -ECONNREFUSED; + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(event->event)); + + if (sc->first_error) + return 0; + + switch (event->event) { + case RDMA_CM_EVENT_ESTABLISHED: + smbdirect_connection_rdma_established(sc); + + /* + * Some drivers (at least mlx5_ib and irdma) might post a + * recv completion before RDMA_CM_EVENT_ESTABLISHED, + * we need to adjust our expectation in that case. + * + * If smbdirect_accept_negotiate_recv_done was called first + * it initialized sc->connect.work only for us to + * start, so that we turned into + * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before + * smbdirect_accept_negotiate_recv_work() runs. + * + * If smbdirect_accept_negotiate_recv_done didn't happen + * yet. sc->connect.work is still be disabled and + * queue_work() is a no-op. + */ + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; + spin_lock_irqsave(&sc->connect.lock, flags); + if (!sc->first_error) + queue_work(sc->workqueues.accept, &sc->connect.work); + spin_unlock_irqrestore(&sc->connect.lock, flags); + + /* + * wait for smbdirect_accept_negotiate_recv_done() + * to get the negotiate request. + */ + return 0; + + default: + break; + } + + /* + * This is an internal error + */ + WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return 0; +} + +static long smbdirect_socket_wait_for_accept(struct smbdirect_socket *lsc, long timeo) +{ + long ret; + + ret = wait_event_interruptible_timeout(lsc->listen.wait_queue, + !list_empty_careful(&lsc->listen.ready) || + lsc->status != SMBDIRECT_SOCKET_LISTENING || + lsc->first_error, + timeo); + if (lsc->status != SMBDIRECT_SOCKET_LISTENING) + return -EINVAL; + if (lsc->first_error) + return lsc->first_error; + if (!ret) + ret = -ETIMEDOUT; + if (ret < 0) + return ret; + + return 0; +} + +struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, + long timeo, + struct proto_accept_arg *arg) +{ + struct smbdirect_socket *nsc; + unsigned long flags; + + if (lsc->status != SMBDIRECT_SOCKET_LISTENING) { + arg->err = -EINVAL; + return NULL; + } + + if (lsc->first_error) { + arg->err = lsc->first_error; + return NULL; + } + + if (list_empty_careful(&lsc->listen.ready)) { + int ret; + + if (timeo == 0) { + arg->err = -EAGAIN; + return NULL; + } + + ret = smbdirect_socket_wait_for_accept(lsc, timeo); + if (ret) { + arg->err = ret; + return NULL; + } + } + + spin_lock_irqsave(&lsc->listen.lock, flags); + nsc = list_first_entry_or_null(&lsc->listen.ready, + struct smbdirect_socket, + accept.list); + if (nsc) { + nsc->accept.listener = NULL; + list_del_init_careful(&nsc->accept.list); + arg->is_empty = list_empty_careful(&lsc->listen.ready); + } + spin_unlock_irqrestore(&lsc->listen.lock, flags); + if (!nsc) { + arg->err = -EAGAIN; + return NULL; + } + + /* + * We did not send the negotiation response + * yet, so we did not grant any credits to the client, + * so it didn't grant any credits to us. + * + * The caller expects a connected socket + * now as there are no credits anyway. + * + * Then we send the negotiation response in + * order to grant credits to the peer. + */ + nsc->status = SMBDIRECT_SOCKET_CONNECTED; + smbdirect_accept_negotiate_finish(nsc, 0); + + return nsc; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_accept); diff --git a/fs/smb/smbdirect/connect.c b/fs/smb/smbdirect/connect.c new file mode 100644 index 000000000000..8addee43a381 --- /dev/null +++ b/fs/smb/smbdirect/connect.c @@ -0,0 +1,925 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2012,2016,2017,2025 Stefan Metzmacher + */ + +#include "internal.h" +#include "../common/smb2status.h" + +static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc); +static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc, + const struct sockaddr *src, + const struct sockaddr *dst); +static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event); +static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc); +static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc); +static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc); + +int smbdirect_connect(struct smbdirect_socket *sc, const struct sockaddr *dst) +{ + const struct sockaddr *src = NULL; + union { + struct sockaddr sa; + struct sockaddr_storage ss; + } src_addr = { + .sa = { + .sa_family = AF_UNSPEC, + }, + }; + int ret; + + if (sc->first_error) + return -ENOTCONN; + + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EALREADY; + + if (WARN_ON_ONCE(!sc->rdma.cm_id)) + return -EINVAL; + + src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; + if (src_addr.sa.sa_family != AF_UNSPEC) + src = &src_addr.sa; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "connect: src: %pISpsfc dst: %pISpsfc\n", + src, dst); + + ret = smbdirect_connect_setup_connection(sc); + if (ret) + return ret; + + ret = smbdirect_connect_resolve_addr(sc, src, dst); + if (ret) + return ret; + + /* + * The rest happens async via smbdirect_connect_rdma_event_handler() + * the caller will decide to wait or not. + */ + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect); + +static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc) +{ + rdma_lock_handler(sc->rdma.cm_id); + sc->rdma.cm_id->event_handler = smbdirect_connect_rdma_event_handler; + rdma_unlock_handler(sc->rdma.cm_id); + + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED)) + return -EINVAL; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED; + + return 0; +} + +static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc, + const struct sockaddr *src, + const struct sockaddr *dst) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct sockaddr *src_addr = NULL; + struct sockaddr *dst_addr = NULL; + int ret; + + src_addr = (struct sockaddr *)src; + if (src_addr && src_addr->sa_family == AF_UNSPEC) + src_addr = NULL; + dst_addr = (struct sockaddr *)dst; + + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED)) + return -EINVAL; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ADDR_RESOLVED; + ret = rdma_resolve_addr(sc->rdma.cm_id, src_addr, dst_addr, + sp->resolve_addr_timeout_msec); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_resolve_addr() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} + +static int smbdirect_connect_resolve_route(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + int ret; + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED)) + return sc->first_error; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ROUTE_RESOLVED; + ret = rdma_resolve_route(sc->rdma.cm_id, sp->resolve_route_timeout_msec); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_resolve_route() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} + +static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc) +{ + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct rdma_conn_param conn_param; + __be32 ird_ord_hdr[2]; + int ret; + + sc->ib.dev = sc->rdma.cm_id->device; + + if (!smbdirect_frwr_is_supported(&sc->ib.dev->attrs)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Fast Registration Work Requests (FRWR) is not supported device %.*s\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", + sc->ib.dev->attrs.device_cap_flags, + sc->ib.dev->attrs.max_fast_reg_page_list_len); + return -EPROTONOSUPPORT; + } + + if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && + !rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num), + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW && + !rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num), + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "rdma connect: device: %.*s local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + + sp->max_frmr_depth = min_t(u32, sp->max_frmr_depth, + sc->ib.dev->attrs.max_fast_reg_page_list_len); + sc->mr_io.type = IB_MR_TYPE_MEM_REG; + if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) + sc->mr_io.type = IB_MR_TYPE_SG_GAPS; + + sp->responder_resources = min_t(u8, sp->responder_resources, + sc->ib.dev->attrs.max_qp_rd_atom); + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, + "responder_resources=%d\n", + sp->responder_resources); + + ret = smbdirect_connection_create_qp(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_qp() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + memset(&conn_param, 0, sizeof(conn_param)); + conn_param.initiator_depth = sp->initiator_depth; + conn_param.responder_resources = sp->responder_resources; + + /* Need to send IRD/ORD in private data for iWARP */ + if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) { + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); + conn_param.private_data = ird_ord_hdr; + conn_param.private_data_len = sizeof(ird_ord_hdr); + } else { + conn_param.private_data = NULL; + conn_param.private_data_len = 0; + } + + conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY; + conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY; + conn_param.flow_control = 0; + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED)) + return sc->first_error; + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED; + ret = rdma_connect_locked(sc->rdma.cm_id, &conn_param); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_connect_locked() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + /* + * start with the rdma connect timeout and SMBDIRECT_KEEPALIVE_PENDING + * so that the timer will cause a disconnect. + */ + INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->rdma_connect_timeout_msec)); + + return 0; +} + +static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + u8 peer_initiator_depth; + u8 peer_responder_resources; + int ret; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here, + * as we might sleep. + * + * We didn't timeout so we cancel our idle timer, + * it will be scheduled again if needed. + */ + WARN_ON_ONCE(in_interrupt()); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + cancel_delayed_work_sync(&sc->idle.timer_work); + + if (event->status || event->event != sc->rdma.expected_event) { + int lvl = SMBDIRECT_LOG_ERR; + + ret = -ECONNABORTED; + + if (event->event == RDMA_CM_EVENT_REJECTED) + ret = -ECONNREFUSED; + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + if (ret == -ENODEV) + lvl = SMBDIRECT_LOG_INFO; + + smbdirect_log_rdma_event(sc, lvl, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + smbdirect_socket_schedule_cleanup_lvl(sc, + lvl, + ret); + return 0; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(event->event)); + + if (sc->first_error) + return 0; + + switch (event->event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED; + + ret = smbdirect_connect_resolve_route(sc); + if (ret) + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + + case RDMA_CM_EVENT_ROUTE_RESOLVED: + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; + + ret = smbdirect_connect_rdma_connect(sc); + if (ret) + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + + case RDMA_CM_EVENT_ESTABLISHED: + smbdirect_connection_rdma_established(sc); + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; + + /* + * Here we work around an inconsistency between + * iWarp and other devices (at least rxe and irdma using RoCEv2) + */ + if (rdma_protocol_iwarp(id->device, id->port_num)) { + /* + * iWarp devices report the peer's values + * with the perspective of the peer here. + * Tested with siw and irdma (in iwarp mode) + * We need to change to our perspective here, + * so we need to switch the values. + */ + peer_initiator_depth = event->param.conn.responder_resources; + peer_responder_resources = event->param.conn.initiator_depth; + } else { + /* + * Non iWarp devices report the peer's values + * already changed to our perspective here. + * Tested with rxe and irdma (in roce mode). + */ + peer_initiator_depth = event->param.conn.initiator_depth; + peer_responder_resources = event->param.conn.responder_resources; + } + smbdirect_connection_negotiate_rdma_resources(sc, + peer_initiator_depth, + peer_responder_resources, + &event->param.conn); + + ret = smbdirect_connect_negotiate_start(sc); + if (ret) + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + + default: + break; + } + + /* + * This is an internal error + */ + WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return 0; +} + +static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io = NULL; + struct smbdirect_send_io *send_io = NULL; + struct smbdirect_negotiate_req *nreq = NULL; + int ret; + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) + return sc->first_error; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; + + ret = smbdirect_connection_create_mem_pools(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_mem_pools() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto create_mem_pools_failed; + } + + /* + * There is only a single batch credit + */ + atomic_set(&sc->send_io.bcredits.count, 1); + + /* + * Initialize the local credits to post + * IB_WR_SEND[_WITH_INV]. + */ + atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target); + + recv_io = smbdirect_connection_get_recv_io(sc); + if (WARN_ON_ONCE(!recv_io)) { + ret = -EINVAL; + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_get_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto get_recv_io_failed; + } + recv_io->cqe.done = smbdirect_connect_negotiate_recv_done; + + send_io = smbdirect_connection_alloc_send_io(sc); + if (IS_ERR(send_io)) { + ret = PTR_ERR(send_io); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_alloc_send_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto alloc_send_io_failed; + } + send_io->cqe.done = smbdirect_connect_negotiate_send_done; + + nreq = (struct smbdirect_negotiate_req *)send_io->packet; + nreq->min_version = cpu_to_le16(SMBDIRECT_V1); + nreq->max_version = cpu_to_le16(SMBDIRECT_V1); + nreq->reserved = 0; + nreq->credits_requested = cpu_to_le16(sp->send_credit_target); + nreq->preferred_send_size = cpu_to_le32(sp->max_send_size); + nreq->max_receive_size = cpu_to_le32(sp->max_recv_size); + nreq->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "ReqOut: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nreq->min_version), + "MaxVersion=0x", + le16_to_cpu(nreq->max_version), + "CreditsRequested=", + le16_to_cpu(nreq->credits_requested), + "PreferredSendSize=", + le32_to_cpu(nreq->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nreq->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nreq->max_fragmented_size)); + + send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev, + nreq, + sizeof(*nreq), + DMA_TO_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "ib_dma_mapping_error() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto dma_mapping_failed; + } + + send_io->sge[0].length = sizeof(*nreq); + send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey; + send_io->num_sge = 1; + + ib_dma_sync_single_for_device(sc->ib.dev, + send_io->sge[0].addr, + send_io->sge[0].length, + DMA_TO_DEVICE); + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "sge addr=0x%llx length=%u lkey=0x%x\n", + send_io->sge[0].addr, + send_io->sge[0].length, + send_io->sge[0].lkey); + + /* + * Now post the recv_io buffer in order to get + * the negotiate response + */ + sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP; + ret = smbdirect_connection_post_recv_io(recv_io); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto post_recv_io_failed; + } + + send_io->wr.next = NULL; + send_io->wr.wr_cqe = &send_io->cqe; + send_io->wr.sg_list = send_io->sge; + send_io->wr.num_sge = send_io->num_sge; + send_io->wr.opcode = IB_WR_SEND; + send_io->wr.send_flags = IB_SEND_SIGNALED; + + ret = smbdirect_connection_post_send_wr(sc, &send_io->wr); + if (ret) { + /* if we reach here, post send failed */ + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_send_wr() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto post_send_wr_failed; + } + + /* + * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING + * so that the timer will cause a disconnect. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->negotiate_timeout_msec)); + + return 0; + +post_send_wr_failed: + /* + * ib_dma_unmap_single is called in + * smbdirect_connection_free_send_io() + */ + smbdirect_connection_free_send_io(send_io); + /* + * recv_io is given to the rdma layer, + * we should not put it even on error + * nor call smbdirect_connection_destroy_mem_pools() + * it will be cleaned up during disconnect. + */ + return ret; + +post_recv_io_failed: + /* + * ib_dma_unmap_single is called in + * smbdirect_connection_free_send_io() + */ +dma_mapping_failed: + smbdirect_connection_free_send_io(send_io); + +alloc_send_io_failed: + smbdirect_connection_put_recv_io(recv_io); + +get_recv_io_failed: + smbdirect_connection_destroy_mem_pools(sc); + +create_mem_pools_failed: + return ret; +} + +static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_send_io *send_io = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = send_io->socket; + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + /* Note this frees wc->wr_cqe, but not wc */ + smbdirect_connection_free_send_io(send_io); + atomic_dec(&sc->send_io.pending.count); + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } +} + +static void smbdirect_connect_negotiate_recv_work(struct work_struct *work); + +static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_recv_io *recv_io = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = recv_io->socket; + unsigned long flags; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + goto error; + } + + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, + "smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + /* + * This is an internal error! + */ + if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REP)) + goto error; + + /* + * Don't reset timer to the keepalive interval in + * this will be done in smbdirect_accept_direct_negotiate_recv_work. + */ + + ib_dma_sync_single_for_cpu(sc->ib.dev, + recv_io->sge.addr, + recv_io->sge.length, + DMA_FROM_DEVICE); + + /* + * Only remember recv_io if it has enough bytes, + * this gives smbdirect_accept_negotiate_recv_work enough + * information in order to disconnect if it was not + * valid. + */ + sc->recv_io.reassembly.full_packet_received = true; + if (wc->byte_len >= sizeof(struct smbdirect_negotiate_resp)) + smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0); + else + smbdirect_connection_put_recv_io(recv_io); + + /* + * We continue via the workqueue as we may have + * complex work that might sleep. + * + * So we defer further processing of the negotiation + * to smbdirect_connect_negotiate_recv_work(). + */ + spin_lock_irqsave(&sc->connect.lock, flags); + if (!sc->first_error) { + INIT_WORK(&sc->connect.work, smbdirect_connect_negotiate_recv_work); + if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) + queue_work(sc->workqueues.connect, &sc->connect.work); + } + spin_unlock_irqrestore(&sc->connect.lock, flags); + + return; + +error: + /* + * recv_io.posted.refill_work is still disabled, + * so smbdirect_connection_put_recv_io() won't + * start it. + */ + smbdirect_connection_put_recv_io(recv_io); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); +} + +static void smbdirect_connect_negotiate_recv_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, connect.work); + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + struct smbdirect_negotiate_resp *nrep; + unsigned long flags; + u16 negotiated_version; + u16 credits_requested; + u16 credits_granted; + u32 status; + u32 max_readwrite_size; + u32 preferred_send_size; + u32 max_receive_size; + u32 max_fragmented_size; + int posted; + int ret; + + if (sc->first_error) + return; + + /* + * make sure we won't start again... + */ + disable_work(work); + + /* + * Reset timer to the keepalive interval in + * order to trigger our next keepalive message. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_interval_msec)); + + /* + * If smbdirect_connect_negotiate_recv_done() detected an + * invalid request we want to disconnect. + */ + recv_io = smbdirect_connection_reassembly_first_recv_io(sc); + if (!recv_io) { + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + sc->recv_io.reassembly.queue_length--; + list_del(&recv_io->list); + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + smbdirect_connection_put_recv_io(recv_io); + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)) + return; + + /* + * Note recv_io is already part of the free list, + * as we just called smbdirect_connection_put_recv_io(), + * but it won't be reused before we call + * smbdirect_connection_recv_io_refill() below. + */ + + nrep = (struct smbdirect_negotiate_resp *)recv_io->packet; + negotiated_version = le16_to_cpu(nrep->negotiated_version); + credits_requested = le16_to_cpu(nrep->credits_requested); + credits_granted = le16_to_cpu(nrep->credits_granted); + status = le32_to_cpu(nrep->status); + max_readwrite_size = le32_to_cpu(nrep->max_readwrite_size); + preferred_send_size = le32_to_cpu(nrep->preferred_send_size); + max_receive_size = le32_to_cpu(nrep->max_receive_size); + max_fragmented_size = le32_to_cpu(nrep->max_fragmented_size); + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "RepIn: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nrep->min_version), + "MaxVersion=0x", + le16_to_cpu(nrep->max_version), + "NegotiatedVersion=0x", + le16_to_cpu(nrep->negotiated_version), + "CreditsRequested=", + le16_to_cpu(nrep->credits_requested), + "CreditsGranted=", + le16_to_cpu(nrep->credits_granted), + "Status=0x", + le32_to_cpu(nrep->status), + "MaxReadWriteSize=", + le32_to_cpu(nrep->max_readwrite_size), + "PreferredSendSize=", + le32_to_cpu(nrep->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nrep->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nrep->max_fragmented_size)); + + if (negotiated_version != SMBDIRECT_V1) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: negotiated_version=0x%x\n", + negotiated_version); + smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED); + return; + } + + if (status != le32_to_cpu(STATUS_SUCCESS)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: status=0x%x != 0x0\n", + status); + smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED); + return; + } + + if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_receive_size=%u < %u\n", + max_receive_size, + SMBDIRECT_MIN_RECEIVE_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_fragmented_size=%u < %u\n", + max_fragmented_size, + SMBDIRECT_MIN_FRAGMENTED_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (credits_granted == 0) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_granted == 0\n"); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (credits_requested == 0) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_requested == 0\n"); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (preferred_send_size > sp->max_recv_size) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: preferred_send_size=%u > max_recv_size=%u\n", + preferred_send_size, + sp->max_recv_size); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * We take the value from the peer, which is checked to be higher than 0, + * but we limit it to the max value we support in order to have + * the main logic simpler. + */ + sc->recv_io.credits.target = credits_requested; + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, + sp->recv_credit_max); + + /* + * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used. + */ + sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size); + sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE); + + /* + * We already sent our sp->max_fragmented_recv_size + * to the peer, so we can't lower it here any more. + * + * TODO: but if the peer lowered sp->max_recv_size + * we will have to adjust our number of buffers. + * + * But for now we keep it as the cifs.ko code + * worked before. + */ + + /* + * Note nrep->max_receive_size was already checked against + * SMBDIRECT_MIN_RECEIVE_SIZE above. + */ + sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size); + + /* + * Make sure the resulting max_frmr_depth is at least 1, + * which means max_read_write_size needs to be at least PAGE_SIZE. + */ + sp->max_read_write_size = min_t(u32, sp->max_frmr_depth * PAGE_SIZE, + max_readwrite_size); + if (sp->max_read_write_size < PAGE_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_readwrite_size=%u < PAGE_SIZE(%lu)\n", + max_readwrite_size, + PAGE_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; + + /* + * Note nrep->credits_granted was already checked against 0 above. + */ + atomic_set(&sc->send_io.credits.count, credits_granted); + + /* + * Note nrep->max_fragmented_size was already checked against + * SMBDIRECT_MIN_FRAGMENTED_SIZE above. + */ + sp->max_fragmented_send_size = max_fragmented_size; + + ret = smbdirect_connection_create_mr_list(sc); + if (ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_mr_list() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + + /* + * Prepare for receiving data_transfer messages + */ + sc->recv_io.reassembly.full_packet_received = true; + sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; + list_for_each_entry(recv_io, &sc->recv_io.free.list, list) + recv_io->cqe.done = smbdirect_connection_recv_io_done; + recv_io = NULL; + + /* + * We should at least post 1 smbdirect_recv_io! + */ + posted = smbdirect_connection_recv_io_refill(sc); + if (posted < 1) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_recv_io_refill() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * smbdirect_connection_negotiation_done() + * will setup all required things and wake up + * the waiter. + */ + smbdirect_connection_negotiation_done(sc); +} + +int smbdirect_connect_sync(struct smbdirect_socket *sc, + const struct sockaddr *dst) +{ + int ret; + + ret = smbdirect_connect(sc, dst); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connect(%pISpsfc) failed %1pe\n", + dst, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + ret = smbdirect_connection_wait_for_connected(sc); + if (ret) { + int lvl = SMBDIRECT_LOG_ERR; + + if (ret == -ENODEV) + lvl = SMBDIRECT_LOG_INFO; + + smbdirect_log_rdma_event(sc, lvl, + "wait for smbdirect_connect(%pISpsfc) failed %1pe\n", + dst, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connect_sync); diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c new file mode 100644 index 000000000000..822366718d45 --- /dev/null +++ b/fs/smb/smbdirect/connection.c @@ -0,0 +1,2181 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include + +struct smbdirect_map_sges { + struct ib_sge *sge; + size_t num_sge; + size_t max_sge; + struct ib_device *device; + u32 local_dma_lkey; + enum dma_data_direction direction; +}; + +static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, + struct smbdirect_map_sges *state); + +static void smbdirect_connection_recv_io_refill_work(struct work_struct *work); +static void smbdirect_connection_send_immediate_work(struct work_struct *work); + +static void smbdirect_connection_qp_event_handler(struct ib_event *event, void *context) +{ + struct smbdirect_socket *sc = context; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s on device %.*s socket %p (cm_id=%p) status %s first_error %1pe\n", + ib_event_msg(event->event), + IB_DEVICE_NAME_MAX, + event->device->name, + sc, sc->rdma.cm_id, + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + switch (event->event) { + case IB_EVENT_CQ_ERR: + case IB_EVENT_QP_FATAL: + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + break; + + default: + break; + } +} + +static int smbdirect_connection_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + int ret = -ECONNRESET; + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here. + */ + WARN_ON_ONCE(in_interrupt()); + + if (event->event != sc->rdma.expected_event) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + /* + * If we get RDMA_CM_EVENT_DEVICE_REMOVAL, + * we should change to SMBDIRECT_SOCKET_DISCONNECTED, + * so that rdma_disconnect() is avoided later via + * smbdirect_socket_schedule_cleanup[_status]() => + * smbdirect_socket_cleanup_work(). + * + * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING, + * but never ever get RDMA_CM_EVENT_DISCONNECTED and + * never reach SMBDIRECT_SOCKET_DISCONNECTED. + */ + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + smbdirect_socket_schedule_cleanup_status(sc, + SMBDIRECT_LOG_ERR, + ret, + SMBDIRECT_SOCKET_DISCONNECTED); + else + smbdirect_socket_schedule_cleanup(sc, ret); + if (sc->ib.qp) + ib_drain_qp(sc->ib.qp); + return 0; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(event->event)); + + switch (event->event) { + case RDMA_CM_EVENT_DISCONNECTED: + /* + * We need to change to SMBDIRECT_SOCKET_DISCONNECTED, + * so that rdma_disconnect() is avoided later via + * smbdirect_socket_schedule_cleanup_status() => + * smbdirect_socket_cleanup_work(). + * + * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING, + * but never ever get RDMA_CM_EVENT_DISCONNECTED and + * never reach SMBDIRECT_SOCKET_DISCONNECTED. + * + * This is also a normal disconnect so + * SMBDIRECT_LOG_INFO should be good enough + * and avoids spamming the default logs. + */ + smbdirect_socket_schedule_cleanup_status(sc, + SMBDIRECT_LOG_INFO, + ret, + SMBDIRECT_SOCKET_DISCONNECTED); + if (sc->ib.qp) + ib_drain_qp(sc->ib.qp); + return 0; + + default: + break; + } + + /* + * This is an internal error, should be handled above via + * event->event != sc->rdma.expected_event already. + */ + WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_DISCONNECTED); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return 0; +} + +void smbdirect_connection_rdma_established(struct smbdirect_socket *sc) +{ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "rdma established: device: %.*s local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + + sc->rdma.cm_id->event_handler = smbdirect_connection_rdma_event_handler; + sc->rdma.expected_event = RDMA_CM_EVENT_DISCONNECTED; +} + +void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc) +{ + if (unlikely(sc->first_error)) + return; + + if (sc->status == SMBDIRECT_SOCKET_CONNECTED) + /* + * This is the accept case where + * smbdirect_socket_accept() already sets + * SMBDIRECT_SOCKET_CONNECTED + */ + goto done; + + if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) { + /* + * Something went wrong... + */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "status=%s first_error=%1pe local: %pISpsfc remote: %pISpsfc\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + return; + } + + /* + * We are done, so we can wake up the waiter. + */ + WARN_ONCE(sc->status == SMBDIRECT_SOCKET_CONNECTED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + sc->status = SMBDIRECT_SOCKET_CONNECTED; + + /* + * We need to setup the refill and send immediate work + * in order to get a working connection. + */ +done: + INIT_WORK(&sc->recv_io.posted.refill_work, smbdirect_connection_recv_io_refill_work); + INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work); + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "negotiated: local: %pISpsfc remote: %pISpsfc\n", + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + + wake_up(&sc->status_wait); +} + +static u32 smbdirect_rdma_rw_send_wrs(struct ib_device *dev, + const struct ib_qp_init_attr *attr) +{ + /* + * This could be split out of rdma_rw_init_qp() + * and be a helper function next to rdma_rw_mr_factor() + * + * We can't check unlikely(rdma_rw_force_mr) here, + * but that is most likely 0 anyway. + */ + u32 factor; + + WARN_ON_ONCE(attr->port_num == 0); + + /* + * Each context needs at least one RDMA READ or WRITE WR. + * + * For some hardware we might need more, eventually we should ask the + * HCA driver for a multiplier here. + */ + factor = 1; + + /* + * If the device needs MRs to perform RDMA READ or WRITE operations, + * we'll need two additional MRs for the registrations and the + * invalidation. + */ + if (rdma_protocol_iwarp(dev, attr->port_num) || dev->attrs.max_sgl_rd) + factor += 2; /* inv + reg */ + + return factor * attr->cap.max_rdma_ctxs; +} + +int smbdirect_connection_create_qp(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct ib_qp_init_attr qp_attr; + struct ib_qp_cap qp_cap; + u32 rdma_send_wr; + u32 max_send_wr; + int ret; + + /* + * Note that {rdma,ib}_create_qp() will call + * rdma_rw_init_qp() if max_rdma_ctxs is not 0. + * It will adjust max_send_wr to the required + * number of additional WRs for the RDMA RW operations. + * It will cap max_send_wr to the device limit. + * + * We use allocate sp->responder_resources * 2 MRs + * and each MR needs WRs for REG and INV, so + * we use '* 4'. + * + * +1 for ib_drain_qp() + */ + memset(&qp_cap, 0, sizeof(qp_cap)); + qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1; + qp_cap.max_recv_wr = sp->recv_credit_max + 1; + qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; + qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; + qp_cap.max_inline_data = 0; + qp_cap.max_rdma_ctxs = sc->rw_io.credits.max; + + /* + * Find out the number of max_send_wr + * after rdma_rw_init_qp() adjusted it. + * + * We only do it on a temporary variable, + * as rdma_create_qp() will trigger + * rdma_rw_init_qp() again. + */ + memset(&qp_attr, 0, sizeof(qp_attr)); + qp_attr.cap = qp_cap; + qp_attr.port_num = sc->rdma.cm_id->port_num; + rdma_send_wr = smbdirect_rdma_rw_send_wrs(sc->ib.dev, &qp_attr); + max_send_wr = qp_cap.max_send_wr + rdma_send_wr; + + if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe || + qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) { + pr_err("Possible CQE overrun: max_send_wr %d\n", + qp_cap.max_send_wr); + pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); + pr_err("consider lowering send_credit_target = %d\n", + sp->send_credit_target); + return -EINVAL; + } + + if (qp_cap.max_rdma_ctxs && + (max_send_wr >= sc->ib.dev->attrs.max_cqe || + max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) { + pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n", + rdma_send_wr, qp_cap.max_send_wr, max_send_wr); + pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); + pr_err("consider lowering send_credit_target = %d, max_rdma_ctxs = %d\n", + sp->send_credit_target, qp_cap.max_rdma_ctxs); + return -EINVAL; + } + + if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe || + qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) { + pr_err("Possible CQE overrun: max_recv_wr %d\n", + qp_cap.max_recv_wr); + pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); + pr_err("consider lowering receive_credit_max = %d\n", + sp->recv_credit_max); + return -EINVAL; + } + + if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge || + qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) { + pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_send_sge, + sc->ib.dev->attrs.max_recv_sge); + return -EINVAL; + } + + sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); + if (IS_ERR(sc->ib.pd)) { + pr_err("Can't create RDMA PD: %1pe\n", sc->ib.pd); + ret = PTR_ERR(sc->ib.pd); + sc->ib.pd = NULL; + return ret; + } + + sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc, + max_send_wr, + sc->ib.poll_ctx); + if (IS_ERR(sc->ib.send_cq)) { + pr_err("Can't create RDMA send CQ: %1pe\n", sc->ib.send_cq); + ret = PTR_ERR(sc->ib.send_cq); + sc->ib.send_cq = NULL; + goto err; + } + + sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc, + qp_cap.max_recv_wr, + sc->ib.poll_ctx); + if (IS_ERR(sc->ib.recv_cq)) { + pr_err("Can't create RDMA recv CQ: %1pe\n", sc->ib.recv_cq); + ret = PTR_ERR(sc->ib.recv_cq); + sc->ib.recv_cq = NULL; + goto err; + } + + /* + * We reset completely here! + * As the above use was just temporary + * to calc max_send_wr and rdma_send_wr. + * + * rdma_create_qp() will trigger rdma_rw_init_qp() + * again if max_rdma_ctxs is not 0. + */ + memset(&qp_attr, 0, sizeof(qp_attr)); + qp_attr.event_handler = smbdirect_connection_qp_event_handler; + qp_attr.qp_context = sc; + qp_attr.cap = qp_cap; + qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; + qp_attr.qp_type = IB_QPT_RC; + qp_attr.send_cq = sc->ib.send_cq; + qp_attr.recv_cq = sc->ib.recv_cq; + qp_attr.port_num = ~0; + + ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); + if (ret) { + pr_err("Can't create RDMA QP: %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto err; + } + sc->ib.qp = sc->rdma.cm_id->qp; + + return 0; +err: + smbdirect_connection_destroy_qp(sc); + return ret; +} + +void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc) +{ + if (sc->ib.qp) { + ib_drain_qp(sc->ib.qp); + sc->ib.qp = NULL; + rdma_destroy_qp(sc->rdma.cm_id); + } + if (sc->ib.recv_cq) { + ib_destroy_cq(sc->ib.recv_cq); + sc->ib.recv_cq = NULL; + } + if (sc->ib.send_cq) { + ib_destroy_cq(sc->ib.send_cq); + sc->ib.send_cq = NULL; + } + if (sc->ib.pd) { + ib_dealloc_pd(sc->ib.pd); + sc->ib.pd = NULL; + } +} + +int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + char name[80]; + size_t i; + + /* + * We use sizeof(struct smbdirect_negotiate_resp) for the + * payload size as it is larger as + * sizeof(struct smbdirect_data_transfer). + * + * This will fit client and server usage for now. + */ + snprintf(name, sizeof(name), "smbdirect_send_io_cache_%p", sc); + struct kmem_cache_args send_io_args = { + .align = __alignof__(struct smbdirect_send_io), + }; + sc->send_io.mem.cache = kmem_cache_create(name, + sizeof(struct smbdirect_send_io) + + sizeof(struct smbdirect_negotiate_resp), + &send_io_args, + SLAB_HWCACHE_ALIGN); + if (!sc->send_io.mem.cache) + goto err; + + sc->send_io.mem.pool = mempool_create_slab_pool(sp->send_credit_target, + sc->send_io.mem.cache); + if (!sc->send_io.mem.pool) + goto err; + + /* + * A payload size of sp->max_recv_size should fit + * any message. + * + * For smbdirect_data_transfer messages the whole + * buffer might be exposed to userspace + * (currently on the client side...) + * The documentation says data_offset = 0 would be + * strange but valid. + */ + snprintf(name, sizeof(name), "smbdirect_recv_io_cache_%p", sc); + struct kmem_cache_args recv_io_args = { + .align = __alignof__(struct smbdirect_recv_io), + .useroffset = sizeof(struct smbdirect_recv_io), + .usersize = sp->max_recv_size, + }; + sc->recv_io.mem.cache = kmem_cache_create(name, + sizeof(struct smbdirect_recv_io) + + sp->max_recv_size, + &recv_io_args, + SLAB_HWCACHE_ALIGN); + if (!sc->recv_io.mem.cache) + goto err; + + sc->recv_io.mem.pool = mempool_create_slab_pool(sp->recv_credit_max, + sc->recv_io.mem.cache); + if (!sc->recv_io.mem.pool) + goto err; + + for (i = 0; i < sp->recv_credit_max; i++) { + struct smbdirect_recv_io *recv_io; + + recv_io = mempool_alloc(sc->recv_io.mem.pool, + sc->recv_io.mem.gfp_mask); + if (!recv_io) + goto err; + recv_io->socket = sc; + recv_io->sge.length = 0; + list_add_tail(&recv_io->list, &sc->recv_io.free.list); + } + + return 0; +err: + smbdirect_connection_destroy_mem_pools(sc); + return -ENOMEM; +} + +void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc) +{ + struct smbdirect_recv_io *recv_io, *next_io; + + list_for_each_entry_safe(recv_io, next_io, &sc->recv_io.free.list, list) { + list_del(&recv_io->list); + mempool_free(recv_io, sc->recv_io.mem.pool); + } + + /* + * Note mempool_destroy() and kmem_cache_destroy() + * work fine with a NULL pointer + */ + + mempool_destroy(sc->recv_io.mem.pool); + sc->recv_io.mem.pool = NULL; + + kmem_cache_destroy(sc->recv_io.mem.cache); + sc->recv_io.mem.cache = NULL; + + mempool_destroy(sc->send_io.mem.pool); + sc->send_io.mem.pool = NULL; + + kmem_cache_destroy(sc->send_io.mem.cache); + sc->send_io.mem.cache = NULL; +} + +struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc) +{ + struct smbdirect_send_io *msg; + + msg = mempool_alloc(sc->send_io.mem.pool, sc->send_io.mem.gfp_mask); + if (!msg) + return ERR_PTR(-ENOMEM); + msg->socket = sc; + INIT_LIST_HEAD(&msg->sibling_list); + msg->num_sge = 0; + + return msg; +} + +void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + size_t i; + + /* + * The list needs to be empty! + * The caller should take care of it. + */ + WARN_ON_ONCE(!list_empty(&msg->sibling_list)); + + /* + * Note we call ib_dma_unmap_page(), even if some sges are mapped using + * ib_dma_map_single(). + * + * The difference between _single() and _page() only matters for the + * ib_dma_map_*() case. + * + * For the ib_dma_unmap_*() case it does not matter as both take the + * dma_addr_t and dma_unmap_single_attrs() is just an alias to + * dma_unmap_page_attrs(). + */ + for (i = 0; i < msg->num_sge; i++) + ib_dma_unmap_page(sc->ib.dev, + msg->sge[i].addr, + msg->sge[i].length, + DMA_TO_DEVICE); + + mempool_free(msg, sc->send_io.mem.pool); +} + +struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc) +{ + struct smbdirect_recv_io *msg = NULL; + unsigned long flags; + + spin_lock_irqsave(&sc->recv_io.free.lock, flags); + if (likely(!sc->first_error)) + msg = list_first_entry_or_null(&sc->recv_io.free.list, + struct smbdirect_recv_io, + list); + if (likely(msg)) { + list_del(&msg->list); + sc->statistics.get_receive_buffer++; + } + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); + + return msg; +} + +void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + unsigned long flags; + + if (likely(msg->sge.length != 0)) { + ib_dma_unmap_single(sc->ib.dev, + msg->sge.addr, + msg->sge.length, + DMA_FROM_DEVICE); + msg->sge.length = 0; + } + + spin_lock_irqsave(&sc->recv_io.free.lock, flags); + list_add_tail(&msg->list, &sc->recv_io.free.list); + sc->statistics.put_receive_buffer++; + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); + + queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); +} + +void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc, + struct smbdirect_recv_io *msg, + u32 data_length) +{ + unsigned long flags; + + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + list_add_tail(&msg->list, &sc->recv_io.reassembly.list); + sc->recv_io.reassembly.queue_length++; + /* + * Make sure reassembly_data_length is updated after list and + * reassembly_queue_length are updated. On the dequeue side + * reassembly_data_length is checked without a lock to determine + * if reassembly_queue_length and list is up to date + */ + virt_wmb(); + sc->recv_io.reassembly.data_length += data_length; + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + sc->statistics.enqueue_reassembly_queue++; +} + +struct smbdirect_recv_io * +smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc) +{ + struct smbdirect_recv_io *msg; + + msg = list_first_entry_or_null(&sc->recv_io.reassembly.list, + struct smbdirect_recv_io, + list); + + return msg; +} + +void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc, + u8 peer_initiator_depth, + u8 peer_responder_resources, + const struct rdma_conn_param *param) +{ + struct smbdirect_socket_parameters *sp = &sc->parameters; + + if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num) && + param->private_data_len == 8) { + /* + * Legacy clients with only iWarp MPA v1 support + * need a private blob in order to negotiate + * the IRD/ORD values. + */ + const __be32 *ird_ord_hdr = param->private_data; + u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); + u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); + + /* + * cifs.ko sends the legacy IRD/ORD negotiation + * event if iWarp MPA v2 was used. + * + * Here we check that the values match and only + * mark the client as legacy if they don't match. + */ + if ((u32)param->initiator_depth != ird32 || + (u32)param->responder_resources != ord32) { + /* + * There are broken clients (old cifs.ko) + * using little endian and also + * struct rdma_conn_param only uses u8 + * for initiator_depth and responder_resources, + * so we truncate the value to U8_MAX. + * + * smb_direct_accept_client() will then + * do the real negotiation in order to + * select the minimum between client and + * server. + */ + ird32 = min_t(u32, ird32, U8_MAX); + ord32 = min_t(u32, ord32, U8_MAX); + + sc->rdma.legacy_iwarp = true; + peer_initiator_depth = (u8)ird32; + peer_responder_resources = (u8)ord32; + } + } + + /* + * negotiate the value by using the minimum + * between client and server if the client provided + * non 0 values. + */ + if (peer_initiator_depth != 0) + sp->initiator_depth = min_t(u8, sp->initiator_depth, + peer_initiator_depth); + if (peer_responder_resources != 0) + sp->responder_resources = min_t(u8, sp->responder_resources, + peer_responder_resources); +} + +bool smbdirect_connection_is_connected(struct smbdirect_socket *sc) +{ + if (unlikely(!sc || sc->first_error || sc->status != SMBDIRECT_SOCKET_CONNECTED)) + return false; + return true; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_is_connected); + +int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + union { + struct sockaddr sa; + struct sockaddr_storage ss; + } src_addr, dst_addr; + const struct sockaddr *src = NULL; + const struct sockaddr *dst = NULL; + char _devname[IB_DEVICE_NAME_MAX] = { 0, }; + const char *devname = NULL; + int ret; + + if (sc->rdma.cm_id) { + src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; + if (src_addr.sa.sa_family != AF_UNSPEC) + src = &src_addr.sa; + dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr; + if (dst_addr.sa.sa_family != AF_UNSPEC) + dst = &dst_addr.sa; + + if (sc->ib.dev) { + memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX); + devname = _devname; + } + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "waiting for connection: device: %.*s local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, devname, src, dst); + + ret = wait_event_interruptible_timeout(sc->status_wait, + sc->status == SMBDIRECT_SOCKET_CONNECTED || + sc->first_error, + msecs_to_jiffies(sp->negotiate_timeout_msec)); + if (sc->rdma.cm_id) { + /* + * Maybe src and dev are updated in the meantime. + */ + src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; + if (src_addr.sa.sa_family != AF_UNSPEC) + src = &src_addr.sa; + dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr; + if (dst_addr.sa.sa_family != AF_UNSPEC) + dst = &dst_addr.sa; + + if (sc->ib.dev) { + memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX); + devname = _devname; + } + } + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) + smbdirect_socket_schedule_cleanup(sc, ret); + if (sc->first_error) { + int lvl = SMBDIRECT_LOG_ERR; + + ret = sc->first_error; + if (ret == -ENODEV) + lvl = SMBDIRECT_LOG_INFO; + + smbdirect_log_rdma_event(sc, lvl, + "connection failed %1pe device: %.*s local: %pISpsfc remote: %pISpsfc\n", + SMBDIRECT_DEBUG_ERR_PTR(ret), + IB_DEVICE_NAME_MAX, devname, src, dst); + return ret; + } + + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_wait_for_connected); + +void smbdirect_connection_idle_timer_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, idle.timer_work.work); + const struct smbdirect_socket_parameters *sp = &sc->parameters; + + if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_ERR, + "%s => timeout sc->idle.keepalive=%s\n", + smbdirect_socket_status_string(sc->status), + sc->idle.keepalive == SMBDIRECT_KEEPALIVE_SENT ? + "SENT" : "PENDING"); + smbdirect_socket_schedule_cleanup(sc, -ETIMEDOUT); + return; + } + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return; + + /* + * Now use the keepalive timeout (instead of keepalive interval) + * in order to wait for a response + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_timeout_msec)); + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "schedule send of empty idle message\n"); + queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); +} + +u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc) +{ + int missing; + int available; + int new_credits; + + if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) + return 0; + + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = min3((int)U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); + return 0; + } + + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + + /* + * Remember we granted the credits + */ + atomic_add(new_credits, &sc->recv_io.credits.count); + return new_credits; +} + +static bool smbdirect_connection_request_keep_alive(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + + if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; + /* + * Now use the keepalive timeout (instead of keepalive interval) + * in order to wait for a response + */ + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_timeout_msec)); + return true; + } + + return false; +} + +int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc, + struct ib_send_wr *wr) +{ + int ret; + + if (unlikely(sc->first_error)) + return sc->first_error; + + atomic_inc(&sc->send_io.pending.count); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + atomic_dec(&sc->send_io.pending.count); + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "ib_post_send() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + } + + return ret; +} + +static void smbdirect_connection_send_batch_init(struct smbdirect_send_batch *batch, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + INIT_LIST_HEAD(&batch->msg_list); + batch->wr_cnt = 0; + batch->need_invalidate_rkey = need_invalidate_rkey; + batch->remote_key = remote_key; + batch->credit = 0; +} + +int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last) +{ + struct smbdirect_send_io *first, *last; + int ret = 0; + + if (list_empty(&batch->msg_list)) + goto release_credit; + + first = list_first_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + + if (batch->need_invalidate_rkey) { + first->wr.opcode = IB_WR_SEND_WITH_INV; + first->wr.ex.invalidate_rkey = batch->remote_key; + batch->need_invalidate_rkey = false; + batch->remote_key = 0; + } + + last->wr.send_flags = IB_SEND_SIGNALED; + last->wr.wr_cqe = &last->cqe; + + /* + * Remove last from send_ctx->msg_list + * and splice the rest of send_ctx->msg_list + * to last->sibling_list. + * + * send_ctx->msg_list is a valid empty list + * at the end. + */ + list_del_init(&last->sibling_list); + list_splice_tail_init(&batch->msg_list, &last->sibling_list); + batch->wr_cnt = 0; + + ret = smbdirect_connection_post_send_wr(sc, &first->wr); + if (ret) { + struct smbdirect_send_io *sibling, *next; + + list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbdirect_connection_free_send_io(sibling); + } + smbdirect_connection_free_send_io(last); + } + +release_credit: + if (is_last && !ret && batch->credit) { + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + + return ret; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_batch_flush); + +struct smbdirect_send_batch * +smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + struct smbdirect_send_batch *batch = (struct smbdirect_send_batch *)storage; + + memset(storage, 0, sizeof(*storage)); + BUILD_BUG_ON(sizeof(*batch) > sizeof(*storage)); + + smbdirect_connection_send_batch_init(batch, + need_invalidate_rkey, + remote_key); + + return batch; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_init_send_batch_storage); + +static int smbdirect_connection_wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + int ret; + + if (batch->credit) + return 0; + + ret = smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + batch->credit = 1; + return 0; +} + +static int smbdirect_connection_wait_for_send_lcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && atomic_read(&sc->send_io.lcredits.count) <= 1) { + int ret; + + ret = smbdirect_connection_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->send_io.lcredits.wait_queue, + &sc->send_io.lcredits.count, + 1); +} + +static int smbdirect_connection_wait_for_send_credits(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { + int ret; + + ret = smbdirect_connection_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->send_io.credits.wait_queue, + &sc->send_io.credits.count, + 1); +} + +static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc); + +static int smbdirect_connection_post_send_io(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *msg) +{ + int i; + + for (i = 0; i < msg->num_sge; i++) + ib_dma_sync_single_for_device(sc->ib.dev, + msg->sge[i].addr, msg->sge[i].length, + DMA_TO_DEVICE); + + msg->cqe.done = smbdirect_connection_send_io_done; + msg->wr.wr_cqe = &msg->cqe; + msg->wr.opcode = IB_WR_SEND; + msg->wr.sg_list = &msg->sge[0]; + msg->wr.num_sge = msg->num_sge; + msg->wr.next = NULL; + + if (batch) { + msg->wr.send_flags = 0; + if (!list_empty(&batch->msg_list)) { + struct smbdirect_send_io *last; + + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last->wr.next = &msg->wr; + } + list_add_tail(&msg->sibling_list, &batch->msg_list); + batch->wr_cnt++; + return 0; + } + + msg->wr.send_flags = IB_SEND_SIGNALED; + return smbdirect_connection_post_send_wr(sc, &msg->wr); +} + +int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct iov_iter *iter, + unsigned int flags, + u32 remaining_data_length) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_send_batch _batch; + struct smbdirect_send_io *msg; + struct smbdirect_data_transfer *packet; + size_t header_length; + u16 new_credits = 0; + u32 data_length = 0; + int ret; + + if (WARN_ON_ONCE(flags)) + return -EINVAL; /* no flags support for now */ + + if (iter) { + if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) + return -EINVAL; /* It's a bug in upper layer to get there */ + + header_length = sizeof(struct smbdirect_data_transfer); + if (WARN_ON_ONCE(remaining_data_length == 0 || + iov_iter_count(iter) > remaining_data_length)) + return -EINVAL; + } else { + /* If this is a packet without payload, don't send padding */ + header_length = offsetof(struct smbdirect_data_transfer, padding); + if (WARN_ON_ONCE(remaining_data_length)) + return -EINVAL; + } + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + if (!batch) { + smbdirect_connection_send_batch_init(&_batch, false, 0); + batch = &_batch; + } + + ret = smbdirect_connection_wait_for_send_bcredit(sc, batch); + if (ret) + goto bcredit_failed; + + ret = smbdirect_connection_wait_for_send_lcredit(sc, batch); + if (ret) + goto lcredit_failed; + + ret = smbdirect_connection_wait_for_send_credits(sc, batch); + if (ret) + goto credit_failed; + + new_credits = smbdirect_connection_grant_recv_credits(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + /* + * queue the refill work in order to + * get some new recv credits we can grant to + * the peer. + */ + queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); + + /* + * wait until either the refill work or the peer + * granted new credits + */ + ret = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + ret = -ENOTCONN; + if (ret < 0) + goto credit_failed; + + new_credits = smbdirect_connection_grant_recv_credits(sc); + } + + msg = smbdirect_connection_alloc_send_io(sc); + if (IS_ERR(msg)) { + ret = PTR_ERR(msg); + goto alloc_failed; + } + + /* Map the packet to DMA */ + msg->sge[0].addr = ib_dma_map_single(sc->ib.dev, + msg->packet, + header_length, + DMA_TO_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, msg->sge[0].addr); + if (ret) + goto err; + + msg->sge[0].length = header_length; + msg->sge[0].lkey = sc->ib.pd->local_dma_lkey; + msg->num_sge = 1; + + if (iter) { + struct smbdirect_map_sges extract = { + .num_sge = msg->num_sge, + .max_sge = ARRAY_SIZE(msg->sge), + .sge = msg->sge, + .device = sc->ib.dev, + .local_dma_lkey = sc->ib.pd->local_dma_lkey, + .direction = DMA_TO_DEVICE, + }; + size_t payload_len = umin(iov_iter_count(iter), + sp->max_send_size - sizeof(*packet)); + + ret = smbdirect_map_sges_from_iter(iter, payload_len, &extract); + if (ret < 0) + goto err; + data_length = ret; + remaining_data_length -= data_length; + msg->num_sge = extract.num_sge; + } + + /* Fill in the packet header */ + packet = (struct smbdirect_data_transfer *)msg->packet; + packet->credits_requested = cpu_to_le16(sp->send_credit_target); + packet->credits_granted = cpu_to_le16(new_credits); + + packet->flags = 0; + if (smbdirect_connection_request_keep_alive(sc)) + packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); + + packet->reserved = 0; + if (!data_length) + packet->data_offset = 0; + else + packet->data_offset = cpu_to_le32(24); + packet->data_length = cpu_to_le32(data_length); + packet->remaining_data_length = cpu_to_le32(remaining_data_length); + packet->padding = 0; + + smbdirect_log_outgoing(sc, SMBDIRECT_LOG_INFO, + "DataOut: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n", + "CreditsRequested", + le16_to_cpu(packet->credits_requested), + "CreditsGranted", + le16_to_cpu(packet->credits_granted), + "Flags", + le16_to_cpu(packet->flags), + "RemainingDataLength", + le32_to_cpu(packet->remaining_data_length), + "DataOffset", + le32_to_cpu(packet->data_offset), + "DataLength", + le32_to_cpu(packet->data_length)); + + ret = smbdirect_connection_post_send_io(sc, batch, msg); + if (ret) + goto err; + + /* + * From here msg is moved to send_ctx + * and we should not free it explicitly. + */ + + if (batch == &_batch) { + ret = smbdirect_connection_send_batch_flush(sc, batch, true); + if (ret) + goto flush_failed; + } + + return data_length; +err: + smbdirect_connection_free_send_io(msg); +flush_failed: +alloc_failed: + atomic_inc(&sc->send_io.credits.count); +credit_failed: + atomic_inc(&sc->send_io.lcredits.count); +lcredit_failed: + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; +bcredit_failed: + return ret; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_single_iter); + +int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc) +{ + /* + * As an optimization, we don't wait for individual I/O to finish + * before sending the next one. + * Send them all and wait for pending send count to get to 0 + * that means all the I/Os have been out and we are good to return + */ + + wait_event(sc->send_io.pending.zero_wait_queue, + atomic_read(&sc->send_io.pending.count) == 0 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_wait_zero_pending); + +int smbdirect_connection_send_iter(struct smbdirect_socket *sc, + struct iov_iter *iter, + unsigned int flags, + bool need_invalidate, + unsigned int remote_key) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_send_batch batch; + int total_count = iov_iter_count(iter); + int ret; + int error = 0; + __be32 hdr; + + if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags)) + return -EINVAL; /* no flags support for now */ + + if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) + return -EINVAL; /* It's a bug in upper layer to get there */ + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_write(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + /* + * For now we expect the iter to have the full + * message, including a 4 byte length header. + */ + if (iov_iter_count(iter) <= 4) + return -EINVAL; + if (!copy_from_iter_full(&hdr, sizeof(hdr), iter)) + return -EFAULT; + if (iov_iter_count(iter) != be32_to_cpu(hdr)) + return -EINVAL; + + /* + * The size must fit into the negotiated + * fragmented send size. + */ + if (iov_iter_count(iter) > sp->max_fragmented_send_size) + return -EMSGSIZE; + + smbdirect_log_write(sc, SMBDIRECT_LOG_INFO, + "Sending (RDMA): length=%zu\n", + iov_iter_count(iter)); + + smbdirect_connection_send_batch_init(&batch, need_invalidate, remote_key); + while (iov_iter_count(iter)) { + ret = smbdirect_connection_send_single_iter(sc, + &batch, + iter, + flags, + iov_iter_count(iter)); + if (unlikely(ret < 0)) { + error = ret; + break; + } + } + + ret = smbdirect_connection_send_batch_flush(sc, &batch, true); + if (unlikely(ret && !error)) + error = ret; + + /* + * As an optimization, we don't wait for individual I/O to finish + * before sending the next one. + * Send them all and wait for pending send count to get to 0 + * that means all the I/Os have been out and we are good to return + */ + + ret = smbdirect_connection_send_wait_zero_pending(sc); + if (unlikely(ret && !error)) + error = ret; + + if (unlikely(error)) + return error; + + return total_count; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_send_iter); + +static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_send_io *msg = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = msg->socket; + struct smbdirect_send_io *sibling, *next; + int lcredits = 0; + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + if (unlikely(!(msg->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * Free possible siblings and then the main send_io + */ + list_for_each_entry_safe(sibling, next, &msg->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbdirect_connection_free_send_io(sibling); + lcredits += 1; + } + /* Note this frees wc->wr_cqe, but not wc */ + smbdirect_connection_free_send_io(msg); + lcredits += 1; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { +skip_free: + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + atomic_add(lcredits, &sc->send_io.lcredits.count); + wake_up(&sc->send_io.lcredits.wait_queue); + + if (atomic_dec_and_test(&sc->send_io.pending.count)) + wake_up(&sc->send_io.pending.zero_wait_queue); +} + +static void smbdirect_connection_send_immediate_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, idle.immediate_work); + int ret; + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return; + + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "send an empty message\n"); + sc->statistics.send_empty++; + ret = smbdirect_connection_send_single_iter(sc, NULL, NULL, 0, 0); + if (ret < 0) { + smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_send_single_iter ret=%1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + } +} + +int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct ib_recv_wr recv_wr = { + .wr_cqe = &msg->cqe, + .sg_list = &msg->sge, + .num_sge = 1, + }; + int ret; + + if (unlikely(sc->first_error)) + return sc->first_error; + + msg->sge.addr = ib_dma_map_single(sc->ib.dev, + msg->packet, + sp->max_recv_size, + DMA_FROM_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, msg->sge.addr); + if (ret) + return ret; + + msg->sge.length = sp->max_recv_size; + msg->sge.lkey = sc->ib.pd->local_dma_lkey; + + ret = ib_post_recv(sc->ib.qp, &recv_wr, NULL); + if (ret) { + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "ib_post_recv failed ret=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + ib_dma_unmap_single(sc->ib.dev, + msg->sge.addr, + msg->sge.length, + DMA_FROM_DEVICE); + msg->sge.length = 0; + smbdirect_socket_schedule_cleanup(sc, ret); + } + + return ret; +} + +void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_recv_io *recv_io = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = recv_io->socket; + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_data_transfer *data_transfer; + int current_recv_credits; + u16 old_recv_credit_target; + u16 credits_requested; + u16 credits_granted; + u16 flags; + u32 data_offset; + u32 data_length; + u32 remaining_data_length; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + goto error; + } + + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, + "recv_io=0x%p type=%d wc status=%s wc opcode %d byte_len=%d pkey_index=%u\n", + recv_io, sc->recv_io.expected, + ib_wc_status_msg(wc->status), wc->opcode, + wc->byte_len, wc->pkey_index); + + /* + * Reset timer to the keepalive interval in + * order to trigger our next keepalive message. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_interval_msec)); + + ib_dma_sync_single_for_cpu(sc->ib.dev, + recv_io->sge.addr, + recv_io->sge.length, + DMA_FROM_DEVICE); + + if (unlikely(wc->byte_len < + offsetof(struct smbdirect_data_transfer, padding))) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "wc->byte_len=%u < %zu\n", + wc->byte_len, + offsetof(struct smbdirect_data_transfer, padding)); + goto error; + } + + data_transfer = (struct smbdirect_data_transfer *)recv_io->packet; + credits_requested = le16_to_cpu(data_transfer->credits_requested); + credits_granted = le16_to_cpu(data_transfer->credits_granted); + flags = le16_to_cpu(data_transfer->flags); + remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); + data_offset = le32_to_cpu(data_transfer->data_offset); + data_length = le32_to_cpu(data_transfer->data_length); + + smbdirect_log_incoming(sc, SMBDIRECT_LOG_INFO, + "DataIn: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n", + "CreditsRequested", + credits_requested, + "CreditsGranted", + credits_granted, + "Flags", + flags, + "RemainingDataLength", + remaining_data_length, + "DataOffset", + data_offset, + "DataLength", + data_length); + + if (unlikely(credits_requested == 0)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_requested == 0\n"); + goto error; + } + + if (unlikely(data_offset % 8 != 0)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: data_offset=%u (0x%x) not aligned to 8\n", + data_offset, data_offset); + goto error; + } + + if (unlikely(wc->byte_len < data_offset || + (u64)wc->byte_len < (u64)data_offset + data_length)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "wc->byte_len=%u < date_offset=%u + data_length=%u\n", + wc->byte_len, data_offset, data_length); + goto error; + } + + if (unlikely(remaining_data_length > sp->max_fragmented_recv_size || + data_length > sp->max_fragmented_recv_size || + (u64)remaining_data_length + (u64)data_length > (u64)sp->max_fragmented_recv_size)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "remaining_data_length=%u + data_length=%u > max_fragmented=%u\n", + remaining_data_length, data_length, sp->max_fragmented_recv_size); + goto error; + } + + if (data_length) { + if (sc->recv_io.reassembly.full_packet_received) + recv_io->first_segment = true; + + if (remaining_data_length) + sc->recv_io.reassembly.full_packet_received = false; + else + sc->recv_io.reassembly.full_packet_received = true; + } + + atomic_dec(&sc->recv_io.posted.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); + + /* + * We take the value from the peer, which is checked to be higher than 0, + * but we limit it to the max value we support in order to have + * the main logic simpler. + */ + old_recv_credit_target = sc->recv_io.credits.target; + sc->recv_io.credits.target = credits_requested; + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, + sp->recv_credit_max); + if (credits_granted) { + atomic_add(credits_granted, &sc->send_io.credits.count); + /* + * We have new send credits granted from remote peer + * If any sender is waiting for credits, unblock it + */ + wake_up(&sc->send_io.credits.wait_queue); + } + + /* Send an immediate response right away if requested */ + if (flags & SMBDIRECT_FLAG_RESPONSE_REQUESTED) { + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "schedule send of immediate response\n"); + queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); + } + + /* + * If this is a packet with data playload place the data in + * reassembly queue and wake up the reading thread + */ + if (data_length) { + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) + queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); + + smbdirect_connection_reassembly_append_recv_io(sc, recv_io, data_length); + wake_up(&sc->recv_io.reassembly.wait_queue); + } else + smbdirect_connection_put_recv_io(recv_io); + + return; + +error: + /* + * Make sure smbdirect_connection_put_recv_io() does not + * start recv_io.posted.refill_work. + */ + disable_work(&sc->recv_io.posted.refill_work); + smbdirect_connection_put_recv_io(recv_io); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); +} + +int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc) +{ + int missing; + int posted = 0; + + if (unlikely(sc->first_error)) + return sc->first_error; + + /* + * Find out how much smbdirect_recv_io buffers we should post. + * + * Note that sc->recv_io.credits.target is the value + * from the peer and it can in theory change over time, + * but it is forced to be at least 1 and at max + * sp->recv_credit_max. + * + * So it can happen that missing will be lower than 0, + * which means the peer has recently lowered its desired + * target, while be already granted a higher number of credits. + * + * Note 'posted' is the number of smbdirect_recv_io buffers + * posted within this function, while sc->recv_io.posted.count + * is the overall value of posted smbdirect_recv_io buffers. + * + * We try to post as much buffers as missing, but + * this is limited if a lot of smbdirect_recv_io buffers + * are still in the sc->recv_io.reassembly.list instead of + * the sc->recv_io.free.list. + * + */ + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.posted.count); + while (posted < missing) { + struct smbdirect_recv_io *recv_io; + int ret; + + /* + * It's ok if smbdirect_connection_get_recv_io() + * returns NULL, it means smbdirect_recv_io structures + * are still be in the reassembly.list. + */ + recv_io = smbdirect_connection_get_recv_io(sc); + if (!recv_io) + break; + + recv_io->first_segment = false; + + ret = smbdirect_connection_post_recv_io(recv_io); + if (ret) { + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_recv_io failed rc=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_connection_put_recv_io(recv_io); + return ret; + } + + atomic_inc(&sc->recv_io.posted.count); + posted += 1; + } + + /* If nothing was posted we're done */ + if (posted == 0) + return 0; + + atomic_add(posted, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + + /* + * If we posted at least one smbdirect_recv_io buffer, + * we need to inform the peer about it and grant + * additional credits. + * + * However there is one case where we don't want to + * do that. + * + * If only a single credit was missing before + * reaching the requested target, we should not + * post an immediate send, as that would cause + * endless ping pong once a keep alive exchange + * is started. + * + * However if sc->recv_io.credits.target is only 1, + * the peer has no credit left and we need to + * grant the credit anyway. + */ + if (missing == 1 && sc->recv_io.credits.target != 1) + return 0; + + return posted; +} + +static void smbdirect_connection_recv_io_refill_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); + int posted; + + posted = smbdirect_connection_recv_io_refill(sc); + if (unlikely(posted < 0)) { + smbdirect_socket_schedule_cleanup(sc, posted); + return; + } + if (posted > 0) { + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "schedule send of an empty message\n"); + queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); + } +} + +int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, + struct msghdr *msg, + unsigned int flags) +{ + struct smbdirect_recv_io *response; + struct smbdirect_data_transfer *data_transfer; + size_t size = iov_iter_count(&msg->msg_iter); + int to_copy, to_read, data_read, offset; + u32 data_length, remaining_data_length, data_offset; + int ret; + + if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags)) + return -EINVAL; /* no flags support for now */ + + if (WARN_ON_ONCE(iov_iter_rw(&msg->msg_iter) != ITER_DEST)) + return -EINVAL; /* It's a bug in upper layer to get there */ + +again: + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + /* + * No need to hold the reassembly queue lock all the time as we are + * the only one reading from the front of the queue. The transport + * may add more entries to the back of the queue at the same time + */ + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "size=%zd sc->recv_io.reassembly.data_length=%d\n", + size, sc->recv_io.reassembly.data_length); + if (sc->recv_io.reassembly.data_length >= size) { + int queue_length; + int queue_removed = 0; + unsigned long flags; + + /* + * Need to make sure reassembly_data_length is read before + * reading reassembly_queue_length and calling + * smbdirect_connection_reassembly_first_recv_io. This call is lock free + * as we never read at the end of the queue which are being + * updated in SOFTIRQ as more data is received + */ + virt_rmb(); + queue_length = sc->recv_io.reassembly.queue_length; + data_read = 0; + to_read = size; + offset = sc->recv_io.reassembly.first_entry_offset; + while (data_read < size) { + response = smbdirect_connection_reassembly_first_recv_io(sc); + data_transfer = (void *)response->packet; + data_length = le32_to_cpu(data_transfer->data_length); + remaining_data_length = + le32_to_cpu( + data_transfer->remaining_data_length); + data_offset = le32_to_cpu(data_transfer->data_offset); + + /* + * The upper layer expects RFC1002 length at the + * beginning of the payload. Return it to indicate + * the total length of the packet. This minimize the + * change to upper layer packet processing logic. This + * will be eventually remove when an intermediate + * transport layer is added + */ + if (response->first_segment && size == 4) { + unsigned int rfc1002_len = + data_length + remaining_data_length; + __be32 rfc1002_hdr = cpu_to_be32(rfc1002_len); + + if (copy_to_iter(&rfc1002_hdr, sizeof(rfc1002_hdr), + &msg->msg_iter) != sizeof(rfc1002_hdr)) + return -EFAULT; + data_read = 4; + response->first_segment = false; + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "returning rfc1002 length %d\n", + rfc1002_len); + goto read_rfc1002_done; + } + + to_copy = min_t(int, data_length - offset, to_read); + if (copy_to_iter((u8 *)data_transfer + data_offset + offset, + to_copy, &msg->msg_iter) != to_copy) + return -EFAULT; + + /* move on to the next buffer? */ + if (to_copy == data_length - offset) { + queue_length--; + /* + * No need to lock if we are not at the + * end of the queue + */ + if (queue_length) + list_del(&response->list); + else { + spin_lock_irqsave( + &sc->recv_io.reassembly.lock, flags); + list_del(&response->list); + spin_unlock_irqrestore( + &sc->recv_io.reassembly.lock, flags); + } + queue_removed++; + sc->statistics.dequeue_reassembly_queue++; + smbdirect_connection_put_recv_io(response); + offset = 0; + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "smbdirect_connection_put_recv_io offset=0\n"); + } else + offset += to_copy; + + to_read -= to_copy; + data_read += to_copy; + + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "memcpy %d bytes len-ofs=%u => todo=%u done=%u ofs=%u\n", + to_copy, data_length - offset, + to_read, data_read, offset); + } + + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + sc->recv_io.reassembly.data_length -= data_read; + sc->recv_io.reassembly.queue_length -= queue_removed; + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + + sc->recv_io.reassembly.first_entry_offset = offset; + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "returning data_read=%d reassembly_length=%d first_ofs=%u\n", + data_read, sc->recv_io.reassembly.data_length, + sc->recv_io.reassembly.first_entry_offset); +read_rfc1002_done: + return data_read; + } + + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "wait_event on more data\n"); + ret = wait_event_interruptible(sc->recv_io.reassembly.wait_queue, + sc->recv_io.reassembly.data_length >= size || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + /* Don't return any data if interrupted */ + if (ret) + return ret; + + goto again; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_recvmsg); + +static bool smbdirect_map_sges_single_page(struct smbdirect_map_sges *state, + struct page *page, size_t off, size_t len) +{ + struct ib_sge *sge; + u64 addr; + + if (state->num_sge >= state->max_sge) + return false; + + addr = ib_dma_map_page(state->device, page, + off, len, state->direction); + if (ib_dma_mapping_error(state->device, addr)) + return false; + + sge = &state->sge[state->num_sge++]; + sge->addr = addr; + sge->length = len; + sge->lkey = state->local_dma_lkey; + + return true; +} + +/* + * Extract page fragments from a BVEC-class iterator and add them to an ib_sge + * list. The pages are not pinned. + */ +static ssize_t smbdirect_map_sges_from_bvec(struct iov_iter *iter, + struct smbdirect_map_sges *state, + ssize_t maxsize) +{ + const struct bio_vec *bv = iter->bvec; + unsigned long start = iter->iov_offset; + unsigned int i; + ssize_t ret = 0; + + for (i = 0; i < iter->nr_segs; i++) { + size_t off, len; + bool ok; + + len = bv[i].bv_len; + if (start >= len) { + start -= len; + continue; + } + + len = min_t(size_t, maxsize, len - start); + off = bv[i].bv_offset + start; + + ok = smbdirect_map_sges_single_page(state, + bv[i].bv_page, + off, + len); + if (!ok) + return -EIO; + + ret += len; + maxsize -= len; + if (state->num_sge >= state->max_sge || maxsize <= 0) + break; + start = 0; + } + + if (ret > 0) + iov_iter_advance(iter, ret); + return ret; +} + +/* + * Extract fragments from a KVEC-class iterator and add them to an ib_sge list. + * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers. + * The pages are not pinned. + */ +static ssize_t smbdirect_map_sges_from_kvec(struct iov_iter *iter, + struct smbdirect_map_sges *state, + ssize_t maxsize) +{ + const struct kvec *kv = iter->kvec; + unsigned long start = iter->iov_offset; + unsigned int i; + ssize_t ret = 0; + + for (i = 0; i < iter->nr_segs; i++) { + struct page *page; + unsigned long kaddr; + size_t off, len, seg; + + len = kv[i].iov_len; + if (start >= len) { + start -= len; + continue; + } + + kaddr = (unsigned long)kv[i].iov_base + start; + off = kaddr & ~PAGE_MASK; + len = min_t(size_t, maxsize, len - start); + kaddr &= PAGE_MASK; + + maxsize -= len; + do { + bool ok; + + seg = min_t(size_t, len, PAGE_SIZE - off); + + if (is_vmalloc_or_module_addr((void *)kaddr)) + page = vmalloc_to_page((void *)kaddr); + else + page = virt_to_page((void *)kaddr); + + ok = smbdirect_map_sges_single_page(state, page, off, seg); + if (!ok) + return -EIO; + + ret += seg; + len -= seg; + kaddr += PAGE_SIZE; + off = 0; + } while (len > 0 && state->num_sge < state->max_sge); + + if (state->num_sge >= state->max_sge || maxsize <= 0) + break; + start = 0; + } + + if (ret > 0) + iov_iter_advance(iter, ret); + return ret; +} + +/* + * Extract folio fragments from a FOLIOQ-class iterator and add them to an + * ib_sge list. The folios are not pinned. + */ +static ssize_t smbdirect_map_sges_from_folioq(struct iov_iter *iter, + struct smbdirect_map_sges *state, + ssize_t maxsize) +{ + const struct folio_queue *folioq = iter->folioq; + unsigned int slot = iter->folioq_slot; + ssize_t ret = 0; + size_t offset = iter->iov_offset; + + if (WARN_ON_ONCE(!folioq)) + return -EIO; + + if (slot >= folioq_nr_slots(folioq)) { + folioq = folioq->next; + if (WARN_ON_ONCE(!folioq)) + return -EIO; + slot = 0; + } + + do { + struct folio *folio = folioq_folio(folioq, slot); + size_t fsize = folioq_folio_size(folioq, slot); + + if (offset < fsize) { + size_t part = umin(maxsize, fsize - offset); + bool ok; + + ok = smbdirect_map_sges_single_page(state, + folio_page(folio, 0), + offset, + part); + if (!ok) + return -EIO; + + offset += part; + ret += part; + maxsize -= part; + } + + if (offset >= fsize) { + offset = 0; + slot++; + if (slot >= folioq_nr_slots(folioq)) { + if (!folioq->next) { + WARN_ON_ONCE(ret < iter->count); + break; + } + folioq = folioq->next; + slot = 0; + } + } + } while (state->num_sge < state->max_sge && maxsize > 0); + + iter->folioq = folioq; + iter->folioq_slot = slot; + iter->iov_offset = offset; + iter->count -= ret; + return ret; +} + +/* + * Extract page fragments from up to the given amount of the source iterator + * and build up an ib_sge list that refers to all of those bits. The ib_sge list + * is appended to, up to the maximum number of elements set in the parameter + * block. + * + * The extracted page fragments are not pinned or ref'd in any way; if an + * IOVEC/UBUF-type iterator is to be used, it should be converted to a + * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some + * way. + */ +static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, + struct smbdirect_map_sges *state) +{ + ssize_t ret; + size_t before = state->num_sge; + + if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) + return -EIO; + + switch (iov_iter_type(iter)) { + case ITER_BVEC: + ret = smbdirect_map_sges_from_bvec(iter, state, len); + break; + case ITER_KVEC: + ret = smbdirect_map_sges_from_kvec(iter, state, len); + break; + case ITER_FOLIOQ: + ret = smbdirect_map_sges_from_folioq(iter, state, len); + break; + default: + WARN_ONCE(1, "iov_iter_type[%u]\n", iov_iter_type(iter)); + return -EIO; + } + + if (ret < 0) { + while (state->num_sge > before) { + struct ib_sge *sge = &state->sge[state->num_sge--]; + + ib_dma_unmap_page(state->device, + sge->addr, + sge->length, + state->direction); + } + } + + return ret; +} diff --git a/fs/smb/smbdirect/debug.c b/fs/smb/smbdirect/debug.c new file mode 100644 index 000000000000..a66a19d4a463 --- /dev/null +++ b/fs/smb/smbdirect/debug.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include + +void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, + unsigned int rdma_readwrite_threshold, + struct seq_file *m) +{ + const struct smbdirect_socket_parameters *sp; + + if (!sc) + return; + sp = &sc->parameters; + + seq_puts(m, "\n"); + seq_printf(m, "SMBDirect protocol version: 0x%x ", + SMBDIRECT_V1); + seq_printf(m, "transport status: %s (%u)", + smbdirect_socket_status_string(sc->status), + sc->status); + + seq_puts(m, "\n"); + seq_printf(m, "Conn receive_credit_max: %u ", + sp->recv_credit_max); + seq_printf(m, "send_credit_target: %u max_send_size: %u", + sp->send_credit_target, + sp->max_send_size); + + seq_puts(m, "\n"); + seq_printf(m, "Conn max_fragmented_recv_size: %u ", + sp->max_fragmented_recv_size); + seq_printf(m, "max_fragmented_send_size: %u max_receive_size:%u", + sp->max_fragmented_send_size, + sp->max_recv_size); + + seq_puts(m, "\n"); + seq_printf(m, "Conn keep_alive_interval: %u ", + sp->keepalive_interval_msec * 1000); + seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u", + sp->max_read_write_size, + rdma_readwrite_threshold); + + seq_puts(m, "\n"); + seq_printf(m, "Debug count_get_receive_buffer: %llu ", + sc->statistics.get_receive_buffer); + seq_printf(m, "count_put_receive_buffer: %llu count_send_empty: %llu", + sc->statistics.put_receive_buffer, + sc->statistics.send_empty); + + seq_puts(m, "\n"); + seq_printf(m, "Read Queue count_enqueue_reassembly_queue: %llu ", + sc->statistics.enqueue_reassembly_queue); + seq_printf(m, "count_dequeue_reassembly_queue: %llu ", + sc->statistics.dequeue_reassembly_queue); + seq_printf(m, "reassembly_data_length: %u ", + sc->recv_io.reassembly.data_length); + seq_printf(m, "reassembly_queue_length: %u", + sc->recv_io.reassembly.queue_length); + + seq_puts(m, "\n"); + seq_printf(m, "Current Credits send_credits: %u ", + atomic_read(&sc->send_io.credits.count)); + seq_printf(m, "receive_credits: %u receive_credit_target: %u", + atomic_read(&sc->recv_io.credits.count), + sc->recv_io.credits.target); + + seq_puts(m, "\n"); + seq_printf(m, "Pending send_pending: %u ", + atomic_read(&sc->send_io.pending.count)); + + seq_puts(m, "\n"); + seq_printf(m, "MR responder_resources: %u ", + sp->responder_resources); + seq_printf(m, "max_frmr_depth: %u mr_type: 0x%x", + sp->max_frmr_depth, + sc->mr_io.type); + + seq_puts(m, "\n"); + seq_printf(m, "MR mr_ready_count: %u mr_used_count: %u", + atomic_read(&sc->mr_io.ready.count), + atomic_read(&sc->mr_io.used.count)); +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_legacy_debug_proc_show); diff --git a/fs/smb/smbdirect/devices.c b/fs/smb/smbdirect/devices.c new file mode 100644 index 000000000000..44962f221c35 --- /dev/null +++ b/fs/smb/smbdirect/devices.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025 Stefan Metzmacher + */ + +#include "internal.h" + +static u8 smbdirect_ib_device_rdma_capable_node_type(struct ib_device *ib_dev) +{ + if (!smbdirect_frwr_is_supported(&ib_dev->attrs)) + return RDMA_NODE_UNSPECIFIED; + + switch (ib_dev->node_type) { + case RDMA_NODE_IB_CA: /* Infiniband, RoCE v1 and v2 */ + case RDMA_NODE_RNIC: /* iWarp */ + return ib_dev->node_type; + } + + return RDMA_NODE_UNSPECIFIED; +} + +static int smbdirect_ib_client_add(struct ib_device *ib_dev) +{ + u8 node_type = smbdirect_ib_device_rdma_capable_node_type(ib_dev); + struct smbdirect_device *sdev; + const char *node_str; + const char *action; + u32 pidx; + + switch (node_type) { + case RDMA_NODE_IB_CA: + node_str = "IB_CA"; + action = "added"; + break; + case RDMA_NODE_RNIC: + node_str = "RNIC"; + action = "added"; + break; + case RDMA_NODE_UNSPECIFIED: + node_str = "UNSPECIFIED"; + action = "ignored"; + break; + default: + node_str = "UNKNOWN"; + action = "ignored"; + node_type = RDMA_NODE_UNSPECIFIED; + break; + } + + pr_info("ib_dev[%.*s]: %s: %s %s=%u %s=0x%llx %s=0x%llx %s=0x%llx\n", + IB_DEVICE_NAME_MAX, + ib_dev->name, + action, + node_str, + "max_fast_reg_page_list_len", + ib_dev->attrs.max_fast_reg_page_list_len, + "device_cap_flags", + ib_dev->attrs.device_cap_flags, + "kernel_cap_flags", + ib_dev->attrs.kernel_cap_flags, + "page_size_cap", + ib_dev->attrs.page_size_cap); + + if (node_type == RDMA_NODE_UNSPECIFIED) + return 0; + + pr_info("ib_dev[%.*s]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u\n", + IB_DEVICE_NAME_MAX, + ib_dev->name, + "num_ports", + rdma_end_port(ib_dev), + "max_qp_rd_atom", + ib_dev->attrs.max_qp_rd_atom, + "max_qp_init_rd_atom", + ib_dev->attrs.max_qp_init_rd_atom, + "max_sgl_rd", + ib_dev->attrs.max_sgl_rd, + "max_sge_rd", + ib_dev->attrs.max_sge_rd, + "max_cqe", + ib_dev->attrs.max_cqe, + "max_qp_wr", + ib_dev->attrs.max_qp_wr, + "max_send_sge", + ib_dev->attrs.max_send_sge, + "max_recv_sge", + ib_dev->attrs.max_recv_sge); + + rdma_for_each_port(ib_dev, pidx) { + const struct ib_port_immutable *ib_pi = + ib_port_immutable_read(ib_dev, pidx); + u32 core_cap_flags = ib_pi ? ib_pi->core_cap_flags : 0; + + pr_info("ib_dev[%.*s]PORT[%u]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=0x%x\n", + IB_DEVICE_NAME_MAX, + ib_dev->name, + pidx, + "iwarp", + rdma_protocol_iwarp(ib_dev, pidx), + "ib", + rdma_protocol_ib(ib_dev, pidx), + "roce", + rdma_protocol_roce(ib_dev, pidx), + "v1", + rdma_protocol_roce_eth_encap(ib_dev, pidx), + "v2", + rdma_protocol_roce_udp_encap(ib_dev, pidx), + "core_cap_flags", + core_cap_flags); + } + + sdev = kzalloc_obj(*sdev); + if (!sdev) + return -ENOMEM; + sdev->ib_dev = ib_dev; + snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", + IB_DEVICE_NAME_MAX, ib_dev->name); + + write_lock(&smbdirect_globals.devices.lock); + list_add(&sdev->list, &smbdirect_globals.devices.list); + write_unlock(&smbdirect_globals.devices.lock); + + return 0; +} + +static void smbdirect_ib_client_remove(struct ib_device *ib_dev, void *client_data) +{ + struct smbdirect_device *sdev, *tmp; + + write_lock(&smbdirect_globals.devices.lock); + list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { + if (sdev->ib_dev == ib_dev) { + list_del(&sdev->list); + pr_info("ib_dev[%.*s] removed\n", + IB_DEVICE_NAME_MAX, sdev->ib_name); + kfree(sdev); + break; + } + } + write_unlock(&smbdirect_globals.devices.lock); +} + +static void smbdirect_ib_client_rename(struct ib_device *ib_dev, void *client_data) +{ + struct smbdirect_device *sdev; + + write_lock(&smbdirect_globals.devices.lock); + list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { + if (sdev->ib_dev == ib_dev) { + pr_info("ib_dev[%.*s] renamed to [%.*s]\n", + IB_DEVICE_NAME_MAX, sdev->ib_name, + IB_DEVICE_NAME_MAX, ib_dev->name); + snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", + IB_DEVICE_NAME_MAX, ib_dev->name); + break; + } + } + write_unlock(&smbdirect_globals.devices.lock); +} + +static struct ib_client smbdirect_ib_client = { + .name = "smbdirect_ib_client", + .add = smbdirect_ib_client_add, + .remove = smbdirect_ib_client_remove, + .rename = smbdirect_ib_client_rename, +}; + +static u8 smbdirect_netdev_find_rdma_capable_node_type(struct net_device *netdev) +{ + struct smbdirect_device *sdev; + u8 node_type = RDMA_NODE_UNSPECIFIED; + + read_lock(&smbdirect_globals.devices.lock); + list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { + u32 pi; + + rdma_for_each_port(sdev->ib_dev, pi) { + struct net_device *ndev; + + ndev = ib_device_get_netdev(sdev->ib_dev, pi); + if (!ndev) + continue; + + if (ndev == netdev) { + dev_put(ndev); + node_type = sdev->ib_dev->node_type; + goto out; + } + dev_put(ndev); + } + } +out: + read_unlock(&smbdirect_globals.devices.lock); + + if (node_type == RDMA_NODE_UNSPECIFIED) { + struct ib_device *ibdev; + + ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); + if (ibdev) { + node_type = smbdirect_ib_device_rdma_capable_node_type(ibdev); + ib_device_put(ibdev); + } + } + + return node_type; +} + +/* + * Returns RDMA_NODE_UNSPECIFIED when the netdev has + * no support for smbdirect capable rdma. + * + * Otherwise RDMA_NODE_RNIC is returned for iwarp devices + * and RDMA_NODE_IB_CA or Infiniband and RoCE (v1 and v2) + */ +u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev) +{ + struct net_device *lower_dev; + struct list_head *iter; + u8 node_type = RDMA_NODE_UNSPECIFIED; + + node_type = smbdirect_netdev_find_rdma_capable_node_type(netdev); + if (node_type != RDMA_NODE_UNSPECIFIED) + return node_type; + + /* check if netdev is bridge or VLAN */ + if (netif_is_bridge_master(netdev) || netdev->priv_flags & IFF_802_1Q_VLAN) + netdev_for_each_lower_dev(netdev, lower_dev, iter) { + node_type = smbdirect_netdev_find_rdma_capable_node_type(lower_dev); + if (node_type != RDMA_NODE_UNSPECIFIED) + return node_type; + } + + /* check if netdev is IPoIB safely without layer violation */ + if (netdev->type == ARPHRD_INFINIBAND) + return RDMA_NODE_IB_CA; + + return RDMA_NODE_UNSPECIFIED; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_netdev_rdma_capable_node_type); + +__init int smbdirect_devices_init(void) +{ + int ret; + + rwlock_init(&smbdirect_globals.devices.lock); + INIT_LIST_HEAD(&smbdirect_globals.devices.list); + + ret = ib_register_client(&smbdirect_ib_client); + if (ret) { + pr_crit("failed to ib_register_client: %d %1pe\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} + +__exit void smbdirect_devices_exit(void) +{ + struct smbdirect_device *sdev, *tmp; + + /* + * On exist we just cleanup so that + * smbdirect_ib_client_remove() won't + * print removals of devices. + */ + write_lock(&smbdirect_globals.devices.lock); + list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { + list_del(&sdev->list); + kfree(sdev); + } + write_unlock(&smbdirect_globals.devices.lock); + + ib_unregister_client(&smbdirect_ib_client); +} diff --git a/fs/smb/smbdirect/internal.h b/fs/smb/smbdirect/internal.h new file mode 100644 index 000000000000..2d5acf2c21bc --- /dev/null +++ b/fs/smb/smbdirect/internal.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2025, Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ +#define __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "smbdirect.h" +#include "pdu.h" +#include "public.h" + +#include + +struct smbdirect_module_state { + struct mutex mutex; + + struct { + struct workqueue_struct *accept; + struct workqueue_struct *connect; + struct workqueue_struct *idle; + struct workqueue_struct *refill; + struct workqueue_struct *immediate; + struct workqueue_struct *cleanup; + } workqueues; + + struct { + rwlock_t lock; + struct list_head list; + } devices; +}; + +extern struct smbdirect_module_state smbdirect_globals; + +#include "socket.h" + +struct smbdirect_device { + struct list_head list; + struct ib_device *ib_dev; + /* + * copy of ib_dev->name, + * in order to print renames + */ + char ib_name[IB_DEVICE_NAME_MAX]; +}; + +int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc); + +int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc); + +void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, + const char *macro_name, + unsigned int lvl, + const char *func, + unsigned int line, + int error, + enum smbdirect_socket_status *force_status); +#define smbdirect_socket_schedule_cleanup(__sc, __error) \ + __smbdirect_socket_schedule_cleanup(__sc, \ + "smbdirect_socket_schedule_cleanup", SMBDIRECT_LOG_ERR, \ + __func__, __LINE__, __error, NULL) +#define smbdirect_socket_schedule_cleanup_lvl(__sc, __lvl, __error) \ + __smbdirect_socket_schedule_cleanup(__sc, \ + "smbdirect_socket_schedule_cleanup_lvl", __lvl, \ + __func__, __LINE__, __error, NULL) +#define smbdirect_socket_schedule_cleanup_status(__sc, __lvl, __error, __status) do { \ + enum smbdirect_socket_status __force_status = __status; \ + __smbdirect_socket_schedule_cleanup(__sc, \ + "smbdirect_socket_schedule_cleanup_status", __lvl, \ + __func__, __LINE__, __error, &__force_status); \ +} while (0) + +void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc); + +int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, + enum smbdirect_socket_status expected_status, + int unexpected_errno, + wait_queue_head_t *waitq, + atomic_t *total_credits, + int needed); + +void smbdirect_connection_rdma_established(struct smbdirect_socket *sc); + +void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc); + +int smbdirect_connection_create_qp(struct smbdirect_socket *sc); + +void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc); + +int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc); + +void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc); + +struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc); + +void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg); + +struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc); + +void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg); + +void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc, + struct smbdirect_recv_io *msg, + u32 data_length); + +struct smbdirect_recv_io * +smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc); + +void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc, + u8 peer_initiator_depth, + u8 peer_responder_resources, + const struct rdma_conn_param *param); + +void smbdirect_connection_idle_timer_work(struct work_struct *work); + +u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc); + +int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc, + struct ib_send_wr *wr); + +int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg); + +void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc); + +int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc); + +int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc); + +void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc); + +int smbdirect_accept_connect_request(struct smbdirect_socket *sc, + const struct rdma_conn_param *param); + +void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus); + +__init int smbdirect_devices_init(void); +__exit void smbdirect_devices_exit(void); + +#endif /* __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ */ diff --git a/fs/smb/smbdirect/listen.c b/fs/smb/smbdirect/listen.c new file mode 100644 index 000000000000..143a7618d95f --- /dev/null +++ b/fs/smb/smbdirect/listen.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event); + +int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog) +{ + int ret; + + if (backlog < 0) + return -EINVAL; + if (!backlog) + backlog = 1; /* use 1 as default for now */ + + if (sc->first_error) + return -EINVAL; + + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + if (WARN_ON_ONCE(!sc->rdma.cm_id)) + return -EINVAL; + + if (sc->rdma.cm_id->device) + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "try to listen on addr: %pISpsfc dev: %.*s\n", + &sc->rdma.cm_id->route.addr.src_addr, + IB_DEVICE_NAME_MAX, + sc->rdma.cm_id->device->name); + else + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "try to listen on addr: %pISpsfc\n", + &sc->rdma.cm_id->route.addr.src_addr); + + /* already checked above */ + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); + sc->status = SMBDIRECT_SOCKET_LISTENING; + sc->rdma.expected_event = RDMA_CM_EVENT_CONNECT_REQUEST; + rdma_lock_handler(sc->rdma.cm_id); + sc->rdma.cm_id->event_handler = smbdirect_listen_rdma_event_handler; + rdma_unlock_handler(sc->rdma.cm_id); + + ret = rdma_listen(sc->rdma.cm_id, backlog); + if (ret) { + sc->first_error = ret; + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; + if (sc->rdma.cm_id->device) + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening failed %1pe on addr: %pISpsfc dev: %.*s\n", + SMBDIRECT_DEBUG_ERR_PTR(ret), + &sc->rdma.cm_id->route.addr.src_addr, + IB_DEVICE_NAME_MAX, + sc->rdma.cm_id->device->name); + else + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening failed %1pe on addr: %pISpsfc\n", + SMBDIRECT_DEBUG_ERR_PTR(ret), + &sc->rdma.cm_id->route.addr.src_addr); + return ret; + } + + /* + * This is a value > 0, checked above, + * so we are able to use sc->listen.backlog == -1, + * as indication that the socket was never + * a listener. + */ + sc->listen.backlog = backlog; + + if (sc->rdma.cm_id->device) + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening on addr: %pISpsfc dev: %.*s\n", + &sc->rdma.cm_id->route.addr.src_addr, + IB_DEVICE_NAME_MAX, + sc->rdma.cm_id->device->name); + else + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening on addr: %pISpsfc\n", + &sc->rdma.cm_id->route.addr.src_addr); + + /* + * The rest happens async via smbdirect_listen_rdma_event_handler() + */ + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_listen); + +static int smbdirect_new_rdma_event_handler(struct rdma_cm_id *new_id, + struct rdma_cm_event *event) +{ + int ret = -ESTALE; + + /* + * This should be replaced before any real work + * starts! So it should never be called! + */ + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + WARN_ONCE(1, + "%s should not be called! event=%s status=%d => ret=%1pe\n", + __func__, + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return -ESTALE; +} + +static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc, + struct rdma_cm_id *new_id, + const struct rdma_cm_event *event); + +static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *new_id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *lsc = new_id->context; + int ret; + + if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { + new_id->context = NULL; + new_id->event_handler = smbdirect_new_rdma_event_handler; + } else + new_id = NULL; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here, + * as we might sleep. + */ + WARN_ON_ONCE(in_interrupt()); + + if (event->status || event->event != lsc->rdma.expected_event) { + ret = -ECONNABORTED; + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(lsc->status), + SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error), + rdma_event_msg(lsc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + /* + * In case of error return it and let the caller + * destroy new_id + */ + smbdirect_socket_schedule_cleanup(lsc, ret); + return new_id ? ret : 0; + } + + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(lsc->status), + SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error), + rdma_event_msg(event->event)); + + /* + * In case of error return it and let the caller + * destroy new_id + */ + if (lsc->first_error) + return new_id ? lsc->first_error : 0; + + switch (event->event) { + case RDMA_CM_EVENT_CONNECT_REQUEST: + WARN_ON_ONCE(lsc->status != SMBDIRECT_SOCKET_LISTENING); + + /* + * In case of error return it and let the caller + * destroy new_id + */ + ret = smbdirect_listen_connect_request(lsc, new_id, event); + if (ret) + return ret; + return 0; + + default: + break; + } + + /* + * This is an internal error + */ + WARN_ON_ONCE(lsc->rdma.expected_event != RDMA_CM_EVENT_CONNECT_REQUEST); + smbdirect_socket_schedule_cleanup(lsc, -EINVAL); + return 0; +} + +static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc, + struct rdma_cm_id *new_id, + const struct rdma_cm_event *event) +{ + const struct smbdirect_socket_parameters *lsp = &lsc->parameters; + struct smbdirect_socket *nsc; + unsigned long flags; + size_t backlog = max_t(size_t, 1, lsc->listen.backlog); + size_t psockets; + size_t rsockets; + int ret; + + if (!smbdirect_frwr_is_supported(&new_id->device->attrs)) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Fast Registration Work Requests (FRWR) is not supported device %.*s\n", + IB_DEVICE_NAME_MAX, + new_id->device->name); + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", + new_id->device->attrs.device_cap_flags, + new_id->device->attrs.max_fast_reg_page_list_len); + return -EPROTONOSUPPORT; + } + + if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && + !rdma_ib_or_roce(new_id->device, new_id->port_num)) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + new_id->device->name, + rdma_protocol_iwarp(new_id->device, new_id->port_num), + &new_id->route.addr.src_addr, + &new_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW && + !rdma_protocol_iwarp(new_id->device, new_id->port_num)) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + new_id->device->name, + rdma_ib_or_roce(new_id->device, new_id->port_num), + &new_id->route.addr.src_addr, + &new_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + + spin_lock_irqsave(&lsc->listen.lock, flags); + psockets = list_count_nodes(&lsc->listen.pending); + rsockets = list_count_nodes(&lsc->listen.ready); + spin_unlock_irqrestore(&lsc->listen.lock, flags); + + if (psockets > backlog || + rsockets > backlog || + (psockets + rsockets) > backlog) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Backlog[%d][%zu] full pending[%zu] ready[%zu]\n", + lsc->listen.backlog, backlog, psockets, rsockets); + return -EBUSY; + } + + ret = smbdirect_socket_create_accepting(new_id, &nsc); + if (ret) + goto socket_init_failed; + + nsc->logging = lsc->logging; + ret = smbdirect_socket_set_initial_parameters(nsc, &lsc->parameters); + if (ret) + goto set_params_failed; + ret = smbdirect_socket_set_kernel_settings(nsc, + lsc->ib.poll_ctx, + lsc->send_io.mem.gfp_mask); + if (ret) + goto set_settings_failed; + + spin_lock_irqsave(&lsc->listen.lock, flags); + list_add_tail(&nsc->accept.list, &lsc->listen.pending); + nsc->accept.listener = lsc; + spin_unlock_irqrestore(&lsc->listen.lock, flags); + + ret = smbdirect_accept_connect_request(nsc, &event->param.conn); + if (ret) + goto accept_connect_failed; + + return 0; + +accept_connect_failed: + spin_lock_irqsave(&lsc->listen.lock, flags); + list_del_init(&nsc->accept.list); + nsc->accept.listener = NULL; + spin_unlock_irqrestore(&lsc->listen.lock, flags); +set_settings_failed: +set_params_failed: + /* + * The caller will destroy new_id + */ + nsc->ib.dev = NULL; + nsc->rdma.cm_id = NULL; + smbdirect_socket_release(nsc); +socket_init_failed: + return ret; +} diff --git a/fs/smb/smbdirect/main.c b/fs/smb/smbdirect/main.c new file mode 100644 index 000000000000..606732fefb69 --- /dev/null +++ b/fs/smb/smbdirect/main.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include + +struct smbdirect_module_state smbdirect_globals = { + .mutex = __MUTEX_INITIALIZER(smbdirect_globals.mutex), +}; + +static __init int smbdirect_module_init(void) +{ + int ret = -ENOMEM; + + pr_notice("subsystem loading...\n"); + mutex_lock(&smbdirect_globals.mutex); + + smbdirect_globals.workqueues.accept = alloc_workqueue("smbdirect-accept", + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.accept == NULL) + goto alloc_accept_wq_failed; + + smbdirect_globals.workqueues.connect = alloc_workqueue("smbdirect-connect", + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.connect == NULL) + goto alloc_connect_wq_failed; + + smbdirect_globals.workqueues.idle = alloc_workqueue("smbdirect-idle", + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.idle == NULL) + goto alloc_idle_wq_failed; + + smbdirect_globals.workqueues.refill = alloc_workqueue("smbdirect-refill", + WQ_HIGHPRI | + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.refill == NULL) + goto alloc_refill_wq_failed; + + smbdirect_globals.workqueues.immediate = alloc_workqueue("smbdirect-immediate", + WQ_HIGHPRI | + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.immediate == NULL) + goto alloc_immediate_wq_failed; + + smbdirect_globals.workqueues.cleanup = alloc_workqueue("smbdirect-cleanup", + WQ_MEM_RECLAIM | + WQ_HIGHPRI | + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.cleanup == NULL) + goto alloc_cleanup_wq_failed; + + ret = smbdirect_devices_init(); + if (ret) + goto devices_init_failed; + + mutex_unlock(&smbdirect_globals.mutex); + pr_notice("subsystem loaded\n"); + return 0; + +devices_init_failed: + destroy_workqueue(smbdirect_globals.workqueues.cleanup); +alloc_cleanup_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.immediate); +alloc_immediate_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.refill); +alloc_refill_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.idle); +alloc_idle_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.connect); +alloc_connect_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.accept); +alloc_accept_wq_failed: + mutex_unlock(&smbdirect_globals.mutex); + pr_crit("failed to loaded: %d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; +} + +static __exit void smbdirect_module_exit(void) +{ + pr_notice("subsystem unloading...\n"); + mutex_lock(&smbdirect_globals.mutex); + + smbdirect_devices_exit(); + + destroy_workqueue(smbdirect_globals.workqueues.accept); + destroy_workqueue(smbdirect_globals.workqueues.connect); + destroy_workqueue(smbdirect_globals.workqueues.idle); + destroy_workqueue(smbdirect_globals.workqueues.refill); + destroy_workqueue(smbdirect_globals.workqueues.immediate); + destroy_workqueue(smbdirect_globals.workqueues.cleanup); + + mutex_unlock(&smbdirect_globals.mutex); + pr_notice("subsystem unloaded\n"); +} + +module_init(smbdirect_module_init); +module_exit(smbdirect_module_exit); + +MODULE_DESCRIPTION("smbdirect subsystem"); +MODULE_LICENSE("GPL"); diff --git a/fs/smb/smbdirect/mr.c b/fs/smb/smbdirect/mr.c new file mode 100644 index 000000000000..5228e699cd5d --- /dev/null +++ b/fs/smb/smbdirect/mr.c @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +/* + * Allocate MRs used for RDMA read/write + * The number of MRs will not exceed hardware capability in responder_resources + * All MRs are kept in mr_list. The MR can be recovered after it's used + * Recovery is done in smbd_mr_recovery_work. The content of list entry changes + * as MRs are used and recovered for I/O, but the list links will not change + */ +int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_mr_io *mr; + int ret; + u32 i; + + if (sp->responder_resources == 0) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "responder_resources negotiated as 0\n"); + return -EINVAL; + } + + /* Allocate more MRs (2x) than hardware responder_resources */ + for (i = 0; i < sp->responder_resources * 2; i++) { + mr = kzalloc_obj(*mr); + if (!mr) { + ret = -ENOMEM; + goto kzalloc_mr_failed; + } + + kref_init(&mr->kref); + mutex_init(&mr->mutex); + + mr->mr = ib_alloc_mr(sc->ib.pd, + sc->mr_io.type, + sp->max_frmr_depth); + if (IS_ERR(mr->mr)) { + ret = PTR_ERR(mr->mr); + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_alloc_mr failed ret=%d (%1pe) type=0x%x max_frmr_depth=%u\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret), + sc->mr_io.type, sp->max_frmr_depth); + goto ib_alloc_mr_failed; + } + mr->sgt.sgl = kzalloc_objs(struct scatterlist, sp->max_frmr_depth); + if (!mr->sgt.sgl) { + ret = -ENOMEM; + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "failed to allocate sgl, max_frmr_depth=%u\n", + sp->max_frmr_depth); + goto kcalloc_sgl_failed; + } + mr->state = SMBDIRECT_MR_READY; + mr->socket = sc; + + list_add_tail(&mr->list, &sc->mr_io.all.list); + atomic_inc(&sc->mr_io.ready.count); + } + + return 0; + +kcalloc_sgl_failed: + ib_dereg_mr(mr->mr); +ib_alloc_mr_failed: + mutex_destroy(&mr->mutex); + kfree(mr); +kzalloc_mr_failed: + smbdirect_connection_destroy_mr_list(sc); + return ret; +} + +static void smbdirect_mr_io_disable_locked(struct smbdirect_mr_io *mr) +{ + struct smbdirect_socket *sc = mr->socket; + + lockdep_assert_held(&mr->mutex); + + if (mr->state == SMBDIRECT_MR_DISABLED) + return; + + if (mr->mr) + ib_dereg_mr(mr->mr); + if (mr->sgt.nents) + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + kfree(mr->sgt.sgl); + + mr->mr = NULL; + mr->sgt.sgl = NULL; + mr->sgt.nents = 0; + + mr->state = SMBDIRECT_MR_DISABLED; +} + +static void smbdirect_mr_io_free_locked(struct kref *kref) +{ + struct smbdirect_mr_io *mr = + container_of(kref, struct smbdirect_mr_io, kref); + + lockdep_assert_held(&mr->mutex); + + /* + * smbdirect_mr_io_disable_locked() should already be called! + */ + if (WARN_ON_ONCE(mr->state != SMBDIRECT_MR_DISABLED)) + smbdirect_mr_io_disable_locked(mr); + + mutex_unlock(&mr->mutex); + mutex_destroy(&mr->mutex); + kfree(mr); +} + +void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc) +{ + struct smbdirect_mr_io *mr, *tmp; + LIST_HEAD(all_list); + unsigned long flags; + + spin_lock_irqsave(&sc->mr_io.all.lock, flags); + list_splice_tail_init(&sc->mr_io.all.list, &all_list); + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); + + list_for_each_entry_safe(mr, tmp, &all_list, list) { + mutex_lock(&mr->mutex); + + smbdirect_mr_io_disable_locked(mr); + list_del(&mr->list); + mr->socket = NULL; + + /* + * No kref_put_mutex() as it's already locked. + * + * If smbdirect_mr_io_free_locked() is called + * and the mutex is unlocked and mr is gone, + * in that case kref_put() returned 1. + * + * If kref_put() returned 0 we know that + * smbdirect_mr_io_free_locked() didn't + * run. Not by us nor by anyone else, as we + * still hold the mutex, so we need to unlock. + * + * If the mr is still registered it will + * be dangling (detached from the connection + * waiting for smbd_deregister_mr() to be + * called in order to free the memory. + */ + if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) + mutex_unlock(&mr->mutex); + } +} + +/* + * Get a MR from mr_list. This function waits until there is at least one MR + * available in the list. There may be several CPUs issuing I/O trying to get MR + * at the same time, mr_list_lock is used to protect this situation. + */ +static struct smbdirect_mr_io * +smbdirect_connection_get_mr_io(struct smbdirect_socket *sc) +{ + struct smbdirect_mr_io *mr; + unsigned long flags; + int ret; + +again: + ret = wait_event_interruptible(sc->mr_io.ready.wait_queue, + atomic_read(&sc->mr_io.ready.count) || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "wait_event_interruptible ret=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return NULL; + } + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "sc->status=%s sc->first_error=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + return NULL; + } + + spin_lock_irqsave(&sc->mr_io.all.lock, flags); + list_for_each_entry(mr, &sc->mr_io.all.list, list) { + if (mr->state == SMBDIRECT_MR_READY) { + mr->state = SMBDIRECT_MR_REGISTERED; + kref_get(&mr->kref); + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); + atomic_dec(&sc->mr_io.ready.count); + atomic_inc(&sc->mr_io.used.count); + return mr; + } + } + + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); + /* + * It is possible that we could fail to get MR because other processes may + * try to acquire a MR at the same time. If this is the case, retry it. + */ + goto again; +} + +static void smbdirect_connection_mr_io_register_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_mr_io *mr = + container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); + struct smbdirect_socket *sc = mr->socket; + + if (wc->status != IB_WC_SUCCESS) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s opcode=%d\n", + ib_wc_status_msg(wc->status), wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + } +} + +static void smbdirect_connection_mr_io_local_inv_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_mr_io *mr = + container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); + struct smbdirect_socket *sc = mr->socket; + + mr->state = SMBDIRECT_MR_INVALIDATED; + if (wc->status != IB_WC_SUCCESS) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "invalidate failed status=%s\n", + ib_wc_status_msg(wc->status)); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + } + complete(&mr->invalidate_done); +} + +/* + * Transcribe the pages from an iterator into an MR scatterlist. + */ +static int smbdirect_iter_to_sgt(struct iov_iter *iter, + struct sg_table *sgt, + unsigned int max_sg) +{ + int ret; + + memset(sgt->sgl, 0, max_sg * sizeof(struct scatterlist)); + + ret = extract_iter_to_sg(iter, iov_iter_count(iter), sgt, max_sg, 0); + WARN_ON(ret < 0); + if (sgt->nents > 0) + sg_mark_end(&sgt->sgl[sgt->nents - 1]); + + return ret; +} + +/* + * Register memory for RDMA read/write + * iter: the buffer to register memory with + * writing: true if this is a RDMA write (SMB read), false for RDMA read + * need_invalidate: true if this MR needs to be locally invalidated after I/O + * return value: the MR registered, NULL if failed. + */ +struct smbdirect_mr_io * +smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, + struct iov_iter *iter, + bool writing, + bool need_invalidate) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_mr_io *mr; + int ret, num_pages; + struct ib_reg_wr *reg_wr; + + num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1); + if (num_pages > sp->max_frmr_depth) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "num_pages=%d max_frmr_depth=%d\n", + num_pages, sp->max_frmr_depth); + WARN_ON_ONCE(1); + return NULL; + } + + mr = smbdirect_connection_get_mr_io(sc); + if (!mr) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_get_mr_io returning NULL\n"); + return NULL; + } + + mutex_lock(&mr->mutex); + + mr->dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + mr->need_invalidate = need_invalidate; + mr->sgt.nents = 0; + mr->sgt.orig_nents = 0; + + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, + "num_pages=%u count=%zu depth=%u\n", + num_pages, iov_iter_count(iter), sp->max_frmr_depth); + smbdirect_iter_to_sgt(iter, &mr->sgt, sp->max_frmr_depth); + + ret = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + if (!ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_dma_map_sg num_pages=%u dir=%x ret=%d (%1pe)\n", + num_pages, mr->dir, ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto dma_map_error; + } + + ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE); + if (ret != mr->sgt.nents) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_map_mr_sg failed ret = %d nents = %u\n", + ret, mr->sgt.nents); + goto map_mr_error; + } + + ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); + reg_wr = &mr->wr; + reg_wr->wr.opcode = IB_WR_REG_MR; + mr->cqe.done = smbdirect_connection_mr_io_register_done; + reg_wr->wr.wr_cqe = &mr->cqe; + reg_wr->wr.num_sge = 0; + reg_wr->wr.send_flags = IB_SEND_SIGNALED; + reg_wr->mr = mr->mr; + reg_wr->key = mr->mr->rkey; + reg_wr->access = writing ? + IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : + IB_ACCESS_REMOTE_READ; + + /* + * There is no need for waiting for complemtion on ib_post_send + * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution + * on the next ib_post_send when we actually send I/O to remote peer + */ + ret = ib_post_send(sc->ib.qp, ®_wr->wr, NULL); + if (!ret) { + /* + * smbdirect_connection_get_mr_io() gave us a reference + * via kref_get(&mr->kref), we keep that and let + * the caller use smbdirect_connection_deregister_mr_io() + * to remove it again. + */ + mutex_unlock(&mr->mutex); + return mr; + } + + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_post_send failed ret=%d (%1pe) reg_wr->key=0x%x\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret), reg_wr->key); + +map_mr_error: + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + +dma_map_error: + mr->sgt.nents = 0; + mr->state = SMBDIRECT_MR_ERROR; + atomic_dec(&sc->mr_io.used.count); + + smbdirect_socket_schedule_cleanup(sc, ret); + + /* + * smbdirect_connection_get_mr_io() gave us a reference + * via kref_get(&mr->kref), we need to remove it again + * on error. + * + * No kref_put_mutex() as it's already locked. + * + * If smbdirect_mr_io_free_locked() is called + * and the mutex is unlocked and mr is gone, + * in that case kref_put() returned 1. + * + * If kref_put() returned 0 we know that + * smbdirect_mr_io_free_locked() didn't + * run. Not by us nor by anyone else, as we + * still hold the mutex, so we need to unlock. + */ + if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) + mutex_unlock(&mr->mutex); + return NULL; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_register_mr_io); + +void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1) +{ + mutex_lock(&mr->mutex); + if (mr->state == SMBDIRECT_MR_REGISTERED) { + v1->offset = cpu_to_le64(mr->mr->iova); + v1->token = cpu_to_le32(mr->mr->rkey); + v1->length = cpu_to_le32(mr->mr->length); + } else { + v1->offset = cpu_to_le64(U64_MAX); + v1->token = cpu_to_le32(U32_MAX); + v1->length = cpu_to_le32(U32_MAX); + } + mutex_unlock(&mr->mutex); +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_mr_io_fill_buffer_descriptor); + +/* + * Deregister a MR after I/O is done + * This function may wait if remote invalidation is not used + * and we have to locally invalidate the buffer to prevent data is being + * modified by remote peer after upper layer consumes it + */ +void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr) +{ + struct smbdirect_socket *sc = mr->socket; + int ret = 0; + +lock_again: + mutex_lock(&mr->mutex); + if (mr->state == SMBDIRECT_MR_DISABLED) + goto put_kref; + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_mr_io_disable_locked(mr); + goto put_kref; + } + + if (mr->need_invalidate) { + struct ib_send_wr *wr = &mr->inv_wr; + + /* Need to finish local invalidation before returning */ + wr->opcode = IB_WR_LOCAL_INV; + mr->cqe.done = smbdirect_connection_mr_io_local_inv_done; + wr->wr_cqe = &mr->cqe; + wr->num_sge = 0; + wr->ex.invalidate_rkey = mr->mr->rkey; + wr->send_flags = IB_SEND_SIGNALED; + + init_completion(&mr->invalidate_done); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_post_send failed ret=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_mr_io_disable_locked(mr); + smbdirect_socket_schedule_cleanup(sc, ret); + goto done; + } + + /* + * We still hold the reference to mr + * so we can unlock while waiting. + */ + mutex_unlock(&mr->mutex); + wait_for_completion(&mr->invalidate_done); + mr->need_invalidate = false; + goto lock_again; + } else + /* + * For remote invalidation, just set it to SMBDIRECT_MR_INVALIDATED + * and defer to mr_recovery_work to recover the MR for next use + */ + mr->state = SMBDIRECT_MR_INVALIDATED; + + if (mr->sgt.nents) { + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + mr->sgt.nents = 0; + } + + WARN_ONCE(mr->state != SMBDIRECT_MR_INVALIDATED, + "mr->state[%u] != SMBDIRECT_MR_INVALIDATED[%u]\n", + mr->state, SMBDIRECT_MR_INVALIDATED); + mr->state = SMBDIRECT_MR_READY; + if (atomic_inc_return(&sc->mr_io.ready.count) == 1) + wake_up(&sc->mr_io.ready.wait_queue); + +done: + atomic_dec(&sc->mr_io.used.count); + +put_kref: + /* + * No kref_put_mutex() as it's already locked. + * + * If smbdirect_mr_io_free_locked() is called + * and the mutex is unlocked and mr is gone, + * in that case kref_put() returned 1. + * + * If kref_put() returned 0 we know that + * smbdirect_mr_io_free_locked() didn't + * run. Not by us nor by anyone else, as we + * still hold the mutex, so we need to unlock + * and keep the mr in SMBDIRECT_MR_READY or + * SMBDIRECT_MR_ERROR state. + */ + if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) + mutex_unlock(&mr->mutex); +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_deregister_mr_io); diff --git a/fs/smb/smbdirect/pdu.h b/fs/smb/smbdirect/pdu.h new file mode 100644 index 000000000000..7693ba337873 --- /dev/null +++ b/fs/smb/smbdirect/pdu.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2017 Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ + +#define SMBDIRECT_V1 0x0100 + +/* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */ +#define SMBDIRECT_MIN_RECEIVE_SIZE 128 +#define SMBDIRECT_MIN_FRAGMENTED_SIZE 131072 + +/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */ +struct smbdirect_negotiate_req { + __le16 min_version; + __le16 max_version; + __le16 reserved; + __le16 credits_requested; + __le32 preferred_send_size; + __le32 max_receive_size; + __le32 max_fragmented_size; +} __packed; + +/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */ +struct smbdirect_negotiate_resp { + __le16 min_version; + __le16 max_version; + __le16 negotiated_version; + __le16 reserved; + __le16 credits_requested; + __le16 credits_granted; + __le32 status; + __le32 max_readwrite_size; + __le32 preferred_send_size; + __le32 max_receive_size; + __le32 max_fragmented_size; +} __packed; + +#define SMBDIRECT_DATA_MIN_HDR_SIZE 0x14 +#define SMBDIRECT_DATA_OFFSET 0x18 + +#define SMBDIRECT_FLAG_RESPONSE_REQUESTED 0x0001 + +/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */ +struct smbdirect_data_transfer { + __le16 credits_requested; + __le16 credits_granted; + __le16 flags; + __le16 reserved; + __le32 remaining_data_length; + __le32 data_offset; + __le32 data_length; + __le32 padding; + __u8 buffer[]; +} __packed; + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ */ diff --git a/fs/smb/smbdirect/public.h b/fs/smb/smbdirect/public.h new file mode 100644 index 000000000000..50088155e7c3 --- /dev/null +++ b/fs/smb/smbdirect/public.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025, Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ + +struct smbdirect_buffer_descriptor_v1; +struct smbdirect_socket_parameters; + +struct smbdirect_socket; +struct smbdirect_send_batch; +struct smbdirect_mr_io; + +#define __SMBDIRECT_EXPORT_SYMBOL__(__sym) EXPORT_SYMBOL_FOR_MODULES(__sym, "cifs,ksmbd") + +#include + +u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev); + +bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs); + +int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc); + +int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc); + +int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, + const struct smbdirect_socket_parameters *sp); + +const struct smbdirect_socket_parameters * +smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc); + +int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, + enum ib_poll_context poll_ctx, + gfp_t gfp_mask); + +#define SMBDIRECT_LOG_ERR 0x0 +#define SMBDIRECT_LOG_INFO 0x1 + +#define SMBDIRECT_LOG_OUTGOING 0x1 +#define SMBDIRECT_LOG_INCOMING 0x2 +#define SMBDIRECT_LOG_READ 0x4 +#define SMBDIRECT_LOG_WRITE 0x8 +#define SMBDIRECT_LOG_RDMA_SEND 0x10 +#define SMBDIRECT_LOG_RDMA_RECV 0x20 +#define SMBDIRECT_LOG_KEEP_ALIVE 0x40 +#define SMBDIRECT_LOG_RDMA_EVENT 0x80 +#define SMBDIRECT_LOG_RDMA_MR 0x100 +#define SMBDIRECT_LOG_RDMA_RW 0x200 +#define SMBDIRECT_LOG_NEGOTIATE 0x400 +void smbdirect_socket_set_logging(struct smbdirect_socket *sc, + void *private_ptr, + bool (*needed)(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls), + void (*vaprintf)(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf)); + +bool smbdirect_connection_is_connected(struct smbdirect_socket *sc); + +int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc); + +int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr); + +void smbdirect_socket_shutdown(struct smbdirect_socket *sc); + +void smbdirect_socket_release(struct smbdirect_socket *sc); + +int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last); + +/* + * This is only temporary and only needed + * as long as the client still requires + * to use smbdirect_connection_send_single_iter() + */ +struct smbdirect_send_batch_storage { + union { + struct list_head __msg_list; + __aligned_u64 __space[5]; + }; +}; + +struct smbdirect_send_batch * +smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, + bool need_invalidate_rkey, + unsigned int remote_key); + +int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct iov_iter *iter, + unsigned int flags, + u32 remaining_data_length); + +int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc); + +int smbdirect_connection_send_iter(struct smbdirect_socket *sc, + struct iov_iter *iter, + unsigned int flags, + bool need_invalidate, + unsigned int remote_key); + +int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, + struct msghdr *msg, + unsigned int flags); + +int smbdirect_connect(struct smbdirect_socket *sc, + const struct sockaddr *dst); + +int smbdirect_connect_sync(struct smbdirect_socket *sc, + const struct sockaddr *dst); + +int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog); + +struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, + long timeo, + struct proto_accept_arg *arg); + +int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, + void *buf, size_t buf_len, + struct smbdirect_buffer_descriptor_v1 *desc, + size_t desc_len, + bool is_read); + +struct smbdirect_mr_io * +smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, + struct iov_iter *iter, + bool writing, + bool need_invalidate); + +void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1); + +void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr); + +void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, + unsigned int rdma_readwrite_threshold, + struct seq_file *m); + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PUBLIC_H__ */ diff --git a/fs/smb/smbdirect/rw.c b/fs/smb/smbdirect/rw.c new file mode 100644 index 000000000000..c2f46b17731e --- /dev/null +++ b/fs/smb/smbdirect/rw.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +static int smbdirect_connection_wait_for_rw_credits(struct smbdirect_socket *sc, + int credits) +{ + return smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->rw_io.credits.wait_queue, + &sc->rw_io.credits.count, + credits); +} + +static int smbdirect_connection_calc_rw_credits(struct smbdirect_socket *sc, + const void *buf, + size_t len) +{ + return DIV_ROUND_UP(smbdirect_get_buf_page_count(buf, len), + sc->rw_io.credits.num_pages); +} + +static int smbdirect_connection_rdma_get_sg_list(void *buf, + size_t size, + struct scatterlist *sg_list, + size_t nentries) +{ + bool high = is_vmalloc_addr(buf); + struct page *page; + size_t offset, len; + int i = 0; + + if (size == 0 || nentries < smbdirect_get_buf_page_count(buf, size)) + return -EINVAL; + + offset = offset_in_page(buf); + buf -= offset; + while (size > 0) { + len = min_t(size_t, PAGE_SIZE - offset, size); + if (high) + page = vmalloc_to_page(buf); + else + page = kmap_to_page(buf); + + if (!sg_list) + return -EINVAL; + sg_set_page(sg_list, page, len, offset); + sg_list = sg_next(sg_list); + + buf += PAGE_SIZE; + size -= len; + offset = 0; + i++; + } + + return i; +} + +static void smbdirect_connection_rw_io_free(struct smbdirect_rw_io *msg, + enum dma_data_direction dir) +{ + struct smbdirect_socket *sc = msg->socket; + + rdma_rw_ctx_destroy(&msg->rdma_ctx, + sc->ib.qp, + sc->ib.qp->port, + msg->sgt.sgl, + msg->sgt.nents, + dir); + sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); + kfree(msg); +} + +static void smbdirect_connection_rdma_rw_done(struct ib_cq *cq, struct ib_wc *wc, + enum dma_data_direction dir) +{ + struct smbdirect_rw_io *msg = + container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe); + struct smbdirect_socket *sc = msg->socket; + + if (wc->status != IB_WC_SUCCESS) { + msg->error = -EIO; + pr_err("read/write error. opcode = %d, status = %s(%d)\n", + wc->opcode, ib_wc_status_msg(wc->status), wc->status); + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_socket_schedule_cleanup(sc, msg->error); + } + + complete(msg->completion); +} + +static void smbdirect_connection_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) +{ + smbdirect_connection_rdma_rw_done(cq, wc, DMA_FROM_DEVICE); +} + +static void smbdirect_connection_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) +{ + smbdirect_connection_rdma_rw_done(cq, wc, DMA_TO_DEVICE); +} + +int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, + void *buf, size_t buf_len, + struct smbdirect_buffer_descriptor_v1 *desc, + size_t desc_len, + bool is_read) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + enum dma_data_direction direction = is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + struct smbdirect_rw_io *msg, *next_msg; + size_t i; + int ret; + DECLARE_COMPLETION_ONSTACK(completion); + struct ib_send_wr *first_wr; + LIST_HEAD(msg_list); + u8 *desc_buf; + int credits_needed; + size_t desc_buf_len, desc_num = 0; + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return -ENOTCONN; + + if (buf_len > sp->max_read_write_size) + return -EINVAL; + + /* calculate needed credits */ + credits_needed = 0; + desc_buf = buf; + for (i = 0; i < desc_len / sizeof(*desc); i++) { + if (!buf_len) + break; + + desc_buf_len = le32_to_cpu(desc[i].length); + if (!desc_buf_len) + return -EINVAL; + + if (desc_buf_len > buf_len) { + desc_buf_len = buf_len; + desc[i].length = cpu_to_le32(desc_buf_len); + buf_len = 0; + } + + credits_needed += smbdirect_connection_calc_rw_credits(sc, + desc_buf, + desc_buf_len); + desc_buf += desc_buf_len; + buf_len -= desc_buf_len; + desc_num++; + } + + smbdirect_log_rdma_rw(sc, SMBDIRECT_LOG_INFO, + "RDMA %s, len %zu, needed credits %d\n", + str_read_write(is_read), buf_len, credits_needed); + + ret = smbdirect_connection_wait_for_rw_credits(sc, credits_needed); + if (ret < 0) + return ret; + + /* build rdma_rw_ctx for each descriptor */ + desc_buf = buf; + for (i = 0; i < desc_num; i++) { + size_t page_count; + + msg = kzalloc_flex(*msg, sg_list, SG_CHUNK_SIZE, + sc->rw_io.mem.gfp_mask); + if (!msg) { + ret = -ENOMEM; + goto out; + } + + desc_buf_len = le32_to_cpu(desc[i].length); + page_count = smbdirect_get_buf_page_count(desc_buf, desc_buf_len); + + msg->socket = sc; + msg->cqe.done = is_read ? + smbdirect_connection_rdma_read_done : + smbdirect_connection_rdma_write_done; + msg->completion = &completion; + + msg->sgt.sgl = &msg->sg_list[0]; + ret = sg_alloc_table_chained(&msg->sgt, + page_count, + msg->sg_list, + SG_CHUNK_SIZE); + if (ret) { + ret = -ENOMEM; + goto free_msg; + } + + ret = smbdirect_connection_rdma_get_sg_list(desc_buf, + desc_buf_len, + msg->sgt.sgl, + msg->sgt.orig_nents); + if (ret < 0) + goto free_table; + + ret = rdma_rw_ctx_init(&msg->rdma_ctx, + sc->ib.qp, + sc->ib.qp->port, + msg->sgt.sgl, + page_count, + 0, + le64_to_cpu(desc[i].offset), + le32_to_cpu(desc[i].token), + direction); + if (ret < 0) { + pr_err("failed to init rdma_rw_ctx: %d\n", ret); + goto free_table; + } + + list_add_tail(&msg->list, &msg_list); + desc_buf += desc_buf_len; + } + + /* concatenate work requests of rdma_rw_ctxs */ + first_wr = NULL; + list_for_each_entry_reverse(msg, &msg_list, list) { + first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, + sc->ib.qp, + sc->ib.qp->port, + &msg->cqe, + first_wr); + } + + ret = ib_post_send(sc->ib.qp, first_wr, NULL); + if (ret) { + pr_err("failed to post send wr for RDMA R/W: %d\n", ret); + goto out; + } + + msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list); + wait_for_completion(&completion); + ret = msg->error; +out: + list_for_each_entry_safe(msg, next_msg, &msg_list, list) { + list_del(&msg->list); + smbdirect_connection_rw_io_free(msg, direction); + } + atomic_add(credits_needed, &sc->rw_io.credits.count); + wake_up(&sc->rw_io.credits.wait_queue); + return ret; + +free_table: + sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); +free_msg: + kfree(msg); + goto out; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_connection_rdma_xmit); diff --git a/fs/smb/smbdirect/smbdirect.h b/fs/smb/smbdirect/smbdirect.h new file mode 100644 index 000000000000..bbab5f7f7cc9 --- /dev/null +++ b/fs/smb/smbdirect/smbdirect.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025 Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ + +#include + +/* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */ +struct smbdirect_buffer_descriptor_v1 { + __le64 offset; + __le32 token; + __le32 length; +} __packed; + +/* + * Connection parameters mostly from [MS-SMBD] 3.1.1.1 + * + * These are setup and negotiated at the beginning of a + * connection and remain constant unless explicitly changed. + * + * Some values are important for the upper layer. + */ +struct smbdirect_socket_parameters { + __u64 flags; +#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1) +#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2) + __u32 resolve_addr_timeout_msec; + __u32 resolve_route_timeout_msec; + __u32 rdma_connect_timeout_msec; + __u32 negotiate_timeout_msec; + __u16 initiator_depth; /* limited to U8_MAX */ + __u16 responder_resources; /* limited to U8_MAX */ + __u16 recv_credit_max; + __u16 send_credit_target; + __u32 max_send_size; + __u32 max_fragmented_send_size; + __u32 max_recv_size; + __u32 max_fragmented_recv_size; + __u32 max_read_write_size; + __u32 max_frmr_depth; + __u32 keepalive_interval_msec; + __u32 keepalive_timeout_msec; +} __packed; + +#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \ + SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \ + SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_H__ */ diff --git a/fs/smb/smbdirect/socket.c b/fs/smb/smbdirect/socket.c new file mode 100644 index 000000000000..1b4ab01b745e --- /dev/null +++ b/fs/smb/smbdirect/socket.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs) +{ + /* + * Test if FRWR (Fast Registration Work Requests) is supported on the + * device This implementation requires FRWR on RDMA read/write return + * value: true if it is supported + */ + + if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) + return false; + if (attrs->max_fast_reg_page_list_len == 0) + return false; + return true; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_frwr_is_supported); + +static void smbdirect_socket_cleanup_work(struct work_struct *work); + +static int smbdirect_socket_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + int ret = -ESTALE; + + /* + * This should be replaced before any real work + * starts! So it should never be called! + */ + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + pr_err("%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + WARN_ONCE(1, "%s should not be called!\n", __func__); + sc->rdma.cm_id = NULL; + return -ESTALE; +} + +int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc) +{ + struct rdma_cm_id *id; + int ret; + + smbdirect_socket_init(sc); + + id = rdma_create_id(net, + smbdirect_socket_rdma_event_handler, + sc, + RDMA_PS_TCP, + IB_QPT_RC); + if (IS_ERR(id)) { + pr_err("%s: rdma_create_id() failed %1pe\n", __func__, id); + return PTR_ERR(id); + } + + ret = rdma_set_afonly(id, 1); + if (ret) { + rdma_destroy_id(id); + pr_err("%s: rdma_set_afonly() failed %1pe\n", + __func__, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + sc->rdma.cm_id = id; + + INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work); + + return 0; +} + +int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc) +{ + struct smbdirect_socket *sc; + int ret; + + ret = -ENOMEM; + sc = kzalloc_obj(*sc); + if (!sc) + goto alloc_failed; + + ret = smbdirect_socket_init_new(net, sc); + if (ret) + goto init_failed; + + kref_init(&sc->refs.destroy); + + *_sc = sc; + return 0; + +init_failed: + kfree(sc); +alloc_failed: + return ret; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_kern); + +int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc) +{ + smbdirect_socket_init(sc); + + sc->rdma.cm_id = id; + sc->rdma.cm_id->context = sc; + sc->rdma.cm_id->event_handler = smbdirect_socket_rdma_event_handler; + + sc->ib.dev = sc->rdma.cm_id->device; + + INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work); + + return 0; +} + +int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc) +{ + struct smbdirect_socket *sc; + int ret; + + ret = -ENOMEM; + sc = kzalloc_obj(*sc); + if (!sc) + goto alloc_failed; + + ret = smbdirect_socket_init_accepting(id, sc); + if (ret) + goto init_failed; + + kref_init(&sc->refs.destroy); + + *_sc = sc; + return 0; + +init_failed: + kfree(sc); +alloc_failed: + return ret; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_create_accepting); + +int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, + const struct smbdirect_socket_parameters *sp) +{ + /* + * This is only allowed before connect or accept + */ + WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + if (sp->flags & ~SMBDIRECT_FLAG_PORT_RANGE_MASK) + return -EINVAL; + + if (sp->initiator_depth > U8_MAX) + return -EINVAL; + if (sp->responder_resources > U8_MAX) + return -EINVAL; + + if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && + sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) + return -EINVAL; + else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB) + rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_IB_CA); + else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) + rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_RNIC); + + /* + * Make a copy of the callers parameters + * from here we only work on the copy + * + * TODO: do we want consistency checking? + */ + sc->parameters = *sp; + + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_initial_parameters); + +const struct smbdirect_socket_parameters * +smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc) +{ + return &sc->parameters; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_get_current_parameters); + +int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, + enum ib_poll_context poll_ctx, + gfp_t gfp_mask) +{ + /* + * This is only allowed before connect or accept + */ + WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + sc->ib.poll_ctx = poll_ctx; + + sc->send_io.mem.gfp_mask = gfp_mask; + sc->recv_io.mem.gfp_mask = gfp_mask; + sc->rw_io.mem.gfp_mask = gfp_mask; + + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_kernel_settings); + +void smbdirect_socket_set_logging(struct smbdirect_socket *sc, + void *private_ptr, + bool (*needed)(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls), + void (*vaprintf)(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf)) +{ + sc->logging.private_ptr = private_ptr; + sc->logging.needed = needed; + sc->logging.vaprintf = vaprintf; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_set_logging); + +static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc) +{ + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + */ + wake_up_all(&sc->status_wait); + wake_up_all(&sc->listen.wait_queue); + wake_up_all(&sc->send_io.bcredits.wait_queue); + wake_up_all(&sc->send_io.lcredits.wait_queue); + wake_up_all(&sc->send_io.credits.wait_queue); + wake_up_all(&sc->send_io.pending.zero_wait_queue); + wake_up_all(&sc->recv_io.reassembly.wait_queue); + wake_up_all(&sc->rw_io.credits.wait_queue); + wake_up_all(&sc->mr_io.ready.wait_queue); +} + +void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, + const char *macro_name, + unsigned int lvl, + const char *func, + unsigned int line, + int error, + enum smbdirect_socket_status *force_status) +{ + struct smbdirect_socket *psc, *tsc; + unsigned long flags; + bool was_first = false; + + if (!sc->first_error) { + ___smbdirect_log_generic(sc, func, line, + lvl, + SMBDIRECT_LOG_RDMA_EVENT, + "%s(%1pe%s%s) called from %s in line=%u status=%s\n", + macro_name, + SMBDIRECT_DEBUG_ERR_PTR(error), + force_status ? ", " : "", + force_status ? smbdirect_socket_status_string(*force_status) : "", + func, line, + smbdirect_socket_status_string(sc->status)); + if (error) + sc->first_error = error; + else + sc->first_error = -ECONNABORTED; + was_first = true; + } + + /* + * make sure other work (than disconnect_work) + * is not queued again but here we don't block and avoid + * disable[_delayed]_work_sync() + */ + disable_work(&sc->connect.work); + disable_work(&sc->recv_io.posted.refill_work); + disable_work(&sc->idle.immediate_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + disable_delayed_work(&sc->idle.timer_work); + + /* + * In case we were a listener we need to + * disconnect all pending and ready sockets + * + * First we move ready sockets to pending again. + */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + + switch (sc->status) { + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: + case SMBDIRECT_SOCKET_ERROR: + case SMBDIRECT_SOCKET_DISCONNECTING: + case SMBDIRECT_SOCKET_DISCONNECTED: + case SMBDIRECT_SOCKET_DESTROYED: + /* + * Keep the current error status + */ + break; + + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; + break; + + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; + break; + + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; + break; + + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; + break; + + case SMBDIRECT_SOCKET_CREATED: + case SMBDIRECT_SOCKET_LISTENING: + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; + break; + + case SMBDIRECT_SOCKET_CONNECTED: + sc->status = SMBDIRECT_SOCKET_ERROR; + break; + } + + if (force_status && (was_first || *force_status > sc->status)) + sc->status = *force_status; + + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + */ + smbdirect_socket_wake_up_all(sc); + + queue_work(sc->workqueues.cleanup, &sc->disconnect_work); +} + +static void smbdirect_socket_cleanup_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, disconnect_work); + struct smbdirect_socket *psc, *tsc; + unsigned long flags; + + /* + * This should not never be called in an interrupt! + */ + WARN_ON_ONCE(in_interrupt()); + + if (!sc->first_error) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s called with first_error==0\n", + smbdirect_socket_status_string(sc->status)); + + sc->first_error = -ECONNABORTED; + } + + /* + * make sure this and other work is not queued again + * but here we don't block and avoid + * disable[_delayed]_work_sync() + */ + disable_work(&sc->disconnect_work); + disable_work(&sc->connect.work); + disable_work(&sc->recv_io.posted.refill_work); + disable_work(&sc->idle.immediate_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + disable_delayed_work(&sc->idle.timer_work); + + /* + * In case we were a listener we need to + * disconnect all pending and ready sockets + * + * First we move ready sockets to pending again. + */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + + switch (sc->status) { + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: + case SMBDIRECT_SOCKET_CONNECTED: + case SMBDIRECT_SOCKET_ERROR: + sc->status = SMBDIRECT_SOCKET_DISCONNECTING; + /* + * Make sure we hold the callback lock + * im order to coordinate with the + * rdma_event handlers, typically + * smbdirect_connection_rdma_event_handler(), + * and smbdirect_socket_destroy(). + * + * So that the order of ib_drain_qp() + * and rdma_disconnect() is controlled + * by the mutex. + */ + rdma_lock_handler(sc->rdma.cm_id); + rdma_disconnect(sc->rdma.cm_id); + rdma_unlock_handler(sc->rdma.cm_id); + break; + + case SMBDIRECT_SOCKET_CREATED: + case SMBDIRECT_SOCKET_LISTENING: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: + /* + * rdma_{accept,connect}() never reached + * RDMA_CM_EVENT_ESTABLISHED + */ + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; + break; + + case SMBDIRECT_SOCKET_DISCONNECTING: + case SMBDIRECT_SOCKET_DISCONNECTED: + case SMBDIRECT_SOCKET_DESTROYED: + break; + } + + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + */ + smbdirect_socket_wake_up_all(sc); +} + +static void smbdirect_socket_destroy(struct smbdirect_socket *sc) +{ + struct smbdirect_socket *psc, *tsc; + size_t psockets; + struct smbdirect_recv_io *recv_io; + struct smbdirect_recv_io *recv_tmp; + LIST_HEAD(all_list); + unsigned long flags; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * This should not never be called in an interrupt! + */ + WARN_ON_ONCE(in_interrupt()); + + if (sc->status == SMBDIRECT_SOCKET_DESTROYED) + return; + + WARN_ONCE(sc->status != SMBDIRECT_SOCKET_DISCONNECTED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * The listener should clear this before we reach this + */ + WARN_ONCE(sc->accept.listener, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + * + * Most likely this was already called via + * smbdirect_socket_cleanup_work(), but call it again... + */ + smbdirect_socket_wake_up_all(sc); + + disable_work_sync(&sc->disconnect_work); + disable_work_sync(&sc->connect.work); + disable_work_sync(&sc->recv_io.posted.refill_work); + disable_work_sync(&sc->idle.immediate_work); + disable_delayed_work_sync(&sc->idle.timer_work); + + if (sc->rdma.cm_id) + rdma_lock_handler(sc->rdma.cm_id); + + if (sc->ib.qp) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "drain qp\n"); + ib_drain_qp(sc->ib.qp); + } + + /* + * In case we were a listener we need to + * disconnect all pending and ready sockets + * + * We move ready sockets to pending again. + */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_tail_init(&sc->listen.ready, &all_list); + list_splice_tail_init(&sc->listen.pending, &all_list); + spin_unlock_irqrestore(&sc->listen.lock, flags); + psockets = list_count_nodes(&all_list); + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "release %zu pending sockets\n", psockets); + list_for_each_entry_safe(psc, tsc, &all_list, accept.list) { + list_del_init(&psc->accept.list); + psc->accept.listener = NULL; + smbdirect_socket_release(psc); + } + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "released %zu pending sockets\n", psockets); + INIT_LIST_HEAD(&all_list); + + /* It's not possible for upper layer to get to reassembly */ + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "drain the reassembly queue\n"); + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + list_splice_tail_init(&sc->recv_io.reassembly.list, &all_list); + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + list_for_each_entry_safe(recv_io, recv_tmp, &all_list, list) + smbdirect_connection_put_recv_io(recv_io); + sc->recv_io.reassembly.data_length = 0; + + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "freeing mr list\n"); + smbdirect_connection_destroy_mr_list(sc); + + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying qp\n"); + smbdirect_connection_destroy_qp(sc); + if (sc->rdma.cm_id) { + rdma_unlock_handler(sc->rdma.cm_id); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying cm_id\n"); + rdma_destroy_id(sc->rdma.cm_id); + sc->rdma.cm_id = NULL; + } + + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying mem pools\n"); + smbdirect_connection_destroy_mem_pools(sc); + + sc->status = SMBDIRECT_SOCKET_DESTROYED; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "rdma session destroyed\n"); +} + +void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc) +{ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * This should not never be called in an interrupt! + */ + WARN_ON_ONCE(in_interrupt()); + + /* + * First we try to disable the work + * without disable_work_sync() in a + * non blocking way, if it's already + * running it will be handles by + * disable_work_sync() below. + * + * Here we just want to make sure queue_work() in + * smbdirect_socket_schedule_cleanup_lvl() + * is a no-op. + */ + disable_work(&sc->disconnect_work); + + if (!sc->first_error) + /* + * SMBDIRECT_LOG_INFO is enough here + * as this is the typical case where + * we terminate the connection ourself. + */ + smbdirect_socket_schedule_cleanup_lvl(sc, + SMBDIRECT_LOG_INFO, + -ESHUTDOWN); + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "cancelling and disable disconnect_work\n"); + disable_work_sync(&sc->disconnect_work); + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying rdma session\n"); + if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) + smbdirect_socket_cleanup_work(&sc->disconnect_work); + if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "wait for transport being disconnected\n"); + wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "waited for transport being disconnected\n"); + } + + /* + * Once we reached SMBDIRECT_SOCKET_DISCONNECTED, + * we should call smbdirect_socket_destroy() + */ + smbdirect_socket_destroy(sc); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); +} + +int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr) +{ + int ret; + + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + ret = rdma_bind_addr(sc->rdma.cm_id, addr); + if (ret) + return ret; + + return 0; +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_bind); + +void smbdirect_socket_shutdown(struct smbdirect_socket *sc) +{ + smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN); +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_shutdown); + +static void smbdirect_socket_release_disconnect(struct kref *kref) +{ + struct smbdirect_socket *sc = + container_of(kref, struct smbdirect_socket, refs.disconnect); + + /* + * For now do a sync disconnect/destroy + */ + smbdirect_socket_destroy_sync(sc); +} + +static void smbdirect_socket_release_destroy(struct kref *kref) +{ + struct smbdirect_socket *sc = + container_of(kref, struct smbdirect_socket, refs.destroy); + + /* + * Do a sync disconnect/destroy... + * hopefully a no-op, as it should be already + * in DESTROYED state, before we free the memory. + */ + smbdirect_socket_destroy_sync(sc); + kfree(sc); +} + +void smbdirect_socket_release(struct smbdirect_socket *sc) +{ + /* + * We expect only 1 disconnect reference + * and if it is already 0, it's a use after free! + */ + WARN_ON_ONCE(kref_read(&sc->refs.disconnect) != 1); + WARN_ON(!kref_put(&sc->refs.disconnect, smbdirect_socket_release_disconnect)); + + /* + * This may not trigger smbdirect_socket_release_destroy(), + * if struct smbdirect_socket is embedded in another structure + * indicated by REFCOUNT_MAX. + */ + kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy); +} +__SMBDIRECT_EXPORT_SYMBOL__(smbdirect_socket_release); + +int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, + enum smbdirect_socket_status expected_status, + int unexpected_errno, + wait_queue_head_t *waitq, + atomic_t *total_credits, + int needed) +{ + int ret; + + if (WARN_ON_ONCE(needed < 0)) + return -EINVAL; + + do { + if (atomic_sub_return(needed, total_credits) >= 0) + return 0; + + atomic_add(needed, total_credits); + ret = wait_event_interruptible(*waitq, + atomic_read(total_credits) >= needed || + sc->status != expected_status); + + if (sc->status != expected_status) + return unexpected_errno; + else if (ret < 0) + return ret; + } while (true); +} diff --git a/fs/smb/smbdirect/socket.h b/fs/smb/smbdirect/socket.h new file mode 100644 index 000000000000..c09eddd8ad16 --- /dev/null +++ b/fs/smb/smbdirect/socket.h @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2025 Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +enum smbdirect_socket_status { + SMBDIRECT_SOCKET_CREATED, + SMBDIRECT_SOCKET_LISTENING, + SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED, + SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, + SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED, + SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED, + SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, + SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED, + SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED, + SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, + SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED, + SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, + SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, + SMBDIRECT_SOCKET_NEGOTIATE_FAILED, + SMBDIRECT_SOCKET_CONNECTED, + SMBDIRECT_SOCKET_ERROR, + SMBDIRECT_SOCKET_DISCONNECTING, + SMBDIRECT_SOCKET_DISCONNECTED, + SMBDIRECT_SOCKET_DESTROYED +}; + +static __always_inline +const char *smbdirect_socket_status_string(enum smbdirect_socket_status status) +{ + switch (status) { + case SMBDIRECT_SOCKET_CREATED: + return "CREATED"; + case SMBDIRECT_SOCKET_LISTENING: + return "LISTENING"; + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: + return "RESOLVE_ADDR_NEEDED"; + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: + return "RESOLVE_ADDR_RUNNING"; + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: + return "RESOLVE_ADDR_FAILED"; + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: + return "RESOLVE_ROUTE_NEEDED"; + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: + return "RESOLVE_ROUTE_RUNNING"; + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: + return "RESOLVE_ROUTE_FAILED"; + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: + return "RDMA_CONNECT_NEEDED"; + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: + return "RDMA_CONNECT_RUNNING"; + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: + return "RDMA_CONNECT_FAILED"; + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: + return "NEGOTIATE_NEEDED"; + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: + return "NEGOTIATE_RUNNING"; + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: + return "NEGOTIATE_FAILED"; + case SMBDIRECT_SOCKET_CONNECTED: + return "CONNECTED"; + case SMBDIRECT_SOCKET_ERROR: + return "ERROR"; + case SMBDIRECT_SOCKET_DISCONNECTING: + return "DISCONNECTING"; + case SMBDIRECT_SOCKET_DISCONNECTED: + return "DISCONNECTED"; + case SMBDIRECT_SOCKET_DESTROYED: + return "DESTROYED"; + } + + return ""; +} + +/* + * This can be used with %1pe to print errors as strings or '0' + * And it avoids warnings like: warn: passing zero to 'ERR_PTR' + * from smatch -p=kernel --pedantic + */ +static __always_inline +const void * __must_check SMBDIRECT_DEBUG_ERR_PTR(long error) +{ + if (error == 0) + return NULL; + return ERR_PTR(error); +} + +enum smbdirect_keepalive_status { + SMBDIRECT_KEEPALIVE_NONE, + SMBDIRECT_KEEPALIVE_PENDING, + SMBDIRECT_KEEPALIVE_SENT +}; + +struct smbdirect_socket { + enum smbdirect_socket_status status; + wait_queue_head_t status_wait; + int first_error; + + /* + * This points to the workqueues to + * be used for this socket. + */ + struct { + struct workqueue_struct *accept; + struct workqueue_struct *connect; + struct workqueue_struct *idle; + struct workqueue_struct *refill; + struct workqueue_struct *immediate; + struct workqueue_struct *cleanup; + } workqueues; + + struct work_struct disconnect_work; + + /* + * The reference counts. + */ + struct { + /* + * This holds the references by the + * frontend, typically the smb layer. + * + * It is typically 1 and a disconnect + * will happen if it reaches 0. + */ + struct kref disconnect; + + /* + * This holds the reference by the + * backend, the code that manages + * the lifetime of the whole + * struct smbdirect_socket, + * if this reaches 0 it can will + * be freed. + * + * Can be REFCOUNT_MAX is part + * of another structure. + * + * This is equal or higher than + * the disconnect refcount. + */ + struct kref destroy; + } refs; + + /* RDMA related */ + struct { + struct rdma_cm_id *cm_id; + /* + * The expected event in our current + * cm_id->event_handler, all other events + * are treated as an error. + */ + enum rdma_cm_event_type expected_event; + /* + * This is for iWarp MPA v1 + */ + bool legacy_iwarp; + } rdma; + + /* IB verbs related */ + struct { + struct ib_pd *pd; + enum ib_poll_context poll_ctx; + struct ib_cq *send_cq; + struct ib_cq *recv_cq; + + /* + * shortcuts for rdma.cm_id->{qp,device}; + */ + struct ib_qp *qp; + struct ib_device *dev; + } ib; + + struct smbdirect_socket_parameters parameters; + + /* + * The state for connect/negotiation + */ + struct { + spinlock_t lock; + struct work_struct work; + } connect; + + /* + * The state for keepalive and timeout handling + */ + struct { + enum smbdirect_keepalive_status keepalive; + struct work_struct immediate_work; + struct delayed_work timer_work; + } idle; + + /* + * The state for listen sockets + */ + struct { + spinlock_t lock; + struct list_head pending; + struct list_head ready; + wait_queue_head_t wait_queue; + /* + * This starts as -1 and a value != -1 + * means this socket was in LISTENING state + * before. Note the valid backlog can + * only be > 0. + */ + int backlog; + } listen; + + /* + * The state for sockets waiting + * for accept, either still waiting + * for the negotiation to finish + * or already ready with a usable + * connection. + */ + struct { + struct smbdirect_socket *listener; + struct list_head list; + } accept; + + /* + * The state for posted send buffers + */ + struct { + /* + * Memory pools for preallocating + * smbdirect_send_io buffers + */ + struct { + struct kmem_cache *cache; + mempool_t *pool; + gfp_t gfp_mask; + } mem; + + /* + * This is a coordination for smbdirect_send_batch. + * + * There's only one possible credit, which means + * only one instance is running at a time. + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } bcredits; + + /* + * The local credit state for ib_post_send() + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } lcredits; + + /* + * The remote credit state for the send side + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } credits; + + /* + * The state about posted/pending sends + */ + struct { + atomic_t count; + /* + * woken when count reached zero + */ + wait_queue_head_t zero_wait_queue; + } pending; + } send_io; + + /* + * The state for posted receive buffers + */ + struct { + /* + * The type of PDU we are expecting + */ + enum { + SMBDIRECT_EXPECT_NEGOTIATE_REQ = 1, + SMBDIRECT_EXPECT_NEGOTIATE_REP = 2, + SMBDIRECT_EXPECT_DATA_TRANSFER = 3, + } expected; + + /* + * Memory pools for preallocating + * smbdirect_recv_io buffers + */ + struct { + struct kmem_cache *cache; + mempool_t *pool; + gfp_t gfp_mask; + } mem; + + /* + * The list of free smbdirect_recv_io + * structures + */ + struct { + struct list_head list; + spinlock_t lock; + } free; + + /* + * The state for posted recv_io messages + * and the refill work struct. + */ + struct { + atomic_t count; + struct work_struct refill_work; + } posted; + + /* + * The credit state for the recv side + */ + struct { + u16 target; + atomic_t available; + atomic_t count; + } credits; + + /* + * The list of arrived non-empty smbdirect_recv_io + * structures + * + * This represents the reassembly queue. + */ + struct { + struct list_head list; + spinlock_t lock; + wait_queue_head_t wait_queue; + /* total data length of reassembly queue */ + int data_length; + int queue_length; + /* the offset to first buffer in reassembly queue */ + int first_entry_offset; + /* + * Indicate if we have received a full packet on the + * connection This is used to identify the first SMBD + * packet of a assembled payload (SMB packet) in + * reassembly queue so we can return a RFC1002 length to + * upper layer to indicate the length of the SMB packet + * received + */ + bool full_packet_received; + } reassembly; + } recv_io; + + /* + * The state for Memory registrations on the client + */ + struct { + enum ib_mr_type type; + + /* + * The list of free smbdirect_mr_io + * structures + */ + struct { + struct list_head list; + spinlock_t lock; + } all; + + /* + * The number of available MRs ready for memory registration + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } ready; + + /* + * The number of used MRs + */ + struct { + atomic_t count; + } used; + } mr_io; + + /* + * The state for RDMA read/write requests on the server + */ + struct { + /* + * Memory hints for + * smbdirect_rw_io structs + */ + struct { + gfp_t gfp_mask; + } mem; + + /* + * The credit state for the send side + */ + struct { + /* + * The maximum number of rw credits + */ + size_t max; + /* + * The number of pages per credit + */ + size_t num_pages; + atomic_t count; + wait_queue_head_t wait_queue; + } credits; + } rw_io; + + /* + * For debug purposes + */ + struct { + u64 get_receive_buffer; + u64 put_receive_buffer; + u64 enqueue_reassembly_queue; + u64 dequeue_reassembly_queue; + u64 send_empty; + } statistics; + + struct { + void *private_ptr; + bool (*needed)(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls); + void (*vaprintf)(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf); + } logging; +}; + +static void __smbdirect_socket_disabled_work(struct work_struct *work) +{ + /* + * Should never be called as disable_[delayed_]work_sync() was used. + */ + WARN_ON_ONCE(1); +} + +static bool __smbdirect_log_needed(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls) +{ + /* + * Should never be called, the caller should + * set it's own functions. + */ + WARN_ON_ONCE(1); + return false; +} + +static void __smbdirect_log_vaprintf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf) +{ + /* + * Should never be called, the caller should + * set it's own functions. + */ + WARN_ON_ONCE(1); +} + +__printf(6, 7) +static void __smbdirect_log_printf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + unsigned int lvl, + unsigned int cls, + const char *fmt, + ...); +__maybe_unused +static void __smbdirect_log_printf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + unsigned int lvl, + unsigned int cls, + const char *fmt, + ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + sc->logging.vaprintf(sc, + func, + line, + sc->logging.private_ptr, + lvl, + cls, + &vaf); + va_end(args); +} + +#define ___smbdirect_log_generic(sc, func, line, lvl, cls, fmt, args...) do { \ + if (sc->logging.needed(sc, sc->logging.private_ptr, lvl, cls)) { \ + __smbdirect_log_printf(sc, func, line, lvl, cls, fmt, ##args); \ + } \ +} while (0) +#define __smbdirect_log_generic(sc, lvl, cls, fmt, args...) \ + ___smbdirect_log_generic(sc, __func__, __LINE__, lvl, cls, fmt, ##args) + +#define smbdirect_log_outgoing(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_OUTGOING, fmt, ##args) +#define smbdirect_log_incoming(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_INCOMING, fmt, ##args) +#define smbdirect_log_read(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_READ, fmt, ##args) +#define smbdirect_log_write(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_WRITE, fmt, ##args) +#define smbdirect_log_rdma_send(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_SEND, fmt, ##args) +#define smbdirect_log_rdma_recv(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RECV, fmt, ##args) +#define smbdirect_log_keep_alive(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_KEEP_ALIVE, fmt, ##args) +#define smbdirect_log_rdma_event(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_EVENT, fmt, ##args) +#define smbdirect_log_rdma_mr(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_MR, fmt, ##args) +#define smbdirect_log_rdma_rw(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RW, fmt, ##args) +#define smbdirect_log_negotiate(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_NEGOTIATE, fmt, ##args) + +static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) +{ + /* + * This also sets status = SMBDIRECT_SOCKET_CREATED + */ + BUILD_BUG_ON(SMBDIRECT_SOCKET_CREATED != 0); + memset(sc, 0, sizeof(*sc)); + + init_waitqueue_head(&sc->status_wait); + + sc->workqueues.accept = smbdirect_globals.workqueues.accept; + sc->workqueues.connect = smbdirect_globals.workqueues.connect; + sc->workqueues.idle = smbdirect_globals.workqueues.idle; + sc->workqueues.refill = smbdirect_globals.workqueues.refill; + sc->workqueues.immediate = smbdirect_globals.workqueues.immediate; + sc->workqueues.cleanup = smbdirect_globals.workqueues.cleanup; + + INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->disconnect_work); + + kref_init(&sc->refs.disconnect); + sc->refs.destroy = (struct kref) KREF_INIT(REFCOUNT_MAX); + + sc->rdma.expected_event = RDMA_CM_EVENT_INTERNAL; + + sc->ib.poll_ctx = IB_POLL_UNBOUND_WORKQUEUE; + + spin_lock_init(&sc->connect.lock); + INIT_WORK(&sc->connect.work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->connect.work); + + INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->idle.immediate_work); + INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); + disable_delayed_work_sync(&sc->idle.timer_work); + + spin_lock_init(&sc->listen.lock); + INIT_LIST_HEAD(&sc->listen.pending); + INIT_LIST_HEAD(&sc->listen.ready); + sc->listen.backlog = -1; /* not a listener */ + init_waitqueue_head(&sc->listen.wait_queue); + + INIT_LIST_HEAD(&sc->accept.list); + + sc->send_io.mem.gfp_mask = GFP_KERNEL; + + atomic_set(&sc->send_io.bcredits.count, 0); + init_waitqueue_head(&sc->send_io.bcredits.wait_queue); + + atomic_set(&sc->send_io.lcredits.count, 0); + init_waitqueue_head(&sc->send_io.lcredits.wait_queue); + + atomic_set(&sc->send_io.credits.count, 0); + init_waitqueue_head(&sc->send_io.credits.wait_queue); + + atomic_set(&sc->send_io.pending.count, 0); + init_waitqueue_head(&sc->send_io.pending.zero_wait_queue); + + sc->recv_io.mem.gfp_mask = GFP_KERNEL; + + INIT_LIST_HEAD(&sc->recv_io.free.list); + spin_lock_init(&sc->recv_io.free.lock); + + atomic_set(&sc->recv_io.posted.count, 0); + INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->recv_io.posted.refill_work); + + atomic_set(&sc->recv_io.credits.available, 0); + atomic_set(&sc->recv_io.credits.count, 0); + + INIT_LIST_HEAD(&sc->recv_io.reassembly.list); + spin_lock_init(&sc->recv_io.reassembly.lock); + init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); + + sc->rw_io.mem.gfp_mask = GFP_KERNEL; + atomic_set(&sc->rw_io.credits.count, 0); + init_waitqueue_head(&sc->rw_io.credits.wait_queue); + + spin_lock_init(&sc->mr_io.all.lock); + INIT_LIST_HEAD(&sc->mr_io.all.list); + atomic_set(&sc->mr_io.ready.count, 0); + init_waitqueue_head(&sc->mr_io.ready.wait_queue); + atomic_set(&sc->mr_io.used.count, 0); + + sc->logging.private_ptr = NULL; + sc->logging.needed = __smbdirect_log_needed; + sc->logging.vaprintf = __smbdirect_log_vaprintf; +} + +#define __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, __error_cmd, __unexpected_cmd) ({ \ + bool __failed = false; \ + if (unlikely((__sc)->first_error)) { \ + __failed = true; \ + __error_cmd \ + } else if (unlikely((__sc)->status != (__expected_status))) { \ + __failed = true; \ + __unexpected_cmd \ + } \ + __failed; \ +}) + +#define __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, __unexpected_cmd) \ + __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, \ + { \ + const struct sockaddr_storage *__src = NULL; \ + const struct sockaddr_storage *__dst = NULL; \ + if ((__sc)->rdma.cm_id) { \ + __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ + __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ + } \ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + }, \ + { \ + const struct sockaddr_storage *__src = NULL; \ + const struct sockaddr_storage *__dst = NULL; \ + if ((__sc)->rdma.cm_id) { \ + __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ + __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ + } \ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + WARN_ONCE(1, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + __unexpected_cmd \ + }) + +#define SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status) \ + __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, /* nothing */) + +#ifndef __SMBDIRECT_SOCKET_DISCONNECT +#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) \ + smbdirect_socket_schedule_cleanup(__sc, -ECONNABORTED) +#endif /* ! __SMBDIRECT_SOCKET_DISCONNECT */ + +#define SMBDIRECT_CHECK_STATUS_DISCONNECT(__sc, __expected_status) \ + __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, \ + __SMBDIRECT_SOCKET_DISCONNECT(__sc);) + +struct smbdirect_send_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * The SGE entries for this work request + * + * The first points to the packet header + */ +#define SMBDIRECT_SEND_IO_MAX_SGE 6 + size_t num_sge; + struct ib_sge sge[SMBDIRECT_SEND_IO_MAX_SGE]; + + /* + * Link to the list of sibling smbdirect_send_io + * messages. + */ + struct list_head sibling_list; + struct ib_send_wr wr; + + /* SMBD packet header follows this structure */ + u8 packet[]; +}; + +struct smbdirect_send_batch { + /* + * List of smbdirect_send_io messages + */ + struct list_head msg_list; + /* + * Number of list entries + */ + size_t wr_cnt; + + /* + * Possible remote key invalidation state + */ + bool need_invalidate_rkey; + u32 remote_key; + + int credit; +}; + +struct smbdirect_recv_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * For now we only use a single SGE + * as we have just one large buffer + * per posted recv. + */ +#define SMBDIRECT_RECV_IO_MAX_SGE 1 + struct ib_sge sge; + + /* Link to free or reassembly list */ + struct list_head list; + + /* Indicate if this is the 1st packet of a payload */ + bool first_segment; + + /* SMBD packet header and payload follows this structure */ + u8 packet[]; +}; + +enum smbdirect_mr_state { + SMBDIRECT_MR_READY, + SMBDIRECT_MR_REGISTERED, + SMBDIRECT_MR_INVALIDATED, + SMBDIRECT_MR_ERROR, + SMBDIRECT_MR_DISABLED +}; + +struct smbdirect_mr_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * We can have up to two references: + * 1. by the connection + * 2. by the registration + */ + struct kref kref; + struct mutex mutex; + + struct list_head list; + + enum smbdirect_mr_state state; + struct ib_mr *mr; + struct sg_table sgt; + enum dma_data_direction dir; + union { + struct ib_reg_wr wr; + struct ib_send_wr inv_wr; + }; + + bool need_invalidate; + struct completion invalidate_done; +}; + +struct smbdirect_rw_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + struct list_head list; + + int error; + struct completion *completion; + + struct rdma_rw_ctx rdma_ctx; + struct sg_table sgt; + struct scatterlist sg_list[]; +}; + +static inline size_t smbdirect_get_buf_page_count(const void *buf, size_t size) +{ + return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - + (uintptr_t)buf / PAGE_SIZE; +} + +/* + * Maximum number of retries on data transfer operations + */ +#define SMBDIRECT_RDMA_CM_RETRY 6 +/* + * No need to retry on Receiver Not Ready since SMB_DIRECT manages credits + */ +#define SMBDIRECT_RDMA_CM_RNR_RETRY 0 + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */ -- cgit v1.2.3