From 55b6dd54c3bcb6edf7ad630a4510759f4b0cf1cd Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 13 Jan 2026 13:37:39 -0500 Subject: nfsd/sunrpc: add svc_rqst->rq_private pointer and remove rq_lease_breaker rq_lease_breaker has always been a NFSv4 specific layering violation in svc_rqst. The reason it's there though is that we need a place that is thread-local, and accessible from the svc_rqst pointer. Add a new rq_private pointer to struct svc_rqst. This is intended for use by the threads that are handling the service. sunrpc code doesn't touch it. In nfsd, define a new struct nfsd_thread_local_info. nfsd declares one of these on the stack and puts a pointer to it in rq_private. Add a new ntli_lease_breaker field to the new struct and convert all of the places that access rq_lease_breaker to use the new field instead. Signed-off-by: Jeff Layton Reviewed-by: Benjamin Coddington Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 4dc14c7a711b..ab8237ba9596 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -175,6 +175,9 @@ static inline unsigned long svc_serv_maxpages(const struct svc_serv *serv) /* * The context of a single thread, including the request currently being * processed. + * + * RPC programs are free to use rq_private to stash thread-local information. + * The sunrpc layer will not access it. */ struct svc_rqst { struct list_head rq_all; /* all threads list */ @@ -251,7 +254,7 @@ struct svc_rqst { unsigned long bc_to_initval; unsigned int bc_to_retries; unsigned int rq_status_counter; /* RPC processing counter */ - void **rq_lease_breaker; /* The v4 client breaking a lease */ + void *rq_private; /* For use by the service thread */ }; /* bits for rq_flags */ -- cgit v1.2.3 From 322ecd01bf8ad7e0da21e174679aff1759e68b2c Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 13 Jan 2026 13:37:40 -0500 Subject: nfsd/sunrpc: move rq_cachetype into struct nfsd_thread_local_info The svc_rqst->rq_cachetype field is only accessed by nfsd. Move it into the nfsd_thread_local_info instead. Signed-off-by: Jeff Layton Reviewed-by: Benjamin Coddington Signed-off-by: Chuck Lever --- fs/nfsd/nfs4xdr.c | 3 ++- fs/nfsd/nfscache.c | 3 ++- fs/nfsd/nfsd.h | 1 + fs/nfsd/nfssvc.c | 5 +++-- include/linux/sunrpc/svc.h | 1 - 5 files changed, 8 insertions(+), 5 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 9d234913100b..690f7a3122ec 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -2598,6 +2598,7 @@ nfsd4_opnum_in_range(struct nfsd4_compoundargs *argp, struct nfsd4_op *op) static bool nfsd4_decode_compound(struct nfsd4_compoundargs *argp) { + struct nfsd_thread_local_info *ntli = argp->rqstp->rq_private; struct nfsd4_op *op; bool cachethis = false; int auth_slack= argp->rqstp->rq_auth_slack; @@ -2690,7 +2691,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp) if (argp->minorversion) cachethis = false; svc_reserve_auth(argp->rqstp, max_reply + readbytes); - argp->rqstp->rq_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE; + ntli->ntli_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE; argp->splice_ok = nfsd_read_splice_ok(argp->rqstp); if (readcount > 1 || max_reply > PAGE_SIZE - auth_slack) diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index ab13ee9c7fd8..154468ceccdc 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -467,10 +467,11 @@ int nfsd_cache_lookup(struct svc_rqst *rqstp, unsigned int start, unsigned int len, struct nfsd_cacherep **cacherep) { struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct nfsd_thread_local_info *ntli = rqstp->rq_private; struct nfsd_cacherep *rp, *found; __wsum csum; struct nfsd_drc_bucket *b; - int type = rqstp->rq_cachetype; + int type = ntli->ntli_cachetype; LIST_HEAD(dispose); int rtn = RC_DOIT; diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 938906c6d10c..a2e35a4fa105 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -84,6 +84,7 @@ extern const struct seq_operations nfs_exports_op; struct nfsd_thread_local_info { struct nfs4_client **ntli_lease_breaker; + int ntli_cachetype; }; /* diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index fd979e5392a1..4f1ab3222a4d 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -972,6 +972,7 @@ nfsd(void *vrqstp) */ int nfsd_dispatch(struct svc_rqst *rqstp) { + struct nfsd_thread_local_info *ntli = rqstp->rq_private; const struct svc_procedure *proc = rqstp->rq_procinfo; __be32 *statp = rqstp->rq_accept_statp; struct nfsd_cacherep *rp; @@ -982,7 +983,7 @@ int nfsd_dispatch(struct svc_rqst *rqstp) * Give the xdr decoder a chance to change this if it wants * (necessary in the NFSv4.0 compound case) */ - rqstp->rq_cachetype = proc->pc_cachetype; + ntli->ntli_cachetype = proc->pc_cachetype; /* * ->pc_decode advances the argument stream past the NFS @@ -1027,7 +1028,7 @@ int nfsd_dispatch(struct svc_rqst *rqstp) */ smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1); - nfsd_cache_update(rqstp, rp, rqstp->rq_cachetype, nfs_reply); + nfsd_cache_update(rqstp, rp, ntli->ntli_cachetype, nfs_reply); out_cached_reply: return 1; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index ab8237ba9596..62152e4f3bcc 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -218,7 +218,6 @@ struct svc_rqst { u32 rq_vers; /* program version */ u32 rq_proc; /* procedure number */ u32 rq_prot; /* IP protocol */ - int rq_cachetype; /* catering to nfsd */ unsigned long rq_flags; /* flags field */ ktime_t rq_qtime; /* enqueue time */ -- cgit v1.2.3 From adcc59114ccd402259c089b0fea24da5e4974563 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 4 Feb 2026 21:21:50 +0100 Subject: sunrpc: Kill RPC_IFDEBUG() RPC_IFDEBUG() is used in only two places. In one the user of the definition is guarded by ifdeffery, in the second one it's implied due to dprintk() usage. Kill the macro and move the ifdeffery to the regular condition with the variable defined inside, while in the second case add the same conditional and move the respective code there. Reviewed-by: Jeff Layton Signed-off-by: Andy Shevchenko Signed-off-by: Chuck Lever --- fs/nfsd/nfsfh.c | 9 ++++++--- include/linux/sunrpc/debug.h | 2 -- net/sunrpc/xprtrdma/svc_rdma_transport.c | 27 ++++++++++++++------------- 3 files changed, 20 insertions(+), 18 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index ed85dd43da18..68b629fbaaeb 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -105,9 +105,12 @@ static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, { /* Check if the request originated from a secure port. */ if (rqstp && !nfsd_originating_port_ok(rqstp, cred, exp)) { - RPC_IFDEBUG(char buf[RPC_MAX_ADDRBUFLEN]); - dprintk("nfsd: request from insecure port %s!\n", - svc_print_addr(rqstp, buf, sizeof(buf))); + if (IS_ENABLED(CONFIG_SUNRPC_DEBUG)) { + char buf[RPC_MAX_ADDRBUFLEN]; + + dprintk("nfsd: request from insecure port %s!\n", + svc_print_addr(rqstp, buf, sizeof(buf))); + } return nfserr_perm; } diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h index eb4bd62df319..93d1a11ffbfb 100644 --- a/include/linux/sunrpc/debug.h +++ b/include/linux/sunrpc/debug.h @@ -49,12 +49,10 @@ do { \ } \ } while (0) -# define RPC_IFDEBUG(x) x #else # define ifdebug(fac) if (0) # define dfprintk(fac, fmt, ...) do {} while (0) # define dfprintk_rcu(fac, fmt, ...) do {} while (0) -# define RPC_IFDEBUG(x) #endif /* diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 9b623849723e..f2d72181a6fe 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -414,7 +414,6 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) struct ib_qp_init_attr qp_attr; struct ib_device *dev; int ret = 0; - RPC_IFDEBUG(struct sockaddr *sap); listen_rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); clear_bit(XPT_CONN, &xprt->xpt_flags); @@ -560,18 +559,20 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) goto errout; } -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) - dprintk("svcrdma: new connection accepted on device %s:\n", dev->name); - sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.src_addr; - dprintk(" local address : %pIS:%u\n", sap, rpc_get_port(sap)); - sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr; - dprintk(" remote address : %pIS:%u\n", sap, rpc_get_port(sap)); - dprintk(" max_sge : %d\n", newxprt->sc_max_send_sges); - dprintk(" sq_depth : %d\n", newxprt->sc_sq_depth); - dprintk(" rdma_rw_ctxs : %d\n", ctxts); - dprintk(" max_requests : %d\n", newxprt->sc_max_requests); - dprintk(" ord : %d\n", conn_param.initiator_depth); -#endif + if (IS_ENABLED(CONFIG_SUNRPC_DEBUG)) { + struct sockaddr *sap; + + dprintk("svcrdma: new connection accepted on device %s:\n", dev->name); + sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.src_addr; + dprintk(" local address : %pIS:%u\n", sap, rpc_get_port(sap)); + sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr; + dprintk(" remote address : %pIS:%u\n", sap, rpc_get_port(sap)); + dprintk(" max_sge : %d\n", newxprt->sc_max_send_sges); + dprintk(" sq_depth : %d\n", newxprt->sc_sq_depth); + dprintk(" rdma_rw_ctxs : %d\n", ctxts); + dprintk(" max_requests : %d\n", newxprt->sc_max_requests); + dprintk(" ord : %d\n", conn_param.initiator_depth); + } return &newxprt->sc_xprt; -- cgit v1.2.3 From 6f57293abb8d087de830dd3f02e66d94b3e59973 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 4 Feb 2026 21:21:51 +0100 Subject: sunrpc: Fix compilation error (`make W=1`) when dprintk() is no-op Clang compiler is not happy about set but unused variables: .../flexfilelayout/flexfilelayoutdev.c:56:9: error: variable 'ret' set but not used [-Werror,-Wunused-but-set-variable] .../flexfilelayout/flexfilelayout.c:1505:6: error: variable 'err' set but not used [-Werror,-Wunused-but-set-variable] .../nfs4proc.c:9244:12: error: variable 'ptr' set but not used [-Werror,-Wunused-but-set-variable] Fix these by forwarding parameters of dprintk() to no_printk(). The positive side-effect is a format-string checker enabled even for the cases when dprintk() is no-op. Fixes: d67ae825a59d ("pnfs/flexfiles: Add the FlexFile Layout Driver") Fixes: fc931582c260 ("nfs41: create_session operation") Acked-by: Geert Uytterhoeven Reviewed-by: Jeff Layton Signed-off-by: Andy Shevchenko Signed-off-by: Chuck Lever --- fs/lockd/svclock.c | 5 +++++ include/linux/sunrpc/debug.h | 8 ++++++-- include/linux/sunrpc/sched.h | 3 --- 3 files changed, 11 insertions(+), 5 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index e687103e42d1..ee23f5802af1 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -74,6 +74,11 @@ static const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) return buf; } +#else +static inline const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) +{ + return "???"; +} #endif /* diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h index 93d1a11ffbfb..ab61bed2f7af 100644 --- a/include/linux/sunrpc/debug.h +++ b/include/linux/sunrpc/debug.h @@ -38,6 +38,8 @@ extern unsigned int nlm_debug; do { \ ifdebug(fac) \ __sunrpc_printk(fmt, ##__VA_ARGS__); \ + else \ + no_printk(fmt, ##__VA_ARGS__); \ } while (0) # define dfprintk_rcu(fac, fmt, ...) \ @@ -46,13 +48,15 @@ do { \ rcu_read_lock(); \ __sunrpc_printk(fmt, ##__VA_ARGS__); \ rcu_read_unlock(); \ + } else { \ + no_printk(fmt, ##__VA_ARGS__); \ } \ } while (0) #else # define ifdebug(fac) if (0) -# define dfprintk(fac, fmt, ...) do {} while (0) -# define dfprintk_rcu(fac, fmt, ...) do {} while (0) +# define dfprintk(fac, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +# define dfprintk_rcu(fac, fmt, ...) no_printk(fmt, ##__VA_ARGS__) #endif /* diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index ccba79ebf893..0dbdf3722537 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -95,10 +95,7 @@ struct rpc_task { int tk_rpc_status; /* Result of last RPC operation */ unsigned short tk_flags; /* misc flags */ unsigned short tk_timeouts; /* maj timeouts */ - -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) unsigned short tk_pid; /* debugging aid */ -#endif unsigned char tk_priority : 2,/* Task priority */ tk_garb_retry : 2, tk_cred_retry : 2; -- cgit v1.2.3 From 5bc37b759ec0cdde2c652a2637d704f2d6306617 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Feb 2026 17:06:53 -0500 Subject: Documentation: Add the RPC language description of NLM version 4 In order to generate source code to encode and decode NLMv4 protocol elements, include a copy of the RPC language description of NLMv4 for xdrgen to process. The language description is an amalgam of RFC 1813 and the Open Group's XNFS specification: https://pubs.opengroup.org/onlinepubs/9629799/chap10.htm The C code committed here was generated from the new nlm4.x file using tools/net/sunrpc/xdrgen/xdrgen. The goals of replacing hand-written XDR functions with ones that are tool-generated are to improve memory safety and make XDR encoding and decoding less brittle to maintain. The xdrgen utility derives both the type definitions and the encode/decode functions directly from protocol specifications, using names and symbols familiar to anyone who knows those specs. Unlike hand-written code that can inadvertently diverge from the specification, xdrgen guarantees that the generated code matches the specification exactly. We would eventually like xdrgen to generate Rust code as well, making the conversion of the kernel's NFS stacks to use Rust just a little easier for us. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- Documentation/sunrpc/xdr/nlm4.x | 211 +++++++++++ fs/lockd/Makefile | 30 +- fs/lockd/nlm4xdr_gen.c | 724 +++++++++++++++++++++++++++++++++++++ fs/lockd/nlm4xdr_gen.h | 32 ++ include/linux/sunrpc/xdrgen/nlm4.h | 233 ++++++++++++ 5 files changed, 1229 insertions(+), 1 deletion(-) create mode 100644 Documentation/sunrpc/xdr/nlm4.x create mode 100644 fs/lockd/nlm4xdr_gen.c create mode 100644 fs/lockd/nlm4xdr_gen.h create mode 100644 include/linux/sunrpc/xdrgen/nlm4.h (limited to 'include/linux/sunrpc') diff --git a/Documentation/sunrpc/xdr/nlm4.x b/Documentation/sunrpc/xdr/nlm4.x new file mode 100644 index 000000000000..0c44a80ef674 --- /dev/null +++ b/Documentation/sunrpc/xdr/nlm4.x @@ -0,0 +1,211 @@ +/* + * This file was extracted by hand from + * https://www.rfc-editor.org/rfc/rfc1813.html . + * + * Note that RFC 1813 is Informational. Its official date of + * publication (June 1995) is before the IETF required its RFCs to + * carry an explicit copyright or other IP ownership notices. + * + * Note also that RFC 1813 does not specify the whole NLM4 protocol. + * In particular, the argument and result types are not present in + * that document, and had to be reverse-engineered. + */ + +/* + * The NLMv4 protocol + */ + +pragma header nlm4; + +/* + * The following definitions are missing in RFC 1813, + * but can be found in the OpenNetworking Network Lock + * Manager protocol: + * + * https://pubs.opengroup.org/onlinepubs/9629799/chap10.htm + */ + +const LM_MAXSTRLEN = 1024; + +const LM_MAXNAMELEN = 1025; + +const MAXNETOBJ_SZ = 1024; + +typedef opaque netobj; + +enum fsh4_mode { + fsm_DN = 0, /* deny none */ + fsm_DR = 1, /* deny read */ + fsm_DW = 2, /* deny write */ + fsm_DRW = 3 /* deny read/write */ +}; + +enum fsh4_access { + fsa_NONE = 0, /* for completeness */ + fsa_R = 1, /* read-only */ + fsa_W = 2, /* write-only */ + fsa_RW = 3 /* read/write */ +}; + +/* + * The following definitions come from the OpenNetworking + * Network Status Monitor protocol: + * + * https://pubs.opengroup.org/onlinepubs/9629799/chap11.htm + */ + +const SM_MAXSTRLEN = 1024; + +/* + * The NLM protocol as extracted from: + * https://tools.ietf.org/html/rfc1813 Appendix II + */ + +typedef unsigned hyper uint64; + +typedef hyper int64; + +typedef unsigned long uint32; + +typedef long int32; + +enum nlm4_stats { + NLM4_GRANTED = 0, + NLM4_DENIED = 1, + NLM4_DENIED_NOLOCKS = 2, + NLM4_BLOCKED = 3, + NLM4_DENIED_GRACE_PERIOD = 4, + NLM4_DEADLCK = 5, + NLM4_ROFS = 6, + NLM4_STALE_FH = 7, + NLM4_FBIG = 8, + NLM4_FAILED = 9 +}; + +pragma big_endian nlm4_stats; + +struct nlm4_holder { + bool exclusive; + int32 svid; + netobj oh; + uint64 l_offset; + uint64 l_len; +}; + +union nlm4_testrply switch (nlm4_stats stat) { + case NLM4_DENIED: + nlm4_holder holder; + default: + void; +}; + +struct nlm4_stat { + nlm4_stats stat; +}; + +struct nlm4_res { + netobj cookie; + nlm4_stat stat; +}; + +struct nlm4_testres { + netobj cookie; + nlm4_testrply stat; +}; + +struct nlm4_lock { + string caller_name; + netobj fh; + netobj oh; + int32 svid; + uint64 l_offset; + uint64 l_len; +}; + +struct nlm4_lockargs { + netobj cookie; + bool block; + bool exclusive; + nlm4_lock alock; + bool reclaim; + int32 state; +}; + +struct nlm4_cancargs { + netobj cookie; + bool block; + bool exclusive; + nlm4_lock alock; +}; + +struct nlm4_testargs { + netobj cookie; + bool exclusive; + nlm4_lock alock; +}; + +struct nlm4_unlockargs { + netobj cookie; + nlm4_lock alock; +}; + +struct nlm4_share { + string caller_name; + netobj fh; + netobj oh; + fsh4_mode mode; + fsh4_access access; +}; + +struct nlm4_shareargs { + netobj cookie; + nlm4_share share; + bool reclaim; +}; + +struct nlm4_shareres { + netobj cookie; + nlm4_stats stat; + int32 sequence; +}; + +struct nlm4_notify { + string name; + int32 state; +}; + +/* + * Argument for the Linux-private SM_NOTIFY procedure + */ +const SM_PRIV_SIZE = 16; + +struct nlm4_notifyargs { + nlm4_notify notify; + opaque private[SM_PRIV_SIZE]; +}; + +program NLM4_PROG { + version NLM4_VERS { + void NLMPROC4_NULL(void) = 0; + nlm4_testres NLMPROC4_TEST(nlm4_testargs) = 1; + nlm4_res NLMPROC4_LOCK(nlm4_lockargs) = 2; + nlm4_res NLMPROC4_CANCEL(nlm4_cancargs) = 3; + nlm4_res NLMPROC4_UNLOCK(nlm4_unlockargs) = 4; + nlm4_res NLMPROC4_GRANTED(nlm4_testargs) = 5; + void NLMPROC4_TEST_MSG(nlm4_testargs) = 6; + void NLMPROC4_LOCK_MSG(nlm4_lockargs) = 7; + void NLMPROC4_CANCEL_MSG(nlm4_cancargs) = 8; + void NLMPROC4_UNLOCK_MSG(nlm4_unlockargs) = 9; + void NLMPROC4_GRANTED_MSG(nlm4_testargs) = 10; + void NLMPROC4_TEST_RES(nlm4_testres) = 11; + void NLMPROC4_LOCK_RES(nlm4_res) = 12; + void NLMPROC4_CANCEL_RES(nlm4_res) = 13; + void NLMPROC4_UNLOCK_RES(nlm4_res) = 14; + void NLMPROC4_GRANTED_RES(nlm4_res) = 15; + void NLMPROC4_SM_NOTIFY(nlm4_notifyargs) = 16; + nlm4_shareres NLMPROC4_SHARE(nlm4_shareargs) = 20; + nlm4_shareres NLMPROC4_UNSHARE(nlm4_shareargs) = 21; + nlm4_res NLMPROC4_NM_LOCK(nlm4_lockargs) = 22; + void NLMPROC4_FREE_ALL(nlm4_notify) = 23; + } = 4; +} = 100021; diff --git a/fs/lockd/Makefile b/fs/lockd/Makefile index 51bbe22d21e3..8e9d18a4348c 100644 --- a/fs/lockd/Makefile +++ b/fs/lockd/Makefile @@ -9,5 +9,33 @@ obj-$(CONFIG_LOCKD) += lockd.o lockd-y := clntlock.o clntproc.o clntxdr.o host.o svc.o svclock.o \ svcshare.o svcproc.o svcsubs.o mon.o trace.o xdr.o netlink.o -lockd-$(CONFIG_LOCKD_V4) += clnt4xdr.o xdr4.o svc4proc.o +lockd-$(CONFIG_LOCKD_V4) += clnt4xdr.o xdr4.o svc4proc.o nlm4xdr_gen.o lockd-$(CONFIG_PROC_FS) += procfs.o + +# +# XDR code generation (requires Python and additional packages) +# +# The generated *xdr_gen.{h,c} files are checked into git. Normal kernel +# builds do not require the xdrgen tool or its Python dependencies. +# +# Developers modifying .x files in Documentation/sunrpc/xdr/ should run +# "make xdrgen" to regenerate the affected files. +# +.PHONY: xdrgen + +XDRGEN = ../../tools/net/sunrpc/xdrgen/xdrgen + +XDRGEN_DEFINITIONS = ../../include/linux/sunrpc/xdrgen/nlm4.h +XDRGEN_DECLARATIONS = nlm4xdr_gen.h +XDRGEN_SOURCE = nlm4xdr_gen.c + +xdrgen: $(XDRGEN_DEFINITIONS) $(XDRGEN_DECLARATIONS) $(XDRGEN_SOURCE) + +../../include/linux/sunrpc/xdrgen/nlm4.h: ../../Documentation/sunrpc/xdr/nlm4.x + $(XDRGEN) definitions $< > $@ + +nlm4xdr_gen.h: ../../Documentation/sunrpc/xdr/nlm4.x + $(XDRGEN) declarations $< > $@ + +nlm4xdr_gen.c: ../../Documentation/sunrpc/xdr/nlm4.x + $(XDRGEN) source --peer server $< > $@ diff --git a/fs/lockd/nlm4xdr_gen.c b/fs/lockd/nlm4xdr_gen.c new file mode 100644 index 000000000000..1c8c221db456 --- /dev/null +++ b/fs/lockd/nlm4xdr_gen.c @@ -0,0 +1,724 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x +// XDR specification modification time: Thu Dec 25 13:10:19 2025 + +#include + +#include "nlm4xdr_gen.h" + +static bool __maybe_unused +xdrgen_decode_netobj(struct xdr_stream *xdr, netobj *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, MAXNETOBJ_SZ); +} + +static bool __maybe_unused +xdrgen_decode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fsh4_access(struct xdr_stream *xdr, fsh4_access *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_uint64(struct xdr_stream *xdr, uint64 *ptr) +{ + return xdrgen_decode_unsigned_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_int64(struct xdr_stream *xdr, int64 *ptr) +{ + return xdrgen_decode_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint32(struct xdr_stream *xdr, uint32 *ptr) +{ + return xdrgen_decode_unsigned_long(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_int32(struct xdr_stream *xdr, int32 *ptr) +{ + return xdrgen_decode_long(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats *ptr) +{ + return xdr_stream_decode_be32(xdr, ptr) == 0; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_holder(struct xdr_stream *xdr, struct nlm4_holder *ptr) +{ + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->svid)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_testrply(struct xdr_stream *xdr, struct nlm4_testrply *ptr) +{ + if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(NLM4_DENIED): + if (!xdrgen_decode_nlm4_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_stat(struct xdr_stream *xdr, struct nlm4_stat *ptr) +{ + if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_res(struct xdr_stream *xdr, struct nlm4_res *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_stat(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_testres(struct xdr_stream *xdr, struct nlm4_testres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_testrply(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_lock(struct xdr_stream *xdr, struct nlm4_lock *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->svid)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_lockargs(struct xdr_stream *xdr, struct nlm4_lockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_cancargs(struct xdr_stream *xdr, struct nlm4_cancargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_testargs(struct xdr_stream *xdr, struct nlm4_testargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_unlockargs(struct xdr_stream *xdr, struct nlm4_unlockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_share(struct xdr_stream *xdr, struct nlm4_share *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_fsh4_mode(xdr, &ptr->mode)) + return false; + if (!xdrgen_decode_fsh4_access(xdr, &ptr->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_shareargs(struct xdr_stream *xdr, struct nlm4_shareargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_share(xdr, &ptr->share)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_shareres(struct xdr_stream *xdr, struct nlm4_shareres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_notify(struct xdr_stream *xdr, struct nlm4_notify *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXNAMELEN)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_notifyargs(struct xdr_stream *xdr, struct nlm4_notifyargs *ptr) +{ + if (!xdrgen_decode_nlm4_notify(xdr, &ptr->notify)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm4_svc_decode_void - Decode a void argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_decode_void(xdr); +} + +/** + * nlm4_svc_decode_nlm4_testargs - Decode a nlm4_testargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_testargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_testargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_lockargs - Decode a nlm4_lockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_lockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_lockargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_cancargs - Decode a nlm4_cancargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_cancargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_cancargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_unlockargs - Decode a nlm4_unlockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_unlockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_unlockargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_testres - Decode a nlm4_testres argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_testres *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_testres(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_res - Decode a nlm4_res argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_res *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_res(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_notifyargs - Decode a nlm4_notifyargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_notifyargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_notifyargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_shareargs - Decode a nlm4_shareargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_shareargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_shareargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_notify - Decode a nlm4_notify argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_notify *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_notify(xdr, argp); +} + +static bool __maybe_unused +xdrgen_encode_netobj(struct xdr_stream *xdr, const netobj value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_fsh4_access(struct xdr_stream *xdr, fsh4_access value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_uint64(struct xdr_stream *xdr, const uint64 value) +{ + return xdrgen_encode_unsigned_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_int64(struct xdr_stream *xdr, const int64 value) +{ + return xdrgen_encode_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint32(struct xdr_stream *xdr, const uint32 value) +{ + return xdrgen_encode_unsigned_long(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_int32(struct xdr_stream *xdr, const int32 value) +{ + return xdrgen_encode_long(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats value) +{ + return xdr_stream_encode_be32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_holder(struct xdr_stream *xdr, const struct nlm4_holder *value) +{ + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_int32(xdr, value->svid)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_testrply(struct xdr_stream *xdr, const struct nlm4_testrply *ptr) +{ + if (!xdrgen_encode_nlm4_stats(xdr, ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(NLM4_DENIED): + if (!xdrgen_encode_nlm4_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_stat(struct xdr_stream *xdr, const struct nlm4_stat *value) +{ + if (!xdrgen_encode_nlm4_stats(xdr, value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_res(struct xdr_stream *xdr, const struct nlm4_res *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_stat(xdr, &value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_testres(struct xdr_stream *xdr, const struct nlm4_testres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_testrply(xdr, &value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_lock(struct xdr_stream *xdr, const struct nlm4_lock *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_int32(xdr, value->svid)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_lockargs(struct xdr_stream *xdr, const struct nlm4_lockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + if (!xdrgen_encode_int32(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_cancargs(struct xdr_stream *xdr, const struct nlm4_cancargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_testargs(struct xdr_stream *xdr, const struct nlm4_testargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_unlockargs(struct xdr_stream *xdr, const struct nlm4_unlockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_share(struct xdr_stream *xdr, const struct nlm4_share *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_fsh4_mode(xdr, value->mode)) + return false; + if (!xdrgen_encode_fsh4_access(xdr, value->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_shareargs(struct xdr_stream *xdr, const struct nlm4_shareargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_share(xdr, &value->share)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_shareres(struct xdr_stream *xdr, const struct nlm4_shareres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_stats(xdr, value->stat)) + return false; + if (!xdrgen_encode_int32(xdr, value->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_notify(struct xdr_stream *xdr, const struct nlm4_notify *value) +{ + if (value->name.len > LM_MAXNAMELEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->name.data, value->name.len) < 0) + return false; + if (!xdrgen_encode_int32(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_notifyargs(struct xdr_stream *xdr, const struct nlm4_notifyargs *value) +{ + if (!xdrgen_encode_nlm4_notify(xdr, &value->notify)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm4_svc_encode_void - Encode a void result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_encode_void(xdr); +} + +/** + * nlm4_svc_encode_nlm4_testres - Encode a nlm4_testres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_testres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm4_testres(xdr, resp); +} + +/** + * nlm4_svc_encode_nlm4_res - Encode a nlm4_res result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_res *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm4_res(xdr, resp); +} + +/** + * nlm4_svc_encode_nlm4_shareres - Encode a nlm4_shareres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_shareres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm4_shareres(xdr, resp); +} diff --git a/fs/lockd/nlm4xdr_gen.h b/fs/lockd/nlm4xdr_gen.h new file mode 100644 index 000000000000..b6008b296a3e --- /dev/null +++ b/fs/lockd/nlm4xdr_gen.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ +/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ + +#ifndef _LINUX_XDRGEN_NLM4_DECL_H +#define _LINUX_XDRGEN_NLM4_DECL_H + +#include + +#include +#include +#include +#include + +bool nlm4_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +bool nlm4_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +#endif /* _LINUX_XDRGEN_NLM4_DECL_H */ diff --git a/include/linux/sunrpc/xdrgen/nlm4.h b/include/linux/sunrpc/xdrgen/nlm4.h new file mode 100644 index 000000000000..e95e8f105624 --- /dev/null +++ b/include/linux/sunrpc/xdrgen/nlm4.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ +/* XDR specification modification time: Thu Dec 25 13:10:19 2025 */ + +#ifndef _LINUX_XDRGEN_NLM4_DEF_H +#define _LINUX_XDRGEN_NLM4_DEF_H + +#include +#include + +enum { LM_MAXSTRLEN = 1024 }; + +enum { LM_MAXNAMELEN = 1025 }; + +enum { MAXNETOBJ_SZ = 1024 }; + +typedef opaque netobj; + +enum fsh4_mode { + fsm_DN = 0, + fsm_DR = 1, + fsm_DW = 2, + fsm_DRW = 3, +}; + +typedef enum fsh4_mode fsh4_mode; + +enum fsh4_access { + fsa_NONE = 0, + fsa_R = 1, + fsa_W = 2, + fsa_RW = 3, +}; + +typedef enum fsh4_access fsh4_access; + +enum { SM_MAXSTRLEN = 1024 }; + +typedef u64 uint64; + +typedef s64 int64; + +typedef u32 uint32; + +typedef s32 int32; + +enum nlm4_stats { + NLM4_GRANTED = 0, + NLM4_DENIED = 1, + NLM4_DENIED_NOLOCKS = 2, + NLM4_BLOCKED = 3, + NLM4_DENIED_GRACE_PERIOD = 4, + NLM4_DEADLCK = 5, + NLM4_ROFS = 6, + NLM4_STALE_FH = 7, + NLM4_FBIG = 8, + NLM4_FAILED = 9, +}; + +typedef __be32 nlm4_stats; + +struct nlm4_holder { + bool exclusive; + int32 svid; + netobj oh; + uint64 l_offset; + uint64 l_len; +}; + +struct nlm4_testrply { + nlm4_stats stat; + union { + struct nlm4_holder holder; + } u; +}; + +struct nlm4_stat { + nlm4_stats stat; +}; + +struct nlm4_res { + netobj cookie; + struct nlm4_stat stat; +}; + +struct nlm4_testres { + netobj cookie; + struct nlm4_testrply stat; +}; + +struct nlm4_lock { + string caller_name; + netobj fh; + netobj oh; + int32 svid; + uint64 l_offset; + uint64 l_len; +}; + +struct nlm4_lockargs { + netobj cookie; + bool block; + bool exclusive; + struct nlm4_lock alock; + bool reclaim; + int32 state; +}; + +struct nlm4_cancargs { + netobj cookie; + bool block; + bool exclusive; + struct nlm4_lock alock; +}; + +struct nlm4_testargs { + netobj cookie; + bool exclusive; + struct nlm4_lock alock; +}; + +struct nlm4_unlockargs { + netobj cookie; + struct nlm4_lock alock; +}; + +struct nlm4_share { + string caller_name; + netobj fh; + netobj oh; + fsh4_mode mode; + fsh4_access access; +}; + +struct nlm4_shareargs { + netobj cookie; + struct nlm4_share share; + bool reclaim; +}; + +struct nlm4_shareres { + netobj cookie; + nlm4_stats stat; + int32 sequence; +}; + +struct nlm4_notify { + string name; + int32 state; +}; + +enum { SM_PRIV_SIZE = 16 }; + +struct nlm4_notifyargs { + struct nlm4_notify notify; + u8 private[SM_PRIV_SIZE]; +}; + +enum { + NLMPROC4_NULL = 0, + NLMPROC4_TEST = 1, + NLMPROC4_LOCK = 2, + NLMPROC4_CANCEL = 3, + NLMPROC4_UNLOCK = 4, + NLMPROC4_GRANTED = 5, + NLMPROC4_TEST_MSG = 6, + NLMPROC4_LOCK_MSG = 7, + NLMPROC4_CANCEL_MSG = 8, + NLMPROC4_UNLOCK_MSG = 9, + NLMPROC4_GRANTED_MSG = 10, + NLMPROC4_TEST_RES = 11, + NLMPROC4_LOCK_RES = 12, + NLMPROC4_CANCEL_RES = 13, + NLMPROC4_UNLOCK_RES = 14, + NLMPROC4_GRANTED_RES = 15, + NLMPROC4_SM_NOTIFY = 16, + NLMPROC4_SHARE = 20, + NLMPROC4_UNSHARE = 21, + NLMPROC4_NM_LOCK = 22, + NLMPROC4_FREE_ALL = 23, +}; + +#ifndef NLM4_PROG +#define NLM4_PROG (100021) +#endif + +#define NLM4_netobj_sz (XDR_unsigned_int + XDR_QUADLEN(MAXNETOBJ_SZ)) +#define NLM4_fsh4_mode_sz (XDR_int) +#define NLM4_fsh4_access_sz (XDR_int) +#define NLM4_uint64_sz \ + (XDR_unsigned_hyper) +#define NLM4_int64_sz \ + (XDR_hyper) +#define NLM4_uint32_sz \ + (XDR_unsigned_long) +#define NLM4_int32_sz \ + (XDR_long) +#define NLM4_nlm4_stats_sz (XDR_int) +#define NLM4_nlm4_holder_sz \ + (XDR_bool + NLM4_int32_sz + NLM4_netobj_sz + NLM4_uint64_sz + NLM4_uint64_sz) +#define NLM4_nlm4_testrply_sz \ + (NLM4_nlm4_stats_sz + NLM4_nlm4_holder_sz) +#define NLM4_nlm4_stat_sz \ + (NLM4_nlm4_stats_sz) +#define NLM4_nlm4_res_sz \ + (NLM4_netobj_sz + NLM4_nlm4_stat_sz) +#define NLM4_nlm4_testres_sz \ + (NLM4_netobj_sz + NLM4_nlm4_testrply_sz) +#define NLM4_nlm4_lock_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM4_netobj_sz + NLM4_netobj_sz + NLM4_int32_sz + NLM4_uint64_sz + NLM4_uint64_sz) +#define NLM4_nlm4_lockargs_sz \ + (NLM4_netobj_sz + XDR_bool + XDR_bool + NLM4_nlm4_lock_sz + XDR_bool + NLM4_int32_sz) +#define NLM4_nlm4_cancargs_sz \ + (NLM4_netobj_sz + XDR_bool + XDR_bool + NLM4_nlm4_lock_sz) +#define NLM4_nlm4_testargs_sz \ + (NLM4_netobj_sz + XDR_bool + NLM4_nlm4_lock_sz) +#define NLM4_nlm4_unlockargs_sz \ + (NLM4_netobj_sz + NLM4_nlm4_lock_sz) +#define NLM4_nlm4_share_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM4_netobj_sz + NLM4_netobj_sz + NLM4_fsh4_mode_sz + NLM4_fsh4_access_sz) +#define NLM4_nlm4_shareargs_sz \ + (NLM4_netobj_sz + NLM4_nlm4_share_sz + XDR_bool) +#define NLM4_nlm4_shareres_sz \ + (NLM4_netobj_sz + NLM4_nlm4_stats_sz + NLM4_int32_sz) +#define NLM4_nlm4_notify_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXNAMELEN) + NLM4_int32_sz) +#define NLM4_nlm4_notifyargs_sz \ + (NLM4_nlm4_notify_sz + XDR_QUADLEN(SM_PRIV_SIZE)) +#define NLM4_MAX_ARGS_SZ \ + (NLM4_nlm4_lockargs_sz) + +#endif /* _LINUX_XDRGEN_NLM4_DEF_H */ -- cgit v1.2.3 From 17c1d66579ff27a7a8f2f407d1425272ff6fdd8c Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 23 Feb 2026 12:09:59 -0500 Subject: sunrpc: convert queue_lock from global spinlock to per-cache-detail lock The global queue_lock serializes all upcall queue operations across every cache_detail instance. Convert it to a per-cache-detail spinlock so that different caches (e.g. auth.unix.ip vs nfsd.fh) no longer contend with each other on queue operations. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/cache.h | 1 + net/sunrpc/cache.c | 47 ++++++++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 24 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h index e783132e481f..3d32dd1f7b05 100644 --- a/include/linux/sunrpc/cache.h +++ b/include/linux/sunrpc/cache.h @@ -113,6 +113,7 @@ struct cache_detail { /* fields for communication over channel */ struct list_head queue; + spinlock_t queue_lock; atomic_t writers; /* how many time is /channel open */ time64_t last_close; /* if no writers, when did last close */ diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 86b3fd5a429d..1cfaae488c6c 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -400,6 +400,7 @@ void sunrpc_init_cache_detail(struct cache_detail *cd) { spin_lock_init(&cd->hash_lock); INIT_LIST_HEAD(&cd->queue); + spin_lock_init(&cd->queue_lock); spin_lock(&cache_list_lock); cd->nextcheck = 0; cd->entries = 0; @@ -803,8 +804,6 @@ void cache_clean_deferred(void *owner) * */ -static DEFINE_SPINLOCK(queue_lock); - struct cache_queue { struct list_head list; int reader; /* if 0, then request */ @@ -847,7 +846,7 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, inode_lock(inode); /* protect against multiple concurrent * readers on this file */ again: - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); /* need to find next request */ while (rp->q.list.next != &cd->queue && list_entry(rp->q.list.next, struct cache_queue, list) @@ -856,7 +855,7 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, list_move(&rp->q.list, next); } if (rp->q.list.next == &cd->queue) { - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); inode_unlock(inode); WARN_ON_ONCE(rp->offset); return 0; @@ -865,7 +864,7 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, WARN_ON_ONCE(rq->q.reader); if (rp->offset == 0) rq->readers++; - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); if (rq->len == 0) { err = cache_request(cd, rq); @@ -876,9 +875,9 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, if (rp->offset == 0 && !test_bit(CACHE_PENDING, &rq->item->flags)) { err = -EAGAIN; - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); list_move(&rp->q.list, &rq->q.list); - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); } else { if (rp->offset + count > rq->len) count = rq->len - rp->offset; @@ -888,26 +887,26 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, rp->offset += count; if (rp->offset >= rq->len) { rp->offset = 0; - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); list_move(&rp->q.list, &rq->q.list); - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); } err = 0; } out: if (rp->offset == 0) { /* need to release rq */ - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); rq->readers--; if (rq->readers == 0 && !test_bit(CACHE_PENDING, &rq->item->flags)) { list_del(&rq->q.list); - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); cache_put(rq->item, cd); kfree(rq->buf); kfree(rq); } else - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); } if (err == -EAGAIN) goto again; @@ -988,7 +987,7 @@ static __poll_t cache_poll(struct file *filp, poll_table *wait, if (!rp) return mask; - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); for (cq= &rp->q; &cq->list != &cd->queue; cq = list_entry(cq->list.next, struct cache_queue, list)) @@ -996,7 +995,7 @@ static __poll_t cache_poll(struct file *filp, poll_table *wait, mask |= EPOLLIN | EPOLLRDNORM; break; } - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); return mask; } @@ -1011,7 +1010,7 @@ static int cache_ioctl(struct inode *ino, struct file *filp, if (cmd != FIONREAD || !rp) return -EINVAL; - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); /* only find the length remaining in current request, * or the length of the next request @@ -1024,7 +1023,7 @@ static int cache_ioctl(struct inode *ino, struct file *filp, len = cr->len - rp->offset; break; } - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); return put_user(len, (int __user *)arg); } @@ -1046,9 +1045,9 @@ static int cache_open(struct inode *inode, struct file *filp, rp->offset = 0; rp->q.reader = 1; - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); list_add(&rp->q.list, &cd->queue); - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); } if (filp->f_mode & FMODE_WRITE) atomic_inc(&cd->writers); @@ -1064,7 +1063,7 @@ static int cache_release(struct inode *inode, struct file *filp, if (rp) { struct cache_request *rq = NULL; - spin_lock(&queue_lock); + spin_lock(&cd->queue_lock); if (rp->offset) { struct cache_queue *cq; for (cq = &rp->q; &cq->list != &cd->queue; @@ -1086,7 +1085,7 @@ static int cache_release(struct inode *inode, struct file *filp, rp->offset = 0; } list_del(&rp->q.list); - spin_unlock(&queue_lock); + spin_unlock(&cd->queue_lock); if (rq) { cache_put(rq->item, cd); @@ -1113,7 +1112,7 @@ static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch) struct cache_request *cr; LIST_HEAD(dequeued); - spin_lock(&queue_lock); + spin_lock(&detail->queue_lock); list_for_each_entry_safe(cq, tmp, &detail->queue, list) if (!cq->reader) { cr = container_of(cq, struct cache_request, q); @@ -1126,7 +1125,7 @@ static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch) continue; list_move(&cr->q.list, &dequeued); } - spin_unlock(&queue_lock); + spin_unlock(&detail->queue_lock); while (!list_empty(&dequeued)) { cr = list_entry(dequeued.next, struct cache_request, q.list); list_del(&cr->q.list); @@ -1251,7 +1250,7 @@ static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h) crq->buf = buf; crq->len = 0; crq->readers = 0; - spin_lock(&queue_lock); + spin_lock(&detail->queue_lock); if (test_bit(CACHE_PENDING, &h->flags)) { crq->item = cache_get(h); list_add_tail(&crq->q.list, &detail->queue); @@ -1259,7 +1258,7 @@ static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h) } else /* Lost a race, no longer PENDING, so don't enqueue */ ret = -EAGAIN; - spin_unlock(&queue_lock); + spin_unlock(&detail->queue_lock); wake_up(&queue_wait); if (ret == -EAGAIN) { kfree(buf); -- cgit v1.2.3 From 552d0e17ea042fc4f959c4543cbbd0e54de7a8e9 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 23 Feb 2026 12:10:00 -0500 Subject: sunrpc: convert queue_wait from global to per-cache-detail waitqueue The queue_wait waitqueue is currently a file-scoped global, so a wake_up for one cache_detail wakes pollers on all caches. Convert it to a per-cache-detail field so that only pollers on the relevant cache are woken. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/cache.h | 2 ++ net/sunrpc/cache.c | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h index 3d32dd1f7b05..031379efba24 100644 --- a/include/linux/sunrpc/cache.h +++ b/include/linux/sunrpc/cache.h @@ -16,6 +16,7 @@ #include #include #include +#include /* * Each cache requires: @@ -114,6 +115,7 @@ struct cache_detail { /* fields for communication over channel */ struct list_head queue; spinlock_t queue_lock; + wait_queue_head_t queue_wait; atomic_t writers; /* how many time is /channel open */ time64_t last_close; /* if no writers, when did last close */ diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 1cfaae488c6c..fd02dca1f07a 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -401,6 +401,7 @@ void sunrpc_init_cache_detail(struct cache_detail *cd) spin_lock_init(&cd->hash_lock); INIT_LIST_HEAD(&cd->queue); spin_lock_init(&cd->queue_lock); + init_waitqueue_head(&cd->queue_wait); spin_lock(&cache_list_lock); cd->nextcheck = 0; cd->entries = 0; @@ -970,8 +971,6 @@ out: return ret; } -static DECLARE_WAIT_QUEUE_HEAD(queue_wait); - static __poll_t cache_poll(struct file *filp, poll_table *wait, struct cache_detail *cd) { @@ -979,7 +978,7 @@ static __poll_t cache_poll(struct file *filp, poll_table *wait, struct cache_reader *rp = filp->private_data; struct cache_queue *cq; - poll_wait(filp, &queue_wait, wait); + poll_wait(filp, &cd->queue_wait, wait); /* alway allow write */ mask = EPOLLOUT | EPOLLWRNORM; @@ -1259,7 +1258,7 @@ static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h) /* Lost a race, no longer PENDING, so don't enqueue */ ret = -EAGAIN; spin_unlock(&detail->queue_lock); - wake_up(&queue_wait); + wake_up(&detail->queue_wait); if (ret == -EAGAIN) { kfree(buf); kfree(crq); -- cgit v1.2.3 From facc4e3c80420e3466003ce09b576e005b56a015 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 23 Feb 2026 12:10:01 -0500 Subject: sunrpc: split cache_detail queue into request and reader lists Replace the single interleaved queue (which mixed cache_request and cache_reader entries distinguished by a ->reader flag) with two dedicated lists: cd->requests for upcall requests and cd->readers for open file handles. Readers now track their position via a monotonically increasing sequence number (next_seqno) rather than by their position in the shared list. Each cache_request is assigned a seqno when enqueued, and a new cache_next_request() helper finds the next request at or after a given seqno. This eliminates the cache_queue wrapper struct entirely, simplifies the reader-skipping loops in cache_read/cache_poll/cache_ioctl/ cache_release, and makes the data flow easier to reason about. Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/cache.h | 4 +- net/sunrpc/cache.c | 143 ++++++++++++++++++------------------------- 2 files changed, 62 insertions(+), 85 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h index 031379efba24..b1e595c2615b 100644 --- a/include/linux/sunrpc/cache.h +++ b/include/linux/sunrpc/cache.h @@ -113,9 +113,11 @@ struct cache_detail { int entries; /* fields for communication over channel */ - struct list_head queue; + struct list_head requests; + struct list_head readers; spinlock_t queue_lock; wait_queue_head_t queue_wait; + u64 next_seqno; atomic_t writers; /* how many time is /channel open */ time64_t last_close; /* if no writers, when did last close */ diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index fd02dca1f07a..7081c1214e6c 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -399,9 +399,11 @@ static struct delayed_work cache_cleaner; void sunrpc_init_cache_detail(struct cache_detail *cd) { spin_lock_init(&cd->hash_lock); - INIT_LIST_HEAD(&cd->queue); + INIT_LIST_HEAD(&cd->requests); + INIT_LIST_HEAD(&cd->readers); spin_lock_init(&cd->queue_lock); init_waitqueue_head(&cd->queue_wait); + cd->next_seqno = 0; spin_lock(&cache_list_lock); cd->nextcheck = 0; cd->entries = 0; @@ -796,29 +798,20 @@ void cache_clean_deferred(void *owner) * On read, you get a full request, or block. * On write, an update request is processed. * Poll works if anything to read, and always allows write. - * - * Implemented by linked list of requests. Each open file has - * a ->private that also exists in this list. New requests are added - * to the end and may wakeup and preceding readers. - * New readers are added to the head. If, on read, an item is found with - * CACHE_UPCALLING clear, we free it from the list. - * */ -struct cache_queue { - struct list_head list; - int reader; /* if 0, then request */ -}; struct cache_request { - struct cache_queue q; + struct list_head list; struct cache_head *item; - char * buf; + char *buf; int len; int readers; + u64 seqno; }; struct cache_reader { - struct cache_queue q; + struct list_head list; int offset; /* if non-0, we have a refcnt on next request */ + u64 next_seqno; }; static int cache_request(struct cache_detail *detail, @@ -833,6 +826,17 @@ static int cache_request(struct cache_detail *detail, return PAGE_SIZE - len; } +static struct cache_request * +cache_next_request(struct cache_detail *cd, u64 seqno) +{ + struct cache_request *rq; + + list_for_each_entry(rq, &cd->requests, list) + if (rq->seqno >= seqno) + return rq; + return NULL; +} + static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos, struct cache_detail *cd) { @@ -849,20 +853,13 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, again: spin_lock(&cd->queue_lock); /* need to find next request */ - while (rp->q.list.next != &cd->queue && - list_entry(rp->q.list.next, struct cache_queue, list) - ->reader) { - struct list_head *next = rp->q.list.next; - list_move(&rp->q.list, next); - } - if (rp->q.list.next == &cd->queue) { + rq = cache_next_request(cd, rp->next_seqno); + if (!rq) { spin_unlock(&cd->queue_lock); inode_unlock(inode); WARN_ON_ONCE(rp->offset); return 0; } - rq = container_of(rp->q.list.next, struct cache_request, q.list); - WARN_ON_ONCE(rq->q.reader); if (rp->offset == 0) rq->readers++; spin_unlock(&cd->queue_lock); @@ -876,9 +873,7 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, if (rp->offset == 0 && !test_bit(CACHE_PENDING, &rq->item->flags)) { err = -EAGAIN; - spin_lock(&cd->queue_lock); - list_move(&rp->q.list, &rq->q.list); - spin_unlock(&cd->queue_lock); + rp->next_seqno = rq->seqno + 1; } else { if (rp->offset + count > rq->len) count = rq->len - rp->offset; @@ -888,9 +883,7 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, rp->offset += count; if (rp->offset >= rq->len) { rp->offset = 0; - spin_lock(&cd->queue_lock); - list_move(&rp->q.list, &rq->q.list); - spin_unlock(&cd->queue_lock); + rp->next_seqno = rq->seqno + 1; } err = 0; } @@ -901,7 +894,7 @@ static ssize_t cache_read(struct file *filp, char __user *buf, size_t count, rq->readers--; if (rq->readers == 0 && !test_bit(CACHE_PENDING, &rq->item->flags)) { - list_del(&rq->q.list); + list_del(&rq->list); spin_unlock(&cd->queue_lock); cache_put(rq->item, cd); kfree(rq->buf); @@ -976,7 +969,6 @@ static __poll_t cache_poll(struct file *filp, poll_table *wait, { __poll_t mask; struct cache_reader *rp = filp->private_data; - struct cache_queue *cq; poll_wait(filp, &cd->queue_wait, wait); @@ -988,12 +980,8 @@ static __poll_t cache_poll(struct file *filp, poll_table *wait, spin_lock(&cd->queue_lock); - for (cq= &rp->q; &cq->list != &cd->queue; - cq = list_entry(cq->list.next, struct cache_queue, list)) - if (!cq->reader) { - mask |= EPOLLIN | EPOLLRDNORM; - break; - } + if (cache_next_request(cd, rp->next_seqno)) + mask |= EPOLLIN | EPOLLRDNORM; spin_unlock(&cd->queue_lock); return mask; } @@ -1004,7 +992,7 @@ static int cache_ioctl(struct inode *ino, struct file *filp, { int len = 0; struct cache_reader *rp = filp->private_data; - struct cache_queue *cq; + struct cache_request *rq; if (cmd != FIONREAD || !rp) return -EINVAL; @@ -1014,14 +1002,9 @@ static int cache_ioctl(struct inode *ino, struct file *filp, /* only find the length remaining in current request, * or the length of the next request */ - for (cq= &rp->q; &cq->list != &cd->queue; - cq = list_entry(cq->list.next, struct cache_queue, list)) - if (!cq->reader) { - struct cache_request *cr = - container_of(cq, struct cache_request, q); - len = cr->len - rp->offset; - break; - } + rq = cache_next_request(cd, rp->next_seqno); + if (rq) + len = rq->len - rp->offset; spin_unlock(&cd->queue_lock); return put_user(len, (int __user *)arg); @@ -1042,10 +1025,10 @@ static int cache_open(struct inode *inode, struct file *filp, return -ENOMEM; } rp->offset = 0; - rp->q.reader = 1; + rp->next_seqno = 0; spin_lock(&cd->queue_lock); - list_add(&rp->q.list, &cd->queue); + list_add(&rp->list, &cd->readers); spin_unlock(&cd->queue_lock); } if (filp->f_mode & FMODE_WRITE) @@ -1064,26 +1047,21 @@ static int cache_release(struct inode *inode, struct file *filp, spin_lock(&cd->queue_lock); if (rp->offset) { - struct cache_queue *cq; - for (cq = &rp->q; &cq->list != &cd->queue; - cq = list_entry(cq->list.next, - struct cache_queue, list)) - if (!cq->reader) { - struct cache_request *cr = - container_of(cq, - struct cache_request, q); - cr->readers--; - if (cr->readers == 0 && - !test_bit(CACHE_PENDING, - &cr->item->flags)) { - list_del(&cr->q.list); - rq = cr; - } - break; + struct cache_request *cr; + + cr = cache_next_request(cd, rp->next_seqno); + if (cr) { + cr->readers--; + if (cr->readers == 0 && + !test_bit(CACHE_PENDING, + &cr->item->flags)) { + list_del(&cr->list); + rq = cr; } + } rp->offset = 0; } - list_del(&rp->q.list); + list_del(&rp->list); spin_unlock(&cd->queue_lock); if (rq) { @@ -1107,27 +1085,24 @@ static int cache_release(struct inode *inode, struct file *filp, static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch) { - struct cache_queue *cq, *tmp; - struct cache_request *cr; + struct cache_request *cr, *tmp; LIST_HEAD(dequeued); spin_lock(&detail->queue_lock); - list_for_each_entry_safe(cq, tmp, &detail->queue, list) - if (!cq->reader) { - cr = container_of(cq, struct cache_request, q); - if (cr->item != ch) - continue; - if (test_bit(CACHE_PENDING, &ch->flags)) - /* Lost a race and it is pending again */ - break; - if (cr->readers != 0) - continue; - list_move(&cr->q.list, &dequeued); - } + list_for_each_entry_safe(cr, tmp, &detail->requests, list) { + if (cr->item != ch) + continue; + if (test_bit(CACHE_PENDING, &ch->flags)) + /* Lost a race and it is pending again */ + break; + if (cr->readers != 0) + continue; + list_move(&cr->list, &dequeued); + } spin_unlock(&detail->queue_lock); while (!list_empty(&dequeued)) { - cr = list_entry(dequeued.next, struct cache_request, q.list); - list_del(&cr->q.list); + cr = list_entry(dequeued.next, struct cache_request, list); + list_del(&cr->list); cache_put(cr->item, detail); kfree(cr->buf); kfree(cr); @@ -1245,14 +1220,14 @@ static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h) return -EAGAIN; } - crq->q.reader = 0; crq->buf = buf; crq->len = 0; crq->readers = 0; spin_lock(&detail->queue_lock); if (test_bit(CACHE_PENDING, &h->flags)) { crq->item = cache_get(h); - list_add_tail(&crq->q.list, &detail->queue); + crq->seqno = detail->next_seqno++; + list_add_tail(&crq->list, &detail->requests); trace_cache_entry_upcall(detail, h); } else /* Lost a race, no longer PENDING, so don't enqueue */ -- cgit v1.2.3 From ee66b9e3e1c69efc986f3932555f07121c3460a7 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 26 Feb 2026 09:47:35 -0500 Subject: SUNRPC: Allocate a separate Reply page array struct svc_rqst uses a single dynamically-allocated page array (rq_pages) for both the incoming RPC Call message and the outgoing RPC Reply message. rq_respages is a sliding pointer into rq_pages that each transport receive path must compute based on how many pages the Call consumed. This boundary tracking is a source of confusion and bugs, and prevents an RPC transaction from having both a large Call and a large Reply simultaneously. Allocate rq_respages as its own page array, eliminating the boundary arithmetic. This decouples Call and Reply buffer lifetimes, following the precedent set by rq_bvec (a separate dynamically- allocated array for I/O vectors). Each svc_rqst now pins twice as many pages as before. For a server running 16 threads with a 1MB maximum payload, the additional cost is roughly 16MB of pinned memory. The new dynamic svc thread count facility keeps this overhead minimal on an idle server. A subsequent patch in this series limits per-request repopulation to only the pages released during the previous RPC, avoiding a full-array scan on each call to svc_alloc_arg(). Note: We've considered several alternatives to maintaining a full second array. Each alternative reintroduces either boundary logic complexity or I/O-path allocation pressure. rq_next_page is initialized in svc_alloc_arg() and svc_process() during Reply construction, and in svc_rdma_recvfrom() as a precaution on error paths. Transport receive paths no longer compute it from the Call size. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 47 ++++++++++++++++----------------- net/sunrpc/svc.c | 29 ++++++++++++++++---- net/sunrpc/svc_xprt.c | 36 ++++++++++++++++++------- net/sunrpc/svcsock.c | 6 ----- net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 15 +++-------- 5 files changed, 77 insertions(+), 56 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 62152e4f3bcc..3b1a98ab5cba 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -134,25 +134,24 @@ enum { extern u32 svc_max_payload(const struct svc_rqst *rqstp); /* - * RPC Requests and replies are stored in one or more pages. - * We maintain an array of pages for each server thread. - * Requests are copied into these pages as they arrive. Remaining - * pages are available to write the reply into. + * RPC Call and Reply messages each have their own page array. + * rq_pages holds the incoming Call message; rq_respages holds + * the outgoing Reply message. Both arrays are sized to + * svc_serv_maxpages() entries and are allocated dynamically. * - * Pages are sent using ->sendmsg with MSG_SPLICE_PAGES so each server thread - * needs to allocate more to replace those used in sending. To help keep track - * of these pages we have a receive list where all pages initialy live, and a - * send list where pages are moved to when there are to be part of a reply. + * Pages are sent using ->sendmsg with MSG_SPLICE_PAGES so each + * server thread needs to allocate more to replace those used in + * sending. * - * We use xdr_buf for holding responses as it fits well with NFS - * read responses (that have a header, and some data pages, and possibly - * a tail) and means we can share some client side routines. + * xdr_buf holds responses; the structure fits NFS read responses + * (header, data pages, optional tail) and enables sharing of + * client-side routines. * - * The xdr_buf.head kvec always points to the first page in the rq_*pages - * list. The xdr_buf.pages pointer points to the second page on that - * list. xdr_buf.tail points to the end of the first page. - * This assumes that the non-page part of an rpc reply will fit - * in a page - NFSd ensures this. lockd also has no trouble. + * The xdr_buf.head kvec always points to the first page in the + * rq_*pages list. The xdr_buf.pages pointer points to the second + * page on that list. xdr_buf.tail points to the end of the first + * page. This assumes that the non-page part of an rpc reply will + * fit in a page - NFSd ensures this. lockd also has no trouble. */ /** @@ -162,10 +161,10 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp); * Returns a count of pages or vectors that can hold the maximum * size RPC message for @serv. * - * Each request/reply pair can have at most one "payload", plus two - * pages, one for the request, and one for the reply. - * nfsd_splice_actor() might need an extra page when a READ payload - * is not page-aligned. + * Each page array can hold at most one payload plus two + * overhead pages (one for the RPC header, one for tail data). + * nfsd_splice_actor() might need an extra page when a READ + * payload is not page-aligned. */ static inline unsigned long svc_serv_maxpages(const struct svc_serv *serv) { @@ -204,11 +203,11 @@ struct svc_rqst { struct xdr_stream rq_res_stream; struct folio *rq_scratch_folio; struct xdr_buf rq_res; - unsigned long rq_maxpages; /* num of entries in rq_pages */ - struct page * *rq_pages; - struct page * *rq_respages; /* points into rq_pages */ + unsigned long rq_maxpages; /* entries per page array */ + struct page * *rq_pages; /* Call buffer pages */ + struct page * *rq_respages; /* Reply buffer pages */ struct page * *rq_next_page; /* next reply page to use */ - struct page * *rq_page_end; /* one past the last page */ + struct page * *rq_page_end; /* one past the last reply page */ struct folio_batch rq_fbatch; struct bio_vec *rq_bvec; diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index f7ec02457328..9abef638b1e0 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -638,13 +638,23 @@ svc_init_buffer(struct svc_rqst *rqstp, const struct svc_serv *serv, int node) { rqstp->rq_maxpages = svc_serv_maxpages(serv); - /* rq_pages' last entry is NULL for historical reasons. */ + /* +1 for a NULL sentinel readable by nfsd_splice_actor() */ rqstp->rq_pages = kcalloc_node(rqstp->rq_maxpages + 1, sizeof(struct page *), GFP_KERNEL, node); if (!rqstp->rq_pages) return false; + /* +1 for a NULL sentinel at rq_page_end (see svc_rqst_replace_page) */ + rqstp->rq_respages = kcalloc_node(rqstp->rq_maxpages + 1, + sizeof(struct page *), + GFP_KERNEL, node); + if (!rqstp->rq_respages) { + kfree(rqstp->rq_pages); + rqstp->rq_pages = NULL; + return false; + } + return true; } @@ -656,10 +666,19 @@ svc_release_buffer(struct svc_rqst *rqstp) { unsigned long i; - for (i = 0; i < rqstp->rq_maxpages; i++) - if (rqstp->rq_pages[i]) - put_page(rqstp->rq_pages[i]); - kfree(rqstp->rq_pages); + if (rqstp->rq_pages) { + for (i = 0; i < rqstp->rq_maxpages; i++) + if (rqstp->rq_pages[i]) + put_page(rqstp->rq_pages[i]); + kfree(rqstp->rq_pages); + } + + if (rqstp->rq_respages) { + for (i = 0; i < rqstp->rq_maxpages; i++) + if (rqstp->rq_respages[i]) + put_page(rqstp->rq_respages[i]); + kfree(rqstp->rq_respages); + } } static void diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 56a663b8939f..e027765f4307 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -650,14 +650,13 @@ static void svc_check_conn_limits(struct svc_serv *serv) } } -static bool svc_alloc_arg(struct svc_rqst *rqstp) +static bool svc_fill_pages(struct svc_rqst *rqstp, struct page **pages, + unsigned long npages) { - struct xdr_buf *arg = &rqstp->rq_arg; - unsigned long pages, filled, ret; + unsigned long filled, ret; - pages = rqstp->rq_maxpages; - for (filled = 0; filled < pages; filled = ret) { - ret = alloc_pages_bulk(GFP_KERNEL, pages, rqstp->rq_pages); + for (filled = 0; filled < npages; filled = ret) { + ret = alloc_pages_bulk(GFP_KERNEL, npages, pages); if (ret > filled) /* Made progress, don't sleep yet */ continue; @@ -667,11 +666,29 @@ static bool svc_alloc_arg(struct svc_rqst *rqstp) set_current_state(TASK_RUNNING); return false; } - trace_svc_alloc_arg_err(pages, ret); + trace_svc_alloc_arg_err(npages, ret); memalloc_retry_wait(GFP_KERNEL); } - rqstp->rq_page_end = &rqstp->rq_pages[pages]; - rqstp->rq_pages[pages] = NULL; /* this might be seen in nfsd_splice_actor() */ + return true; +} + +static bool svc_alloc_arg(struct svc_rqst *rqstp) +{ + struct xdr_buf *arg = &rqstp->rq_arg; + unsigned long pages; + + pages = rqstp->rq_maxpages; + + if (!svc_fill_pages(rqstp, rqstp->rq_pages, pages)) + return false; + if (!svc_fill_pages(rqstp, rqstp->rq_respages, pages)) + return false; + rqstp->rq_next_page = rqstp->rq_respages; + rqstp->rq_page_end = &rqstp->rq_respages[pages]; + /* svc_rqst_replace_page() dereferences *rq_next_page even + * at rq_page_end; NULL prevents releasing a garbage page. + */ + rqstp->rq_page_end[0] = NULL; /* Make arg->head point to first page and arg->pages point to rest */ arg->head[0].iov_base = page_address(rqstp->rq_pages[0]); @@ -1277,7 +1294,6 @@ static noinline int svc_deferred_recv(struct svc_rqst *rqstp) rqstp->rq_addrlen = dr->addrlen; /* Save off transport header len in case we get deferred again */ rqstp->rq_daddr = dr->daddr; - rqstp->rq_respages = rqstp->rq_pages; rqstp->rq_xprt_ctxt = dr->xprt_ctxt; dr->xprt_ctxt = NULL; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index f28c6076f7e8..c86f28f720f7 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -351,8 +351,6 @@ static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen, for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE) bvec_set_page(&bvec[i], rqstp->rq_pages[i], PAGE_SIZE, 0); - rqstp->rq_respages = &rqstp->rq_pages[i]; - rqstp->rq_next_page = rqstp->rq_respages + 1; iov_iter_bvec(&msg.msg_iter, ITER_DEST, bvec, i, buflen); if (seek) { @@ -677,13 +675,9 @@ static int svc_udp_recvfrom(struct svc_rqst *rqstp) if (len <= rqstp->rq_arg.head[0].iov_len) { rqstp->rq_arg.head[0].iov_len = len; rqstp->rq_arg.page_len = 0; - rqstp->rq_respages = rqstp->rq_pages+1; } else { rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len; - rqstp->rq_respages = rqstp->rq_pages + 1 + - DIV_ROUND_UP(rqstp->rq_arg.page_len, PAGE_SIZE); } - rqstp->rq_next_page = rqstp->rq_respages+1; if (serv->sv_stats) serv->sv_stats->netudpcnt++; diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index e7e4a39ca6c6..3081a37a5896 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -861,18 +861,12 @@ static noinline void svc_rdma_read_complete(struct svc_rqst *rqstp, unsigned int i; /* Transfer the Read chunk pages into @rqstp.rq_pages, replacing - * the rq_pages that were already allocated for this rqstp. + * the receive buffer pages already allocated for this rqstp. */ - release_pages(rqstp->rq_respages, ctxt->rc_page_count); + release_pages(rqstp->rq_pages, ctxt->rc_page_count); for (i = 0; i < ctxt->rc_page_count; i++) rqstp->rq_pages[i] = ctxt->rc_pages[i]; - /* Update @rqstp's result send buffer to start after the - * last page in the RDMA Read payload. - */ - rqstp->rq_respages = &rqstp->rq_pages[ctxt->rc_page_count]; - rqstp->rq_next_page = rqstp->rq_respages + 1; - /* Prevent svc_rdma_recv_ctxt_put() from releasing the * pages in ctxt::rc_pages a second time. */ @@ -931,10 +925,9 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp) struct svc_rdma_recv_ctxt *ctxt; int ret; - /* Prevent svc_xprt_release() from releasing pages in rq_pages - * when returning 0 or an error. + /* Precaution: a zero page count on error return causes + * svc_rqst_release_pages() to release nothing. */ - rqstp->rq_respages = rqstp->rq_pages; rqstp->rq_next_page = rqstp->rq_respages; rqstp->rq_xprt_ctxt = NULL; -- cgit v1.2.3 From 7ed7504287a627834f2a35ef04e5dfd26d1c8986 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 26 Feb 2026 09:47:38 -0500 Subject: SUNRPC: Track consumed rq_pages entries The rq_pages array holds pages allocated for incoming RPC requests. Two transport receive paths NULL entries in rq_pages to prevent svc_rqst_release_pages() from freeing pages that the transport has taken ownership of: - svc_tcp_save_pages() moves partial request data pages to svsk->sk_pages during multi-fragment TCP reassembly. - svc_rdma_clear_rqst_pages() moves request data pages to head->rc_pages because they are targets of active RDMA Read WRs. A new rq_pages_nfree field in struct svc_rqst records how many entries were NULLed. svc_alloc_arg() uses it to refill only those entries rather than scanning the full rq_pages array. In steady state, the transport NULLs a handful of entries per RPC, so the allocator visits only those entries instead of the full ~259 slots (for 1MB messages). Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 10 ++++++++++ net/sunrpc/svc.c | 1 + net/sunrpc/svc_xprt.c | 11 ++++++++--- net/sunrpc/svcsock.c | 1 + net/sunrpc/xprtrdma/svc_rdma_rw.c | 1 + 5 files changed, 21 insertions(+), 3 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 3b1a98ab5cba..c3399cf64524 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -143,6 +143,15 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp); * server thread needs to allocate more to replace those used in * sending. * + * rq_pages request page contract: + * + * Transport receive paths that move request data pages out of + * rq_pages -- TCP multi-fragment reassembly (svc_tcp_save_pages) + * and RDMA Read I/O (svc_rdma_clear_rqst_pages) -- NULL those + * entries to prevent svc_rqst_release_pages() from freeing pages + * still in transport use, and set rq_pages_nfree to the count. + * svc_alloc_arg() refills only that many rq_pages entries. + * * xdr_buf holds responses; the structure fits NFS read responses * (header, data pages, optional tail) and enables sharing of * client-side routines. @@ -204,6 +213,7 @@ struct svc_rqst { struct folio *rq_scratch_folio; struct xdr_buf rq_res; unsigned long rq_maxpages; /* entries per page array */ + unsigned long rq_pages_nfree; /* rq_pages entries NULLed by transport */ struct page * *rq_pages; /* Call buffer pages */ struct page * *rq_respages; /* Reply buffer pages */ struct page * *rq_next_page; /* next reply page to use */ diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 0ce16e9abdf6..6e57e35fa6d6 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -655,6 +655,7 @@ svc_init_buffer(struct svc_rqst *rqstp, const struct svc_serv *serv, int node) return false; } + rqstp->rq_pages_nfree = rqstp->rq_maxpages; return true; } diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index e027765f4307..795b5729525f 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -675,12 +675,17 @@ static bool svc_fill_pages(struct svc_rqst *rqstp, struct page **pages, static bool svc_alloc_arg(struct svc_rqst *rqstp) { struct xdr_buf *arg = &rqstp->rq_arg; - unsigned long pages; + unsigned long pages, nfree; pages = rqstp->rq_maxpages; - if (!svc_fill_pages(rqstp, rqstp->rq_pages, pages)) - return false; + nfree = rqstp->rq_pages_nfree; + if (nfree) { + if (!svc_fill_pages(rqstp, rqstp->rq_pages, nfree)) + return false; + rqstp->rq_pages_nfree = 0; + } + if (!svc_fill_pages(rqstp, rqstp->rq_respages, pages)) return false; rqstp->rq_next_page = rqstp->rq_respages; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index c86f28f720f7..2ce43f9995f1 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -1009,6 +1009,7 @@ static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp) svsk->sk_pages[i] = rqstp->rq_pages[i]; rqstp->rq_pages[i] = NULL; } + rqstp->rq_pages_nfree = npages; } static void svc_tcp_clear_pages(struct svc_sock *svsk) diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 4ec2f9ae06aa..cf4a1762b629 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -1107,6 +1107,7 @@ static void svc_rdma_clear_rqst_pages(struct svc_rqst *rqstp, head->rc_pages[i] = rqstp->rq_pages[i]; rqstp->rq_pages[i] = NULL; } + rqstp->rq_pages_nfree = head->rc_page_count; } /** -- cgit v1.2.3 From d7f3efd9ff474867b04e1ea784690f02450a245b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 26 Feb 2026 09:47:39 -0500 Subject: SUNRPC: Optimize rq_respages allocation in svc_alloc_arg svc_alloc_arg() invokes alloc_pages_bulk() with the full rq_maxpages count (~259 for 1MB messages) for the rq_respages array, causing a full-array scan despite most slots holding valid pages. svc_rqst_release_pages() NULLs only the range [rq_respages, rq_next_page) after each RPC, so only that range contains NULL entries. Limit the rq_respages fill in svc_alloc_arg() to that range instead of scanning the full array. svc_init_buffer() initializes rq_next_page to span the entire rq_respages array, so the first svc_alloc_arg() call fills all slots. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 4 ++++ net/sunrpc/svc.c | 1 + net/sunrpc/svc_xprt.c | 8 +++++++- 3 files changed, 12 insertions(+), 1 deletion(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index c3399cf64524..669c944eaf7f 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -152,6 +152,10 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp); * still in transport use, and set rq_pages_nfree to the count. * svc_alloc_arg() refills only that many rq_pages entries. * + * For rq_respages, svc_rqst_release_pages() NULLs entries in + * [rq_respages, rq_next_page) after each RPC. svc_alloc_arg() + * refills only that range. + * * xdr_buf holds responses; the structure fits NFS read responses * (header, data pages, optional tail) and enables sharing of * client-side routines. diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 6e57e35fa6d6..5e0b5ec2fd52 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -656,6 +656,7 @@ svc_init_buffer(struct svc_rqst *rqstp, const struct svc_serv *serv, int node) } rqstp->rq_pages_nfree = rqstp->rq_maxpages; + rqstp->rq_next_page = rqstp->rq_respages + rqstp->rq_maxpages; return true; } diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 795b5729525f..b16e710926c1 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -686,8 +686,14 @@ static bool svc_alloc_arg(struct svc_rqst *rqstp) rqstp->rq_pages_nfree = 0; } - if (!svc_fill_pages(rqstp, rqstp->rq_respages, pages)) + if (WARN_ON_ONCE(rqstp->rq_next_page < rqstp->rq_respages)) return false; + nfree = rqstp->rq_next_page - rqstp->rq_respages; + if (nfree) { + if (!svc_fill_pages(rqstp, rqstp->rq_respages, nfree)) + return false; + } + rqstp->rq_next_page = rqstp->rq_respages; rqstp->rq_page_end = &rqstp->rq_respages[pages]; /* svc_rqst_replace_page() dereferences *rq_next_page even -- cgit v1.2.3 From ccc89b9d1ed233349cfe8d87b842e7351b74d8de Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 27 Feb 2026 09:03:28 -0500 Subject: svcrdma: Add fair queuing for Send Queue access When the Send Queue fills, multiple threads may wait for SQ slots. The previous implementation had no ordering guarantee, allowing starvation when one thread repeatedly acquires slots while others wait indefinitely. Introduce a ticket-based fair queuing system. Each waiter takes a ticket number and is served in FIFO order. This ensures forward progress for all waiters when SQ capacity is constrained. The implementation has two phases: 1. Fast path: attempt to reserve SQ slots without waiting 2. Slow path: take a ticket, wait for turn, then wait for slots The ticket system adds two atomic counters to the transport: - sc_sq_ticket_head: next ticket to issue - sc_sq_ticket_tail: ticket currently being served A dedicated wait queue (sc_sq_ticket_wait) handles ticket ordering, separate from sc_send_wait which handles SQ capacity. This separation ensures that send completions (the high-frequency wake source) wake only the current ticket holder rather than all queued waiters. Ticket handoff wakes only the ticket wait queue, and each ticket holder that exits via connection close propagates the wake to the next waiter in line. When a waiter successfully reserves slots, it advances the tail counter and wakes the next waiter. This creates an orderly handoff that prevents starvation while maintaining good throughput on the fast path when contention is low. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 10 ++ net/sunrpc/xprtrdma/svc_rdma_rw.c | 37 ++----- net/sunrpc/xprtrdma/svc_rdma_sendto.c | 162 +++++++++++++++++++++++-------- net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +- 4 files changed, 146 insertions(+), 69 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 57f4fd94166a..658b8498177e 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -84,6 +84,9 @@ struct svcxprt_rdma { atomic_t sc_sq_avail; /* SQEs ready to be consumed */ unsigned int sc_sq_depth; /* Depth of SQ */ + atomic_t sc_sq_ticket_head; /* Next ticket to issue */ + atomic_t sc_sq_ticket_tail; /* Ticket currently serving */ + wait_queue_head_t sc_sq_ticket_wait; /* Ticket ordering waitlist */ __be32 sc_fc_credits; /* Forward credits */ u32 sc_max_requests; /* Max requests */ u32 sc_max_bc_requests;/* Backward credits */ @@ -306,6 +309,13 @@ extern void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma, struct svc_rdma_recv_ctxt *rctxt, int status); extern void svc_rdma_wake_send_waiters(struct svcxprt_rdma *rdma, int avail); +extern int svc_rdma_sq_wait(struct svcxprt_rdma *rdma, + const struct rpc_rdma_cid *cid, int sqecount); +extern int svc_rdma_post_send_err(struct svcxprt_rdma *rdma, + const struct rpc_rdma_cid *cid, + const struct ib_send_wr *bad_wr, + const struct ib_send_wr *first_wr, + int sqecount, int ret); extern int svc_rdma_sendto(struct svc_rqst *); extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset, unsigned int length); diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index cf4a1762b629..97bce806974b 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -405,34 +405,17 @@ static int svc_rdma_post_chunk_ctxt(struct svcxprt_rdma *rdma, cqe = NULL; } - do { - if (atomic_sub_return(cc->cc_sqecount, - &rdma->sc_sq_avail) > 0) { - cc->cc_posttime = ktime_get(); - ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); - if (ret) - break; - return 0; - } - - percpu_counter_inc(&svcrdma_stat_sq_starve); - trace_svcrdma_sq_full(rdma, &cc->cc_cid); - atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); - wait_event(rdma->sc_send_wait, - atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount); - trace_svcrdma_sq_retry(rdma, &cc->cc_cid); - } while (1); - - trace_svcrdma_sq_post_err(rdma, &cc->cc_cid, ret); - svc_xprt_deferred_close(&rdma->sc_xprt); - - /* If even one was posted, there will be a completion. */ - if (bad_wr != first_wr) - return 0; + ret = svc_rdma_sq_wait(rdma, &cc->cc_cid, cc->cc_sqecount); + if (ret < 0) + return ret; - atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); - wake_up(&rdma->sc_send_wait); - return -ENOTCONN; + cc->cc_posttime = ktime_get(); + ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); + if (ret) + return svc_rdma_post_send_err(rdma, &cc->cc_cid, bad_wr, + first_wr, cc->cc_sqecount, + ret); + return 0; } /* Build a bvec that covers one kvec in an xdr_buf. diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 17c8429da9d5..02559947272a 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -294,6 +294,117 @@ void svc_rdma_wake_send_waiters(struct svcxprt_rdma *rdma, int avail) wake_up(&rdma->sc_send_wait); } +/** + * svc_rdma_sq_wait - Wait for SQ slots using fair queuing + * @rdma: controlling transport + * @cid: completion ID for tracing + * @sqecount: number of SQ entries needed + * + * A ticket-based system ensures fair ordering when multiple threads + * wait for Send Queue capacity. Each waiter takes a ticket and is + * served in order, preventing starvation. + * + * Protocol invariant: every ticket holder must increment + * sc_sq_ticket_tail exactly once, whether the reservation + * succeeds or the connection closes. Failing to advance the + * tail stalls all subsequent waiters. + * + * The ticket counters are signed 32-bit atomics. After + * wrapping through INT_MAX, the equality check + * (tail == ticket) remains correct because both counters + * advance monotonically and the comparison uses exact + * equality rather than relational operators. + * + * Return values: + * %0: SQ slots were reserved successfully + * %-ENOTCONN: The connection was lost + */ +int svc_rdma_sq_wait(struct svcxprt_rdma *rdma, + const struct rpc_rdma_cid *cid, int sqecount) +{ + int ticket; + + /* Fast path: try to reserve SQ slots without waiting. + * + * A failed reservation temporarily understates sc_sq_avail + * until the compensating atomic_add restores it. A Send + * completion arriving in that window sees a lower count + * than reality, but the value self-corrects once the add + * completes. No ordering guarantee is needed here because + * the slow path serializes all contended waiters. + */ + if (likely(atomic_sub_return(sqecount, &rdma->sc_sq_avail) >= 0)) + return 0; + atomic_add(sqecount, &rdma->sc_sq_avail); + + /* Slow path: take a ticket and wait in line */ + ticket = atomic_fetch_inc(&rdma->sc_sq_ticket_head); + + percpu_counter_inc(&svcrdma_stat_sq_starve); + trace_svcrdma_sq_full(rdma, cid); + + /* Wait until all earlier tickets have been served */ + wait_event(rdma->sc_sq_ticket_wait, + test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags) || + atomic_read(&rdma->sc_sq_ticket_tail) == ticket); + if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags)) + goto out_close; + + /* It's our turn. Wait for enough SQ slots to be available. */ + while (atomic_sub_return(sqecount, &rdma->sc_sq_avail) < 0) { + atomic_add(sqecount, &rdma->sc_sq_avail); + + wait_event(rdma->sc_send_wait, + test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags) || + atomic_read(&rdma->sc_sq_avail) >= sqecount); + if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags)) + goto out_close; + } + + /* Slots reserved successfully. Let the next waiter proceed. */ + atomic_inc(&rdma->sc_sq_ticket_tail); + wake_up(&rdma->sc_sq_ticket_wait); + trace_svcrdma_sq_retry(rdma, cid); + return 0; + +out_close: + atomic_inc(&rdma->sc_sq_ticket_tail); + wake_up(&rdma->sc_sq_ticket_wait); + return -ENOTCONN; +} + +/** + * svc_rdma_post_send_err - Handle ib_post_send failure + * @rdma: controlling transport + * @cid: completion ID for tracing + * @bad_wr: first WR that was not posted + * @first_wr: first WR in the chain + * @sqecount: number of SQ entries that were reserved + * @ret: error code from ib_post_send + * + * Return values: + * %0: At least one WR was posted; a completion handles cleanup + * %-ENOTCONN: No WRs were posted; SQ slots are released + */ +int svc_rdma_post_send_err(struct svcxprt_rdma *rdma, + const struct rpc_rdma_cid *cid, + const struct ib_send_wr *bad_wr, + const struct ib_send_wr *first_wr, + int sqecount, int ret) +{ + trace_svcrdma_sq_post_err(rdma, cid, ret); + svc_xprt_deferred_close(&rdma->sc_xprt); + + /* If even one WR was posted, a Send completion will + * return the reserved SQ slots. + */ + if (bad_wr != first_wr) + return 0; + + svc_rdma_wake_send_waiters(rdma, sqecount); + return -ENOTCONN; +} + /** * svc_rdma_wc_send - Invoked by RDMA provider for each polled Send WC * @cq: Completion Queue context @@ -336,11 +447,6 @@ flushed: * that these values remain available after the ib_post_send() call. * In some error flow cases, svc_rdma_wc_send() releases @ctxt. * - * Note there is potential for starvation when the Send Queue is - * full because there is no order to when waiting threads are - * awoken. The transport is typically provisioned with a deep - * enough Send Queue that SQ exhaustion should be a rare event. - * * Return values: * %0: @ctxt's WR chain was posted successfully * %-ENOTCONN: The connection was lost @@ -362,42 +468,16 @@ int svc_rdma_post_send(struct svcxprt_rdma *rdma, send_wr->sg_list[0].length, DMA_TO_DEVICE); - /* If the SQ is full, wait until an SQ entry is available */ - while (!test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags)) { - if (atomic_sub_return(sqecount, &rdma->sc_sq_avail) < 0) { - svc_rdma_wake_send_waiters(rdma, sqecount); - - /* When the transport is torn down, assume - * ib_drain_sq() will trigger enough Send - * completions to wake us. The XPT_CLOSE test - * above should then cause the while loop to - * exit. - */ - percpu_counter_inc(&svcrdma_stat_sq_starve); - trace_svcrdma_sq_full(rdma, &cid); - wait_event(rdma->sc_send_wait, - atomic_read(&rdma->sc_sq_avail) > 0); - trace_svcrdma_sq_retry(rdma, &cid); - continue; - } - - trace_svcrdma_post_send(ctxt); - ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); - if (ret) { - trace_svcrdma_sq_post_err(rdma, &cid, ret); - svc_xprt_deferred_close(&rdma->sc_xprt); - - /* If even one WR was posted, there will be a - * Send completion that bumps sc_sq_avail. - */ - if (bad_wr == first_wr) { - svc_rdma_wake_send_waiters(rdma, sqecount); - break; - } - } - return 0; - } - return -ENOTCONN; + ret = svc_rdma_sq_wait(rdma, &cid, sqecount); + if (ret < 0) + return ret; + + trace_svcrdma_post_send(ctxt); + ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); + if (ret) + return svc_rdma_post_send_err(rdma, &cid, bad_wr, + first_wr, sqecount, ret); + return 0; } /** diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index f2d72181a6fe..f18bc60d9f4f 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -179,6 +179,7 @@ static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv, init_llist_head(&cma_xprt->sc_recv_ctxts); init_llist_head(&cma_xprt->sc_rw_ctxts); init_waitqueue_head(&cma_xprt->sc_send_wait); + init_waitqueue_head(&cma_xprt->sc_sq_ticket_wait); spin_lock_init(&cma_xprt->sc_lock); spin_lock_init(&cma_xprt->sc_rq_dto_lock); @@ -477,6 +478,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) newxprt->sc_sq_depth = dev->attrs.max_qp_wr; atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth); + atomic_set(&newxprt->sc_sq_ticket_head, 0); + atomic_set(&newxprt->sc_sq_ticket_tail, 0); newxprt->sc_pd = ib_alloc_pd(dev, 0); if (IS_ERR(newxprt->sc_pd)) { @@ -649,7 +652,8 @@ static int svc_rdma_has_wspace(struct svc_xprt *xprt) * If there are already waiters on the SQ, * return false. */ - if (waitqueue_active(&rdma->sc_send_wait)) + if (waitqueue_active(&rdma->sc_send_wait) || + waitqueue_active(&rdma->sc_sq_ticket_wait)) return 0; /* Otherwise return true. */ -- cgit v1.2.3 From d16f060f3ee297424c0aba047b1d49208adb9318 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 27 Feb 2026 09:03:31 -0500 Subject: svcrdma: Add Write chunk WRs to the RPC's Send WR chain Previously, Write chunk RDMA Writes were posted via a separate ib_post_send() call with their own completion handler. Each Write chunk incurred a doorbell and generated a completion event. Link Write chunk WRs onto the RPC Reply's Send WR chain so that a single ib_post_send() call posts both the RDMA Writes and the Send WR. A single completion event signals that all operations have finished. This reduces both doorbell rate and completion rate, as well as eliminating the latency of a round-trip between the Write chunk completion and the subsequent Send WR posting. The lifecycle of Write chunk resources changes: previously, the svc_rdma_write_done() completion handler released Write chunk resources when RDMA Writes completed. With WR chaining, resources remain live until the Send completion. A new sc_write_info_list tracks Write chunk metadata attached to each Send context, and svc_rdma_write_chunk_release() frees these resources when the Send context is released. The svc_rdma_write_done() handler now handles only error cases. On success it returns immediately since the Send completion handles resource release. On failure (WR flush), it closes the connection to signal to the client that the RPC Reply is incomplete. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 13 +++-- net/sunrpc/xprtrdma/svc_rdma_rw.c | 94 +++++++++++++++++++++++++++-------- net/sunrpc/xprtrdma/svc_rdma_sendto.c | 10 +++- 3 files changed, 91 insertions(+), 26 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 658b8498177e..df6e08aaad57 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -216,6 +216,7 @@ struct svc_rdma_recv_ctxt { */ struct svc_rdma_write_info { struct svcxprt_rdma *wi_rdma; + struct list_head wi_list; const struct svc_rdma_chunk *wi_chunk; @@ -244,7 +245,10 @@ struct svc_rdma_send_ctxt { struct ib_cqe sc_cqe; struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; + + struct list_head sc_write_info_list; struct svc_rdma_write_info sc_reply_info; + void *sc_xprt_buf; int sc_page_count; int sc_cur_sge_no; @@ -277,11 +281,14 @@ extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma, extern void svc_rdma_cc_release(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc, enum dma_data_direction dir); +extern void svc_rdma_write_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt); extern void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt); -extern int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - const struct xdr_buf *xdr); +extern int svc_rdma_prepare_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_recv_ctxt *rctxt, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr); extern int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_pcl *write_pcl, const struct svc_rdma_pcl *reply_pcl, diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 97bce806974b..ebc90c12c835 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -251,6 +251,28 @@ static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) queue_work(svcrdma_wq, &info->wi_work); } +/** + * svc_rdma_write_chunk_release - Release Write chunk I/O resources + * @rdma: controlling transport + * @ctxt: Send context that is being released + * + * Write chunk resources remain live until Send completion because + * Write WRs are chained to the Send WR. This function releases all + * write_info structures accumulated on @ctxt->sc_write_info_list. + */ +void svc_rdma_write_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt) +{ + struct svc_rdma_write_info *info; + + while (!list_empty(&ctxt->sc_write_info_list)) { + info = list_first_entry(&ctxt->sc_write_info_list, + struct svc_rdma_write_info, wi_list); + list_del(&info->wi_list); + svc_rdma_write_info_free(info); + } +} + /** * svc_rdma_reply_chunk_release - Release Reply chunk I/O resources * @rdma: controlling transport @@ -307,13 +329,11 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) struct ib_cqe *cqe = wc->wr_cqe; struct svc_rdma_chunk_ctxt *cc = container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); - struct svc_rdma_write_info *info = - container_of(cc, struct svc_rdma_write_info, wi_cc); switch (wc->status) { case IB_WC_SUCCESS: trace_svcrdma_wc_write(&cc->cc_cid); - break; + return; case IB_WC_WR_FLUSH_ERR: trace_svcrdma_wc_write_flush(wc, &cc->cc_cid); break; @@ -321,12 +341,11 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) trace_svcrdma_wc_write_err(wc, &cc->cc_cid); } - svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); - - if (unlikely(wc->status != IB_WC_SUCCESS)) - svc_xprt_deferred_close(&rdma->sc_xprt); - - svc_rdma_write_info_free(info); + /* The RDMA Write has flushed, so the client won't get + * some of the outgoing RPC message. Signal the loss + * to the client by closing the connection. + */ + svc_xprt_deferred_close(&rdma->sc_xprt); } /** @@ -600,13 +619,27 @@ static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) return xdr->len; } -static int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, - const struct svc_rdma_chunk *chunk, - const struct xdr_buf *xdr) +/* + * svc_rdma_prepare_write_chunk - Link Write WRs for @chunk onto @sctxt's chain + * + * Write WRs are prepended to the Send WR chain so that a single + * ib_post_send() posts both RDMA Writes and the final Send. Only + * the first WR in each chunk gets a CQE for error detection; + * subsequent WRs complete without individual completion events. + * The Send WR's signaled completion indicates all chained + * operations have finished. + */ +static int svc_rdma_prepare_write_chunk(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *sctxt, + const struct svc_rdma_chunk *chunk, + const struct xdr_buf *xdr) { struct svc_rdma_write_info *info; struct svc_rdma_chunk_ctxt *cc; + struct ib_send_wr *first_wr; struct xdr_buf payload; + struct list_head *pos; + struct ib_cqe *cqe; int ret; if (xdr_buf_subsegment(xdr, &payload, chunk->ch_position, @@ -622,10 +655,25 @@ static int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, if (ret != payload.len) goto out_err; - trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); - ret = svc_rdma_post_chunk_ctxt(rdma, cc); - if (ret < 0) + ret = -EINVAL; + if (unlikely(sctxt->sc_sqecount + cc->cc_sqecount > rdma->sc_sq_depth)) goto out_err; + + first_wr = sctxt->sc_wr_chain; + cqe = &cc->cc_cqe; + list_for_each(pos, &cc->cc_rwctxts) { + struct svc_rdma_rw_ctxt *rwc; + + rwc = list_entry(pos, struct svc_rdma_rw_ctxt, rw_list); + first_wr = rdma_rw_ctx_wrs(&rwc->rw_ctx, rdma->sc_qp, + rdma->sc_port_num, cqe, first_wr); + cqe = NULL; + } + sctxt->sc_wr_chain = first_wr; + sctxt->sc_sqecount += cc->cc_sqecount; + list_add(&info->wi_list, &sctxt->sc_write_info_list); + + trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); return 0; out_err: @@ -634,17 +682,19 @@ out_err: } /** - * svc_rdma_send_write_list - Send all chunks on the Write list + * svc_rdma_prepare_write_list - Construct WR chain for sending Write list * @rdma: controlling RDMA transport * @rctxt: Write list provisioned by the client + * @sctxt: Send WR resources * @xdr: xdr_buf containing an RPC Reply message * - * Returns zero on success, or a negative errno if one or more - * Write chunks could not be sent. + * Returns zero on success, or a negative errno if WR chain + * construction fails for one or more Write chunks. */ -int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - const struct xdr_buf *xdr) +int svc_rdma_prepare_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_recv_ctxt *rctxt, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr) { struct svc_rdma_chunk *chunk; int ret; @@ -652,7 +702,7 @@ int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, pcl_for_each_chunk(chunk, &rctxt->rc_write_pcl) { if (!chunk->ch_payload_length) break; - ret = svc_rdma_send_write_chunk(rdma, chunk, xdr); + ret = svc_rdma_prepare_write_chunk(rdma, sctxt, chunk, xdr); if (ret < 0) return ret; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index bef68efa7034..8b3f0c8c14b2 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -150,6 +150,7 @@ svc_rdma_send_ctxt_alloc(struct svcxprt_rdma *rdma) ctxt->sc_send_wr.sg_list = ctxt->sc_sges; ctxt->sc_send_wr.send_flags = IB_SEND_SIGNALED; ctxt->sc_cqe.done = svc_rdma_wc_send; + INIT_LIST_HEAD(&ctxt->sc_write_info_list); ctxt->sc_xprt_buf = buffer; xdr_buf_init(&ctxt->sc_hdrbuf, ctxt->sc_xprt_buf, rdma->sc_max_req_size); @@ -237,6 +238,7 @@ static void svc_rdma_send_ctxt_release(struct svcxprt_rdma *rdma, struct ib_device *device = rdma->sc_cm_id->device; unsigned int i; + svc_rdma_write_chunk_release(rdma, ctxt); svc_rdma_reply_chunk_release(rdma, ctxt); if (ctxt->sc_page_count) @@ -1054,6 +1056,12 @@ void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma, sctxt->sc_send_wr.num_sge = 1; sctxt->sc_send_wr.opcode = IB_WR_SEND; sctxt->sc_sges[0].length = sctxt->sc_hdrbuf.len; + + /* Ensure only the error message is posted, not any previously + * prepared Write chunk WRs. + */ + sctxt->sc_wr_chain = &sctxt->sc_send_wr; + sctxt->sc_sqecount = 1; if (svc_rdma_post_send(rdma, sctxt)) goto put_ctxt; return; @@ -1101,7 +1109,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) if (!p) goto put_ctxt; - ret = svc_rdma_send_write_list(rdma, rctxt, &rqstp->rq_res); + ret = svc_rdma_prepare_write_list(rdma, rctxt, sctxt, &rqstp->rq_res); if (ret < 0) goto put_ctxt; -- cgit v1.2.3 From 3603bf99062c6d563df4fba3848f829d5401d959 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sat, 28 Feb 2026 14:09:22 -0800 Subject: SUNRPC: xdr.h: fix all kernel-doc warnings Correct a function parameter name (s/page/folio/) and add function return value sections for multiple functions to eliminate kernel-doc warnings: Warning: include/linux/sunrpc/xdr.h:298 function parameter 'folio' not described in 'xdr_set_scratch_folio' Warning: include/linux/sunrpc/xdr.h:337 No description found for return value of 'xdr_stream_remaining' Warning: include/linux/sunrpc/xdr.h:357 No description found for return value of 'xdr_align_size' Warning: include/linux/sunrpc/xdr.h:374 No description found for return value of 'xdr_pad_size' Warning: include/linux/sunrpc/xdr.h:387 No description found for return value of 'xdr_stream_encode_item_present' Signed-off-by: Randy Dunlap Signed-off-by: Chuck Lever --- include/linux/sunrpc/xdr.h | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 152597750f55..b639a6fafcbc 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -290,7 +290,7 @@ xdr_set_scratch_buffer(struct xdr_stream *xdr, void *buf, size_t buflen) /** * xdr_set_scratch_folio - Attach a scratch buffer for decoding data * @xdr: pointer to xdr_stream struct - * @page: an anonymous folio + * @folio: an anonymous folio * * See xdr_set_scratch_buffer(). */ @@ -330,7 +330,7 @@ static inline void xdr_commit_encode(struct xdr_stream *xdr) * xdr_stream_remaining - Return the number of bytes remaining in the stream * @xdr: pointer to struct xdr_stream * - * Return value: + * Returns: * Number of bytes remaining in @xdr before xdr->end */ static inline size_t @@ -350,7 +350,7 @@ ssize_t xdr_stream_encode_opaque_auth(struct xdr_stream *xdr, u32 flavor, * xdr_align_size - Calculate padded size of an object * @n: Size of an object being XDR encoded (in bytes) * - * Return value: + * Returns: * Size (in bytes) of the object including xdr padding */ static inline size_t @@ -368,7 +368,7 @@ xdr_align_size(size_t n) * This implementation avoids the need for conditional * branches or modulo division. * - * Return value: + * Returns: * Size (in bytes) of the needed XDR pad */ static inline size_t xdr_pad_size(size_t n) @@ -380,7 +380,7 @@ static inline size_t xdr_pad_size(size_t n) * xdr_stream_encode_item_present - Encode a "present" list item * @xdr: pointer to xdr_stream * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -399,7 +399,7 @@ static inline ssize_t xdr_stream_encode_item_present(struct xdr_stream *xdr) * xdr_stream_encode_item_absent - Encode a "not present" list item * @xdr: pointer to xdr_stream * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -419,7 +419,7 @@ static inline int xdr_stream_encode_item_absent(struct xdr_stream *xdr) * @p: address in a buffer into which to encode * @n: boolean value to encode * - * Return value: + * Returns: * Address of item following the encoded boolean */ static inline __be32 *xdr_encode_bool(__be32 *p, u32 n) @@ -433,7 +433,7 @@ static inline __be32 *xdr_encode_bool(__be32 *p, u32 n) * @xdr: pointer to xdr_stream * @n: boolean value to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -453,7 +453,7 @@ static inline int xdr_stream_encode_bool(struct xdr_stream *xdr, __u32 n) * @xdr: pointer to xdr_stream * @n: integer to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -474,7 +474,7 @@ xdr_stream_encode_u32(struct xdr_stream *xdr, __u32 n) * @xdr: pointer to xdr_stream * @n: integer to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -495,7 +495,7 @@ xdr_stream_encode_be32(struct xdr_stream *xdr, __be32 n) * @xdr: pointer to xdr_stream * @n: 64-bit integer to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -517,7 +517,7 @@ xdr_stream_encode_u64(struct xdr_stream *xdr, __u64 n) * @ptr: pointer to void pointer * @len: size of object * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -542,7 +542,7 @@ xdr_stream_encode_opaque_inline(struct xdr_stream *xdr, void **ptr, size_t len) * @ptr: pointer to opaque data object * @len: size of object pointed to by @ptr * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -563,7 +563,7 @@ xdr_stream_encode_opaque_fixed(struct xdr_stream *xdr, const void *ptr, size_t l * @ptr: pointer to opaque data object * @len: size of object pointed to by @ptr * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -585,7 +585,7 @@ xdr_stream_encode_opaque(struct xdr_stream *xdr, const void *ptr, size_t len) * @array: array of integers * @array_size: number of elements in @array * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -608,7 +608,7 @@ xdr_stream_encode_uint32_array(struct xdr_stream *xdr, * xdr_item_is_absent - symbolically handle XDR discriminators * @p: pointer to undecoded discriminator * - * Return values: + * Returns: * %true if the following XDR item is absent * %false if the following XDR item is present */ @@ -621,7 +621,7 @@ static inline bool xdr_item_is_absent(const __be32 *p) * xdr_item_is_present - symbolically handle XDR discriminators * @p: pointer to undecoded discriminator * - * Return values: + * Returns: * %true if the following XDR item is present * %false if the following XDR item is absent */ @@ -635,7 +635,7 @@ static inline bool xdr_item_is_present(const __be32 *p) * @xdr: pointer to xdr_stream * @ptr: pointer to a u32 in which to store the result * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -656,7 +656,7 @@ xdr_stream_decode_bool(struct xdr_stream *xdr, __u32 *ptr) * @xdr: pointer to xdr_stream * @ptr: location to store integer * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -677,7 +677,7 @@ xdr_stream_decode_u32(struct xdr_stream *xdr, __u32 *ptr) * @xdr: pointer to xdr_stream * @ptr: location to store integer * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -698,7 +698,7 @@ xdr_stream_decode_be32(struct xdr_stream *xdr, __be32 *ptr) * @xdr: pointer to xdr_stream * @ptr: location to store 64-bit integer * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -720,7 +720,7 @@ xdr_stream_decode_u64(struct xdr_stream *xdr, __u64 *ptr) * @ptr: location to store data * @len: size of buffer pointed to by @ptr * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -746,7 +746,7 @@ xdr_stream_decode_opaque_fixed(struct xdr_stream *xdr, void *ptr, size_t len) * on @xdr. It is therefore expected that the object it points to should * be processed immediately. * - * Return values: + * Returns: * On success, returns size of object stored in *@ptr * %-EBADMSG on XDR buffer overflow * %-EMSGSIZE if the size of the object would exceed @maxlen @@ -777,7 +777,7 @@ xdr_stream_decode_opaque_inline(struct xdr_stream *xdr, void **ptr, size_t maxle * @array: location to store the integer array or NULL * @array_size: number of elements to store * - * Return values: + * Returns: * On success, returns number of elements stored in @array * %-EBADMSG on XDR buffer overflow * %-EMSGSIZE if the size of the array exceeds @array_size -- cgit v1.2.3 From 4e2866b2baaddfff6069a2f18fc134c1d5a08f2b Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 11 Mar 2026 12:18:54 -0400 Subject: SUNRPC: Add svc_rqst_page_release() helper svc_rqst_replace_page() releases displaced pages through a per-rqst folio batch, but exposes the add-or-flush sequence directly. svc_tcp_restore_pages() releases displaced pages individually with put_page(). Introduce svc_rqst_page_release() to encapsulate the batched release mechanism. Convert svc_rqst_replace_page() and svc_tcp_restore_pages() to use it. The latter now benefits from the same batched release that svc_rqst_replace_page() already uses. Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 15 +++++++++++++++ net/sunrpc/svc.c | 7 ++----- net/sunrpc/svcsock.c | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) (limited to 'include/linux/sunrpc') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 669c944eaf7f..1ebd9c7efa70 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -498,6 +498,21 @@ int svc_generic_rpcbind_set(struct net *net, #define RPC_MAX_ADDRBUFLEN (63U) +/** + * svc_rqst_page_release - release a page associated with an RPC transaction + * @rqstp: RPC transaction context + * @page: page to release + * + * Released pages are batched and freed together, reducing + * allocator pressure under heavy RPC workloads. + */ +static inline void svc_rqst_page_release(struct svc_rqst *rqstp, + struct page *page) +{ + if (!folio_batch_add(&rqstp->rq_fbatch, page_folio(page))) + __folio_batch_release(&rqstp->rq_fbatch); +} + /* * When we want to reduce the size of the reserved space in the response * buffer, we need to take into account the size of any checksum data that diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 5e0b5ec2fd52..576fa42e7abf 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -976,11 +976,8 @@ bool svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page) return false; } - if (*rqstp->rq_next_page) { - if (!folio_batch_add(&rqstp->rq_fbatch, - page_folio(*rqstp->rq_next_page))) - __folio_batch_release(&rqstp->rq_fbatch); - } + if (*rqstp->rq_next_page) + svc_rqst_page_release(rqstp, *rqstp->rq_next_page); get_page(page); *(rqstp->rq_next_page++) = page; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 2ce43f9995f1..7be3de1a1aed 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -988,7 +988,7 @@ static size_t svc_tcp_restore_pages(struct svc_sock *svsk, npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; for (i = 0; i < npages; i++) { if (rqstp->rq_pages[i] != NULL) - put_page(rqstp->rq_pages[i]); + svc_rqst_page_release(rqstp, rqstp->rq_pages[i]); BUG_ON(svsk->sk_pages[i] == NULL); rqstp->rq_pages[i] = svsk->sk_pages[i]; svsk->sk_pages[i] = NULL; -- cgit v1.2.3