From e091853ebdb486fd8bde86b87178fdf3850914fc Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:46 -0400 Subject: SUNRPC mark the first transport When an RPC client gets created it's first transport is special and should be marked a main transport. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 1 + 1 file changed, 1 insertion(+) (limited to 'net') diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 9bf820bad84c..408618765aa5 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -412,6 +412,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args, } rpc_clnt_set_transport(clnt, xprt, timeout); + xprt->main = true; xprt_iter_init(&clnt->cl_xpi, xps); xprt_switch_put(xps); -- cgit v1.2.3 From 0e65ea43d9c7c038e167b20165a0e9ed1e9cca83 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:47 -0400 Subject: SUNRPC display xprt's main value in sysfs's xprt_info Display in sysfs in the information about the xprt if this is a main transport or not. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/sysfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'net') diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index 2fbaba27d5c6..124f2e1e3409 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -103,10 +103,10 @@ static ssize_t rpc_sysfs_xprt_info_show(struct kobject *kobj, ret = sprintf(buf, "last_used=%lu\ncur_cong=%lu\ncong_win=%lu\n" "max_num_slots=%u\nmin_num_slots=%u\nnum_reqs=%u\n" "binding_q_len=%u\nsending_q_len=%u\npending_q_len=%u\n" - "backlog_q_len=%u\n", xprt->last_used, xprt->cong, - xprt->cwnd, xprt->max_reqs, xprt->min_reqs, + "backlog_q_len=%u\nmain_xprt=%d\n", xprt->last_used, + xprt->cong, xprt->cwnd, xprt->max_reqs, xprt->min_reqs, xprt->num_reqs, xprt->binding.qlen, xprt->sending.qlen, - xprt->pending.qlen, xprt->backlog.qlen); + xprt->pending.qlen, xprt->backlog.qlen, xprt->main); xprt_put(xprt); return ret + 1; } -- cgit v1.2.3 From a8482488a7d6d320f63a9ee1912dbb5ae5b80a61 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:48 -0400 Subject: SUNRPC query transport's source port Provide ability to query transport's source port. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/xprtsock.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'net') diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 2ad4d0df45fe..4611845ec1eb 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -1653,6 +1653,13 @@ static int xs_get_srcport(struct sock_xprt *transport) return port; } +unsigned short get_srcport(struct rpc_xprt *xprt) +{ + struct sock_xprt *sock = container_of(xprt, struct sock_xprt, xprt); + return sock->srcport; +} +EXPORT_SYMBOL(get_srcport); + static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port) { if (transport->srcport != 0) -- cgit v1.2.3 From c1830a63c79aa90f725ed6feaad097473f2b990d Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:49 -0400 Subject: SUNRPC for TCP display xprt's source port in sysfs xprt_info Using TCP connection's source port it is useful to match connections seen on the network traces to the xprts used by the linux nfs client. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/sysfs.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'net') diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index 124f2e1e3409..08aa503295b7 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "sysfs.h" @@ -103,10 +104,13 @@ static ssize_t rpc_sysfs_xprt_info_show(struct kobject *kobj, ret = sprintf(buf, "last_used=%lu\ncur_cong=%lu\ncong_win=%lu\n" "max_num_slots=%u\nmin_num_slots=%u\nnum_reqs=%u\n" "binding_q_len=%u\nsending_q_len=%u\npending_q_len=%u\n" - "backlog_q_len=%u\nmain_xprt=%d\n", xprt->last_used, - xprt->cong, xprt->cwnd, xprt->max_reqs, xprt->min_reqs, - xprt->num_reqs, xprt->binding.qlen, xprt->sending.qlen, - xprt->pending.qlen, xprt->backlog.qlen, xprt->main); + "backlog_q_len=%u\nmain_xprt=%d\nsrc_port=%u\n", + xprt->last_used, xprt->cong, xprt->cwnd, xprt->max_reqs, + xprt->min_reqs, xprt->num_reqs, xprt->binding.qlen, + xprt->sending.qlen, xprt->pending.qlen, + xprt->backlog.qlen, xprt->main, + (xprt->xprt_class->ident == XPRT_TRANSPORT_TCP) ? + get_srcport(xprt) : 0); xprt_put(xprt); return ret + 1; } -- cgit v1.2.3 From 5b7eb78486cd9ac58bfbd6d84ea0fe2d9fead03b Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:50 -0400 Subject: SUNRPC: take a xprt offline using sysfs Using sysfs's xprt_state attribute, mark a particular transport offline. It will not be picked during the round-robin selection. It's not allowed to take the main (1st created transport associated with the rpc_client) offline. Also bring a transport back online via sysfs by writing "online" and that would allow for this transport to be picked during the round- robin selection. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/sysfs.c | 66 +++++++++++++++++++++++++++++++++++++++++++--- net/sunrpc/sysfs.h | 1 + net/sunrpc/xprtmultipath.c | 6 +++-- 3 files changed, 67 insertions(+), 6 deletions(-) (limited to 'net') diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index 08aa503295b7..a30ad18aa7dc 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -68,6 +68,15 @@ rpc_sysfs_xprt_kobj_get_xprt(struct kobject *kobj) return xprt_get(x->xprt); } +static inline struct rpc_xprt_switch * +rpc_sysfs_xprt_kobj_get_xprt_switch(struct kobject *kobj) +{ + struct rpc_sysfs_xprt *x = container_of(kobj, + struct rpc_sysfs_xprt, kobject); + + return xprt_switch_get(x->xprt_switch); +} + static inline struct rpc_xprt_switch * rpc_sysfs_xprt_switch_kobj_get_xprt(struct kobject *kobj) { @@ -122,7 +131,7 @@ static ssize_t rpc_sysfs_xprt_state_show(struct kobject *kobj, struct rpc_xprt *xprt = rpc_sysfs_xprt_kobj_get_xprt(kobj); ssize_t ret; int locked, connected, connecting, close_wait, bound, binding, - closing, congested, cwnd_wait, write_space; + closing, congested, cwnd_wait, write_space, offline; if (!xprt) return 0; @@ -140,8 +149,9 @@ static ssize_t rpc_sysfs_xprt_state_show(struct kobject *kobj, congested = test_bit(XPRT_CONGESTED, &xprt->state); cwnd_wait = test_bit(XPRT_CWND_WAIT, &xprt->state); write_space = test_bit(XPRT_WRITE_SPACE, &xprt->state); + offline = test_bit(XPRT_OFFLINE, &xprt->state); - ret = sprintf(buf, "state=%s %s %s %s %s %s %s %s %s %s\n", + ret = sprintf(buf, "state=%s %s %s %s %s %s %s %s %s %s %s\n", locked ? "LOCKED" : "", connected ? "CONNECTED" : "", connecting ? "CONNECTING" : "", @@ -151,7 +161,8 @@ static ssize_t rpc_sysfs_xprt_state_show(struct kobject *kobj, closing ? "CLOSING" : "", congested ? "CONGESTED" : "", cwnd_wait ? "CWND_WAIT" : "", - write_space ? "WRITE_SPACE" : ""); + write_space ? "WRITE_SPACE" : "", + offline ? "OFFLINE" : ""); } xprt_put(xprt); @@ -233,6 +244,52 @@ out_err: goto out; } +static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct rpc_xprt *xprt = rpc_sysfs_xprt_kobj_get_xprt(kobj); + int offline = 0, online = 0; + struct rpc_xprt_switch *xps = rpc_sysfs_xprt_kobj_get_xprt_switch(kobj); + + if (!xprt) + return 0; + + if (!strncmp(buf, "offline", 7)) + offline = 1; + else if (!strncmp(buf, "online", 6)) + online = 1; + else + return -EINVAL; + + if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) { + count = -EINTR; + goto out_put; + } + if (xprt->main) { + count = -EINVAL; + goto release_tasks; + } + if (offline) { + set_bit(XPRT_OFFLINE, &xprt->state); + spin_lock(&xps->xps_lock); + xps->xps_nactive--; + spin_unlock(&xps->xps_lock); + } else if (online) { + clear_bit(XPRT_OFFLINE, &xprt->state); + spin_lock(&xps->xps_lock); + xps->xps_nactive++; + spin_unlock(&xps->xps_lock); + } + +release_tasks: + xprt_release_write(xprt, NULL); +out_put: + xprt_put(xprt); + xprt_switch_put(xps); + return count; +} + int rpc_sysfs_init(void) { rpc_sunrpc_kset = kset_create_and_add("sunrpc", NULL, kernel_kobj); @@ -303,7 +360,7 @@ static struct kobj_attribute rpc_sysfs_xprt_info = __ATTR(xprt_info, 0444, rpc_sysfs_xprt_info_show, NULL); static struct kobj_attribute rpc_sysfs_xprt_change_state = __ATTR(xprt_state, - 0644, rpc_sysfs_xprt_state_show, NULL); + 0644, rpc_sysfs_xprt_state_show, rpc_sysfs_xprt_state_change); static struct attribute *rpc_sysfs_xprt_attrs[] = { &rpc_sysfs_xprt_dstaddr.attr, @@ -466,6 +523,7 @@ void rpc_sysfs_xprt_setup(struct rpc_xprt_switch *xprt_switch, if (rpc_xprt) { xprt->xprt_sysfs = rpc_xprt; rpc_xprt->xprt = xprt; + rpc_xprt->xprt_switch = xprt_switch; kobject_uevent(&rpc_xprt->kobject, KOBJ_ADD); } } diff --git a/net/sunrpc/sysfs.h b/net/sunrpc/sysfs.h index ff10451de6fa..6620cebd1037 100644 --- a/net/sunrpc/sysfs.h +++ b/net/sunrpc/sysfs.h @@ -22,6 +22,7 @@ struct rpc_sysfs_xprt_switch { struct rpc_sysfs_xprt { struct kobject kobject; struct rpc_xprt *xprt; + struct rpc_xprt_switch *xprt_switch; }; int rpc_sysfs_init(void); diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c index 07e76ae1028a..5f4845d1e922 100644 --- a/net/sunrpc/xprtmultipath.c +++ b/net/sunrpc/xprtmultipath.c @@ -65,7 +65,8 @@ static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps, { if (unlikely(xprt == NULL)) return; - xps->xps_nactive--; + if (!test_bit(XPRT_OFFLINE, &xprt->state)) + xps->xps_nactive--; xps->xps_nxprts--; if (xps->xps_nxprts == 0) xps->xps_net = NULL; @@ -230,7 +231,8 @@ void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi) static bool xprt_is_active(const struct rpc_xprt *xprt) { - return kref_read(&xprt->kref) != 0; + return (kref_read(&xprt->kref) != 0 && + !test_bit(XPRT_OFFLINE, &xprt->state)); } static -- cgit v1.2.3 From 6a2840590b66e4914d583be61e40445386bb5835 Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:52 -0400 Subject: sunrpc: display xprt's queuelen of assigned tasks via sysfs Once a task grabs a trasnport it's reflected in the queuelen of the rpc_xprt structure. Add display of that value in the xprt's info file in sysfs. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/sysfs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'net') diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index a30ad18aa7dc..b576c7f06829 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -113,13 +113,15 @@ static ssize_t rpc_sysfs_xprt_info_show(struct kobject *kobj, ret = sprintf(buf, "last_used=%lu\ncur_cong=%lu\ncong_win=%lu\n" "max_num_slots=%u\nmin_num_slots=%u\nnum_reqs=%u\n" "binding_q_len=%u\nsending_q_len=%u\npending_q_len=%u\n" - "backlog_q_len=%u\nmain_xprt=%d\nsrc_port=%u\n", + "backlog_q_len=%u\nmain_xprt=%d\nsrc_port=%u\n" + "tasks_queuelen=%ld\n", xprt->last_used, xprt->cong, xprt->cwnd, xprt->max_reqs, xprt->min_reqs, xprt->num_reqs, xprt->binding.qlen, xprt->sending.qlen, xprt->pending.qlen, xprt->backlog.qlen, xprt->main, (xprt->xprt_class->ident == XPRT_TRANSPORT_TCP) ? - get_srcport(xprt) : 0); + get_srcport(xprt) : 0, + atomic_long_read(&xprt->queuelen)); xprt_put(xprt); return ret + 1; } -- cgit v1.2.3 From 6f081693e7b2ba63422b735684b05a850a6351ba Mon Sep 17 00:00:00 2001 From: Olga Kornievskaia Date: Wed, 23 Jun 2021 23:28:53 -0400 Subject: sunrpc: remove an offlined xprt using sysfs Once a transport has been put offline, this transport can be also removed from the list of transports. Any tasks that have been stuck on this transport would find the next available active transport and be re-tried. This transport would be removed from the xprt_switch list and freed. Signed-off-by: Olga Kornievskaia Signed-off-by: Trond Myklebust --- net/sunrpc/clnt.c | 24 ++++++++++++++++++++++++ net/sunrpc/sysfs.c | 26 ++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 4 deletions(-) (limited to 'net') diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 408618765aa5..8b4de70e8ead 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -2106,6 +2106,30 @@ call_connect_status(struct rpc_task *task) case -ENOTCONN: case -EAGAIN: case -ETIMEDOUT: + if (!(task->tk_flags & RPC_TASK_NO_ROUND_ROBIN) && + (task->tk_flags & RPC_TASK_MOVEABLE) && + test_bit(XPRT_REMOVE, &xprt->state)) { + struct rpc_xprt *saved = task->tk_xprt; + struct rpc_xprt_switch *xps; + + rcu_read_lock(); + xps = xprt_switch_get(rcu_dereference(clnt->cl_xpi.xpi_xpswitch)); + rcu_read_unlock(); + if (xps->xps_nxprts > 1) { + long value; + + xprt_release(task); + value = atomic_long_dec_return(&xprt->queuelen); + if (value == 0) + rpc_xprt_switch_remove_xprt(xps, saved); + xprt_put(saved); + task->tk_xprt = NULL; + task->tk_action = call_start; + } + xprt_switch_put(xps); + if (!task->tk_xprt) + return; + } goto out_retry; case -ENOBUFS: rpc_delay(task, HZ >> 2); diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index b576c7f06829..64da3bfd28e6 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -133,7 +133,7 @@ static ssize_t rpc_sysfs_xprt_state_show(struct kobject *kobj, struct rpc_xprt *xprt = rpc_sysfs_xprt_kobj_get_xprt(kobj); ssize_t ret; int locked, connected, connecting, close_wait, bound, binding, - closing, congested, cwnd_wait, write_space, offline; + closing, congested, cwnd_wait, write_space, offline, remove; if (!xprt) return 0; @@ -152,8 +152,9 @@ static ssize_t rpc_sysfs_xprt_state_show(struct kobject *kobj, cwnd_wait = test_bit(XPRT_CWND_WAIT, &xprt->state); write_space = test_bit(XPRT_WRITE_SPACE, &xprt->state); offline = test_bit(XPRT_OFFLINE, &xprt->state); + remove = test_bit(XPRT_REMOVE, &xprt->state); - ret = sprintf(buf, "state=%s %s %s %s %s %s %s %s %s %s %s\n", + ret = sprintf(buf, "state=%s %s %s %s %s %s %s %s %s %s %s %s\n", locked ? "LOCKED" : "", connected ? "CONNECTED" : "", connecting ? "CONNECTING" : "", @@ -164,7 +165,8 @@ static ssize_t rpc_sysfs_xprt_state_show(struct kobject *kobj, congested ? "CONGESTED" : "", cwnd_wait ? "CWND_WAIT" : "", write_space ? "WRITE_SPACE" : "", - offline ? "OFFLINE" : ""); + offline ? "OFFLINE" : "", + remove ? "REMOVE" : ""); } xprt_put(xprt); @@ -251,7 +253,7 @@ static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj, const char *buf, size_t count) { struct rpc_xprt *xprt = rpc_sysfs_xprt_kobj_get_xprt(kobj); - int offline = 0, online = 0; + int offline = 0, online = 0, remove = 0; struct rpc_xprt_switch *xps = rpc_sysfs_xprt_kobj_get_xprt_switch(kobj); if (!xprt) @@ -261,6 +263,8 @@ static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj, offline = 1; else if (!strncmp(buf, "online", 6)) online = 1; + else if (!strncmp(buf, "remove", 6)) + remove = 1; else return -EINVAL; @@ -282,6 +286,20 @@ static ssize_t rpc_sysfs_xprt_state_change(struct kobject *kobj, spin_lock(&xps->xps_lock); xps->xps_nactive++; spin_unlock(&xps->xps_lock); + } else if (remove) { + if (test_bit(XPRT_OFFLINE, &xprt->state)) { + set_bit(XPRT_REMOVE, &xprt->state); + xprt_force_disconnect(xprt); + if (test_bit(XPRT_CONNECTED, &xprt->state)) { + if (!xprt->sending.qlen && + !xprt->pending.qlen && + !xprt->backlog.qlen && + !atomic_long_read(&xprt->queuelen)) + rpc_xprt_switch_remove_xprt(xps, xprt); + } + } else { + count = -EINVAL; + } } release_tasks: -- cgit v1.2.3