diff options
| author | Eric Dumazet <edumazet@google.com> | 2023-06-06 07:41:14 +0000 |
|---|---|---|
| committer | Greg Kroah-Hartman <gregkh@linuxfoundation.org> | 2023-06-14 11:16:49 +0200 |
| commit | cf7436ec9d18a3a3cab187ffb8ee5db6d3e9d778 (patch) | |
| tree | c106ab2bbbb25d915d6f015b5ee327e23c7e8d49 /include | |
| parent | 679fed7e3b30e8ff8695983d3553d89b2b110059 (diff) | |
| download | linux-stable-cf7436ec9d18a3a3cab187ffb8ee5db6d3e9d778.tar.gz linux-stable-cf7436ec9d18a3a3cab187ffb8ee5db6d3e9d778.tar.bz2 linux-stable-cf7436ec9d18a3a3cab187ffb8ee5db6d3e9d778.zip | |
rfs: annotate lockless accesses to sk->sk_rxhash
[ Upstream commit 1e5c647c3f6d4f8497dedcd226204e1880e0ffb3 ]
Add READ_ONCE()/WRITE_ONCE() on accesses to sk->sk_rxhash.
This also prevents a (smart ?) compiler to remove the condition in:
if (sk->sk_rxhash != newval)
sk->sk_rxhash = newval;
We need the condition to avoid dirtying a shared cache line.
Fixes: fec5e652e58f ("rfs: Receive Flow Steering")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Simon Horman <simon.horman@corigine.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Sasha Levin <sashal@kernel.org>
Diffstat (limited to 'include')
| -rw-r--r-- | include/net/sock.h | 18 |
1 files changed, 13 insertions, 5 deletions
diff --git a/include/net/sock.h b/include/net/sock.h index 45e46a1c4afc..f0654c44acf5 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1152,8 +1152,12 @@ static inline void sock_rps_record_flow(const struct sock *sk) * OR an additional socket flag * [1] : sk_state and sk_prot are in the same cache line. */ - if (sk->sk_state == TCP_ESTABLISHED) - sock_rps_record_flow_hash(sk->sk_rxhash); + if (sk->sk_state == TCP_ESTABLISHED) { + /* This READ_ONCE() is paired with the WRITE_ONCE() + * from sock_rps_save_rxhash() and sock_rps_reset_rxhash(). + */ + sock_rps_record_flow_hash(READ_ONCE(sk->sk_rxhash)); + } } #endif } @@ -1162,15 +1166,19 @@ static inline void sock_rps_save_rxhash(struct sock *sk, const struct sk_buff *skb) { #ifdef CONFIG_RPS - if (unlikely(sk->sk_rxhash != skb->hash)) - sk->sk_rxhash = skb->hash; + /* The following WRITE_ONCE() is paired with the READ_ONCE() + * here, and another one in sock_rps_record_flow(). + */ + if (unlikely(READ_ONCE(sk->sk_rxhash) != skb->hash)) + WRITE_ONCE(sk->sk_rxhash, skb->hash); #endif } static inline void sock_rps_reset_rxhash(struct sock *sk) { #ifdef CONFIG_RPS - sk->sk_rxhash = 0; + /* Paired with READ_ONCE() in sock_rps_record_flow() */ + WRITE_ONCE(sk->sk_rxhash, 0); #endif } |