diff options
| author | Paul E. McKenney <paulmck@kernel.org> | 2020-08-08 07:56:31 -0700 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@kernel.org> | 2020-08-24 18:40:27 -0700 |
| commit | a657f2617010ae237db5693f875968c28e8f732f (patch) | |
| tree | a4cf1d4ed7377d6829cfc0c1a968ce9b093d5607 /kernel | |
| parent | 3d29aaf1ef992b5b4612fe32b9e6f517f7bba904 (diff) | |
| download | linux-a657f2617010ae237db5693f875968c28e8f732f.tar.gz linux-a657f2617010ae237db5693f875968c28e8f732f.tar.bz2 linux-a657f2617010ae237db5693f875968c28e8f732f.zip | |
rcu: Execute RCU reader shortly after rcu_core for strict GPs
A kernel built with CONFIG_RCU_STRICT_GRACE_PERIOD=y needs a quiescent
state to appear very shortly after a CPU has noticed a new grace period.
Placing an RCU reader immediately after this point is ineffective because
this normally happens in softirq context, which acts as a big RCU reader.
This commit therefore introduces a new per-CPU work_struct, which is
used at the end of rcu_core() processing to schedule an RCU read-side
critical section from within a clean environment.
Reported-by Jann Horn <jannh@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/rcu/tree.c | 13 | ||||
| -rw-r--r-- | kernel/rcu/tree.h | 1 |
2 files changed, 14 insertions, 0 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 4bbedfc0f79b..31995b3f0ed9 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2646,6 +2646,14 @@ void rcu_force_quiescent_state(void) } EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); +// Workqueue handler for an RCU reader for kernels enforcing struct RCU +// grace periods. +static void strict_work_handler(struct work_struct *work) +{ + rcu_read_lock(); + rcu_read_unlock(); +} + /* Perform RCU core processing work for the current CPU. */ static __latent_entropy void rcu_core(void) { @@ -2690,6 +2698,10 @@ static __latent_entropy void rcu_core(void) /* Do any needed deferred wakeups of rcuo kthreads. */ do_nocb_deferred_wakeup(rdp); trace_rcu_utilization(TPS("End RCU core")); + + // If strict GPs, schedule an RCU reader in a clean environment. + if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) + queue_work_on(rdp->cpu, rcu_gp_wq, &rdp->strict_work); } static void rcu_core_si(struct softirq_action *h) @@ -3887,6 +3899,7 @@ rcu_boot_init_percpu_data(int cpu) /* Set up local state, ensuring consistent view of global state. */ rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu); + INIT_WORK(&rdp->strict_work, strict_work_handler); WARN_ON_ONCE(rdp->dynticks_nesting != 1); WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp))); rdp->rcu_ofl_gp_seq = rcu_state.gp_seq; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index c96ae351688b..5831ac0b254f 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -164,6 +164,7 @@ struct rcu_data { /* period it is aware of. */ struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ bool defer_qs_iw_pending; /* Scheduler attention pending? */ + struct work_struct strict_work; /* Schedule readers for strict GPs. */ /* 2) batch handling */ struct rcu_segcblist cblist; /* Segmented callback list, with */ |