diff options
| author | Sebastian Andrzej Siewior <bigeasy@linutronix.de> | 2024-07-04 19:03:39 +0200 |
|---|---|---|
| committer | Peter Zijlstra <peterz@infradead.org> | 2024-07-09 13:26:36 +0200 |
| commit | 0d40a6d83e3e6751f1107ba33587262d937c969f (patch) | |
| tree | 691c1d8f6993be280af09b4da362eb714ce34e6e /kernel/events | |
| parent | 5af42f928f3ac555c228740fb4a92d05b19fdd49 (diff) | |
| download | linux-0d40a6d83e3e6751f1107ba33587262d937c969f.tar.gz linux-0d40a6d83e3e6751f1107ba33587262d937c969f.tar.bz2 linux-0d40a6d83e3e6751f1107ba33587262d937c969f.zip | |
perf: Move swevent_htable::recursion into task_struct.
The swevent_htable::recursion counter is used to avoid creating an
swevent while an event is processed to avoid recursion. The counter is
per-CPU and preemption must be disabled to have a stable counter.
perf_pending_task() disables preemption to access the counter and then
signal. This is problematic on PREEMPT_RT because sending a signal uses
a spinlock_t which must not be acquired in atomic on PREEMPT_RT because
it becomes a sleeping lock.
The atomic context can be avoided by moving the counter into the
task_struct. There is a 4 byte hole between futex_state (usually always
on) and the following perf pointer (perf_event_ctxp). After the
recursion lost some weight it fits perfectly.
Move swevent_htable::recursion into task_struct.
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: Marco Elver <elver@google.com>
Link: https://lore.kernel.org/r/20240704170424.1466941-6-bigeasy@linutronix.de
Diffstat (limited to 'kernel/events')
| -rw-r--r-- | kernel/events/core.c | 13 | ||||
| -rw-r--r-- | kernel/events/internal.h | 2 |
2 files changed, 4 insertions, 11 deletions
diff --git a/kernel/events/core.c b/kernel/events/core.c index 53e2750bf720..b5232257bc83 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9763,11 +9763,7 @@ struct swevent_htable { struct swevent_hlist *swevent_hlist; struct mutex hlist_mutex; int hlist_refcount; - - /* Recursion avoidance in each contexts */ - u8 recursion[PERF_NR_CONTEXTS]; }; - static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); /* @@ -9965,17 +9961,13 @@ DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); int perf_swevent_get_recursion_context(void) { - struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); - - return get_recursion_context(swhash->recursion); + return get_recursion_context(current->perf_recursion); } EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); void perf_swevent_put_recursion_context(int rctx) { - struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); - - put_recursion_context(swhash->recursion, rctx); + put_recursion_context(current->perf_recursion, rctx); } void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) @@ -13642,6 +13634,7 @@ int perf_event_init_task(struct task_struct *child, u64 clone_flags) { int ret; + memset(child->perf_recursion, 0, sizeof(child->perf_recursion)); child->perf_event_ctxp = NULL; mutex_init(&child->perf_event_mutex); INIT_LIST_HEAD(&child->perf_event_list); diff --git a/kernel/events/internal.h b/kernel/events/internal.h index 7f06b79b3b9f..451514442a1b 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -221,7 +221,7 @@ static inline int get_recursion_context(u8 *recursion) return rctx; } -static inline void put_recursion_context(u8 *recursion, int rctx) +static inline void put_recursion_context(u8 *recursion, unsigned char rctx) { barrier(); recursion[rctx]--; |