diff options
author | Peter Zijlstra <peterz@infradead.org> | 2017-12-21 10:11:09 +0100 |
---|---|---|
committer | Ingo Molnar <mingo@kernel.org> | 2018-03-09 07:59:16 +0100 |
commit | b7031a02ec753bf9b52a94a966b05e1abad3b7a9 (patch) | |
tree | 0a5d1d9105bce0a900850e6704e808d345751668 /kernel/sched/fair.c | |
parent | a22e47a4e3f5a9e50a827c5d94705ace3b1eac0b (diff) | |
download | linux-stable-b7031a02ec753bf9b52a94a966b05e1abad3b7a9.tar.gz linux-stable-b7031a02ec753bf9b52a94a966b05e1abad3b7a9.tar.bz2 linux-stable-b7031a02ec753bf9b52a94a966b05e1abad3b7a9.zip |
sched/fair: Add NOHZ_STATS_KICK
Split the NOHZ idle balancer into doing two separate actions:
- update blocked load statistic
- actually load-balance
Since the latter requires the former, ensure this happens. For now
always tag both bits at the same time.
Prepares for a future where we can toggle only the STATS bit.
Suggested-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'kernel/sched/fair.c')
-rw-r--r-- | kernel/sched/fair.c | 52 |
1 files changed, 35 insertions, 17 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5d150478dd58..fc058967c999 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9082,8 +9082,8 @@ static void nohz_balancer_kick(void) if (ilb_cpu >= nr_cpu_ids) return; - flags = atomic_fetch_or(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu)); - if (flags & NOHZ_BALANCE_KICK) + flags = atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(ilb_cpu)); + if (flags & NOHZ_KICK_MASK) return; /* * Use smp_send_reschedule() instead of resched_cpu(). @@ -9202,8 +9202,6 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) int need_serialize, need_decay = 0; u64 max_cost = 0; - update_blocked_averages(cpu); - rcu_read_lock(); for_each_domain(cpu, sd) { /* @@ -9298,20 +9296,27 @@ out: * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the * rebalancing for all the CPUs for whom scheduler ticks are stopped. */ -static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) +static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { - int this_cpu = this_rq->cpu; - struct rq *rq; - int balance_cpu; /* Earliest time when we have to do rebalance again */ unsigned long next_balance = jiffies + 60*HZ; int update_next_balance = 0; + int this_cpu = this_rq->cpu; + unsigned int flags; + int balance_cpu; + struct rq *rq; - if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_BALANCE_KICK)) - return; + if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) + return false; - if (idle != CPU_IDLE) - goto end; + if (idle != CPU_IDLE) { + atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); + return false; + } + + flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); + + SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) @@ -9339,7 +9344,9 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) cpu_load_update_idle(rq); rq_unlock_irq(rq, &rf); - rebalance_domains(rq, CPU_IDLE); + update_blocked_averages(rq->cpu); + if (flags & NOHZ_BALANCE_KICK) + rebalance_domains(rq, CPU_IDLE); } if (time_after(next_balance, rq->next_balance)) { @@ -9348,6 +9355,10 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) } } + update_blocked_averages(this_cpu); + if (flags & NOHZ_BALANCE_KICK) + rebalance_domains(this_rq, CPU_IDLE); + /* * next_balance will be updated only when there is a need. * When the CPU is attached to null domain for ex, it will not be @@ -9355,8 +9366,8 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) */ if (likely(update_next_balance)) nohz.next_balance = next_balance; -end: - atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); + + return true; } /* @@ -9443,7 +9454,10 @@ unlock: return kick; } #else -static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } +static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) +{ + return false; +} #endif /* @@ -9464,7 +9478,11 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h) * load balance only within the local sched_domain hierarchy * and abort nohz_idle_balance altogether if we pull some load. */ - nohz_idle_balance(this_rq, idle); + if (nohz_idle_balance(this_rq, idle)) + return; + + /* normal load balance */ + update_blocked_averages(this_rq->cpu); rebalance_domains(this_rq, idle); } |