summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/tick.h2
-rw-r--r--kernel/sched/idle.c11
-rw-r--r--kernel/time/tick-sched.c61
-rw-r--r--kernel/time/tick-sched.h2
4 files changed, 59 insertions, 17 deletions
diff --git a/include/linux/tick.h b/include/linux/tick.h
index ef0717e5e526..e8e7ff16b929 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -116,6 +116,7 @@ extern bool tick_nohz_enabled;
extern bool tick_nohz_tick_stopped(void);
extern bool tick_nohz_tick_stopped_cpu(int cpu);
extern void tick_nohz_idle_stop_tick(void);
+extern void tick_nohz_idle_retain_tick(void);
extern void tick_nohz_idle_restart_tick(void);
extern void tick_nohz_idle_enter(void);
extern void tick_nohz_idle_exit(void);
@@ -139,6 +140,7 @@ static inline void tick_nohz_idle_stop_tick_protected(void)
static inline int tick_nohz_tick_stopped(void) { return 0; }
static inline int tick_nohz_tick_stopped_cpu(int cpu) { return 0; }
static inline void tick_nohz_idle_stop_tick(void) { }
+static inline void tick_nohz_idle_retain_tick(void) { }
static inline void tick_nohz_idle_restart_tick(void) { }
static inline void tick_nohz_idle_enter(void) { }
static inline void tick_nohz_idle_exit(void) { }
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index a966bd2a6fa0..1a3e9bddd17b 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -185,13 +185,18 @@ static void cpuidle_idle_call(void)
} else {
bool stop_tick = true;
- tick_nohz_idle_stop_tick();
- rcu_idle_enter();
-
/*
* Ask the cpuidle framework to choose a convenient idle state.
*/
next_state = cpuidle_select(drv, dev, &stop_tick);
+
+ if (stop_tick)
+ tick_nohz_idle_stop_tick();
+ else
+ tick_nohz_idle_retain_tick();
+
+ rcu_idle_enter();
+
entered_state = call_cpuidle(drv, dev, next_state);
/*
* Give the governor an opportunity to reflect on the outcome
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index f56d2c695712..c57c98c7e953 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -913,16 +913,19 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
{
- struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
ktime_t expires;
int cpu = smp_processor_id();
- WARN_ON_ONCE(ts->timer_expires_base);
-
- if (!can_stop_idle_tick(cpu, ts))
- goto out;
-
- expires = tick_nohz_next_event(ts, cpu);
+ /*
+ * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
+ * tick timer expiration time is known already.
+ */
+ if (ts->timer_expires_base)
+ expires = ts->timer_expires;
+ else if (can_stop_idle_tick(cpu, ts))
+ expires = tick_nohz_next_event(ts, cpu);
+ else
+ return;
ts->idle_calls++;
@@ -941,9 +944,6 @@ static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
} else {
tick_nohz_retain_tick(ts);
}
-
-out:
- ts->sleep_length = ktime_sub(dev->next_event, ts->idle_entrytime);
}
/**
@@ -956,6 +956,16 @@ void tick_nohz_idle_stop_tick(void)
__tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
}
+void tick_nohz_idle_retain_tick(void)
+{
+ tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
+ /*
+ * Undo the effect of get_next_timer_interrupt() called from
+ * tick_nohz_next_event().
+ */
+ timer_clear_idle();
+}
+
/**
* tick_nohz_idle_enter - prepare for entering idle on the current CPU
*
@@ -1012,15 +1022,42 @@ bool tick_nohz_idle_got_tick(void)
}
/**
- * tick_nohz_get_sleep_length - return the length of the current sleep
+ * tick_nohz_get_sleep_length - return the expected length of the current sleep
*
* Called from power state control code with interrupts disabled
*/
ktime_t tick_nohz_get_sleep_length(void)
{
+ struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
+ int cpu = smp_processor_id();
+ /*
+ * The idle entry time is expected to be a sufficient approximation of
+ * the current time at this point.
+ */
+ ktime_t now = ts->idle_entrytime;
+ ktime_t next_event;
+
+ WARN_ON_ONCE(!ts->inidle);
+
+ if (!can_stop_idle_tick(cpu, ts))
+ goto out_dev;
+
+ next_event = tick_nohz_next_event(ts, cpu);
+ if (!next_event)
+ goto out_dev;
+
+ /*
+ * If the next highres timer to expire is earlier than next_event, the
+ * idle governor needs to know that.
+ */
+ next_event = min_t(u64, next_event,
+ hrtimer_next_event_without(&ts->sched_timer));
+
+ return ktime_sub(next_event, now);
- return ts->sleep_length;
+out_dev:
+ return ktime_sub(dev->next_event, now);
}
/**
diff --git a/kernel/time/tick-sched.h b/kernel/time/tick-sched.h
index 53e45a39bdbc..2b845f2c44b1 100644
--- a/kernel/time/tick-sched.h
+++ b/kernel/time/tick-sched.h
@@ -38,7 +38,6 @@ enum tick_nohz_mode {
* @idle_exittime: Time when the idle state was left
* @idle_sleeptime: Sum of the time slept in idle with sched tick stopped
* @iowait_sleeptime: Sum of the time slept in idle with sched tick stopped, with IO outstanding
- * @sleep_length: Duration of the current idle sleep
* @timer_expires: Anticipated timer expiration time (in case sched tick is stopped)
* @timer_expires_base: Base time clock monotonic for @timer_expires
* @do_timer_lst: CPU was the last one doing do_timer before going idle
@@ -60,7 +59,6 @@ struct tick_sched {
ktime_t idle_exittime;
ktime_t idle_sleeptime;
ktime_t iowait_sleeptime;
- ktime_t sleep_length;
unsigned long last_jiffies;
u64 timer_expires;
u64 timer_expires_base;