diff options
-rw-r--r-- | drivers/base/power/domain.c | 342 | ||||
-rw-r--r-- | drivers/cpuidle/cpuidle.c | 1 | ||||
-rw-r--r-- | drivers/cpuidle/governors/menu.c | 3 | ||||
-rw-r--r-- | include/linux/cpuidle.h | 1 | ||||
-rw-r--r-- | include/linux/pm_domain.h | 18 |
5 files changed, 290 insertions, 75 deletions
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 83aa694a8efe..ba3487c9835b 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -75,19 +75,6 @@ static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev) start_latency_ns, "start"); } -static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev, - save_state_latency_ns, "state save"); -} - -static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev) -{ - return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev, - restore_state_latency_ns, - "state restore"); -} - static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd) { bool ret = false; @@ -139,6 +126,19 @@ static void genpd_set_active(struct generic_pm_domain *genpd) genpd->status = GPD_STATE_ACTIVE; } +static void genpd_recalc_cpu_exit_latency(struct generic_pm_domain *genpd) +{ + s64 usecs64; + + if (!genpd->cpu_data) + return; + + usecs64 = genpd->power_on_latency_ns; + do_div(usecs64, NSEC_PER_USEC); + usecs64 += genpd->cpu_data->saved_exit_latency; + genpd->cpu_data->idle_state->exit_latency = usecs64; +} + /** * __pm_genpd_poweron - Restore power to a given PM domain and its masters. * @genpd: PM domain to power up. @@ -146,7 +146,7 @@ static void genpd_set_active(struct generic_pm_domain *genpd) * Restore power to @genpd and all of its masters so that it is possible to * resume a device belonging to it. */ -int __pm_genpd_poweron(struct generic_pm_domain *genpd) +static int __pm_genpd_poweron(struct generic_pm_domain *genpd) __releases(&genpd->lock) __acquires(&genpd->lock) { struct gpd_link *link; @@ -176,6 +176,13 @@ int __pm_genpd_poweron(struct generic_pm_domain *genpd) return 0; } + if (genpd->cpu_data) { + cpuidle_pause_and_lock(); + genpd->cpu_data->idle_state->disabled = true; + cpuidle_resume_and_unlock(); + goto out; + } + /* * The list is guaranteed not to change while the loop below is being * executed, unless one of the masters' .power_on() callbacks fiddles @@ -215,6 +222,7 @@ int __pm_genpd_poweron(struct generic_pm_domain *genpd) if (elapsed_ns > genpd->power_on_latency_ns) { genpd->power_on_latency_ns = elapsed_ns; genpd->max_off_time_changed = true; + genpd_recalc_cpu_exit_latency(genpd); if (genpd->name) pr_warning("%s: Power-on latency exceeded, " "new value %lld ns\n", genpd->name, @@ -222,6 +230,7 @@ int __pm_genpd_poweron(struct generic_pm_domain *genpd) } } + out: genpd_set_active(genpd); return 0; @@ -251,6 +260,19 @@ int pm_genpd_poweron(struct generic_pm_domain *genpd) #ifdef CONFIG_PM_RUNTIME +static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev) +{ + return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev, + save_state_latency_ns, "state save"); +} + +static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev) +{ + return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev, + restore_state_latency_ns, + "state restore"); +} + static int genpd_dev_pm_qos_notifier(struct notifier_block *nb, unsigned long val, void *ptr) { @@ -275,7 +297,7 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb, pdd = dev->power.subsys_data ? dev->power.subsys_data->domain_data : NULL; - if (pdd) { + if (pdd && pdd->dev) { to_gpd_data(pdd)->td.constraint_changed = true; genpd = dev_to_genpd(dev); } else { @@ -339,19 +361,16 @@ static void __pm_genpd_restore_device(struct pm_domain_data *pdd, { struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); struct device *dev = pdd->dev; + bool need_restore = gpd_data->need_restore; - if (!gpd_data->need_restore) - return; - + gpd_data->need_restore = false; mutex_unlock(&genpd->lock); genpd_start_dev(genpd, dev); - genpd_restore_dev(genpd, dev); - genpd_stop_dev(genpd, dev); + if (need_restore) + genpd_restore_dev(genpd, dev); mutex_lock(&genpd->lock); - - gpd_data->need_restore = false; } /** @@ -458,6 +477,21 @@ static int pm_genpd_poweroff(struct generic_pm_domain *genpd) } } + if (genpd->cpu_data) { + /* + * If cpu_data is set, cpuidle should turn the domain off when + * the CPU in it is idle. In that case we don't decrement the + * subdomain counts of the master domains, so that power is not + * removed from the current domain prematurely as a result of + * cutting off the masters' power. + */ + genpd->status = GPD_STATE_POWER_OFF; + cpuidle_pause_and_lock(); + genpd->cpu_data->idle_state->disabled = false; + cpuidle_resume_and_unlock(); + goto out; + } + if (genpd->power_off) { ktime_t time_start; s64 elapsed_ns; @@ -595,7 +629,7 @@ static int pm_genpd_runtime_resume(struct device *dev) /* If power.irq_safe, the PM domain is never powered off. */ if (dev->power.irq_safe) - goto out; + return genpd_start_dev(genpd, dev); mutex_lock(&genpd->lock); ret = __pm_genpd_poweron(genpd); @@ -628,9 +662,6 @@ static int pm_genpd_runtime_resume(struct device *dev) wake_up_all(&genpd->status_wait_queue); mutex_unlock(&genpd->lock); - out: - genpd_start_dev(genpd, dev); - return 0; } @@ -1235,6 +1266,27 @@ static void pm_genpd_complete(struct device *dev) #endif /* CONFIG_PM_SLEEP */ +static struct generic_pm_domain_data *__pm_genpd_alloc_dev_data(struct device *dev) +{ + struct generic_pm_domain_data *gpd_data; + + gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL); + if (!gpd_data) + return NULL; + + mutex_init(&gpd_data->lock); + gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier; + dev_pm_qos_add_notifier(dev, &gpd_data->nb); + return gpd_data; +} + +static void __pm_genpd_free_dev_data(struct device *dev, + struct generic_pm_domain_data *gpd_data) +{ + dev_pm_qos_remove_notifier(dev, &gpd_data->nb); + kfree(gpd_data); +} + /** * __pm_genpd_add_device - Add a device to an I/O PM domain. * @genpd: PM domain to add the device to. @@ -1244,7 +1296,7 @@ static void pm_genpd_complete(struct device *dev) int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, struct gpd_timing_data *td) { - struct generic_pm_domain_data *gpd_data; + struct generic_pm_domain_data *gpd_data_new, *gpd_data = NULL; struct pm_domain_data *pdd; int ret = 0; @@ -1253,14 +1305,10 @@ int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev)) return -EINVAL; - gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL); - if (!gpd_data) + gpd_data_new = __pm_genpd_alloc_dev_data(dev); + if (!gpd_data_new) return -ENOMEM; - mutex_init(&gpd_data->lock); - gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier; - dev_pm_qos_add_notifier(dev, &gpd_data->nb); - genpd_acquire_lock(genpd); if (genpd->prepared_count > 0) { @@ -1274,35 +1322,42 @@ int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, goto out; } + ret = dev_pm_get_subsys_data(dev); + if (ret) + goto out; + genpd->device_count++; genpd->max_off_time_changed = true; - dev_pm_get_subsys_data(dev); - - mutex_lock(&gpd_data->lock); spin_lock_irq(&dev->power.lock); + dev->pm_domain = &genpd->domain; - dev->power.subsys_data->domain_data = &gpd_data->base; - gpd_data->base.dev = dev; - list_add_tail(&gpd_data->base.list_node, &genpd->dev_list); - gpd_data->need_restore = genpd->status == GPD_STATE_POWER_OFF; + if (dev->power.subsys_data->domain_data) { + gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); + } else { + gpd_data = gpd_data_new; + dev->power.subsys_data->domain_data = &gpd_data->base; + } + gpd_data->refcount++; if (td) gpd_data->td = *td; + spin_unlock_irq(&dev->power.lock); + + mutex_lock(&gpd_data->lock); + gpd_data->base.dev = dev; + list_add_tail(&gpd_data->base.list_node, &genpd->dev_list); + gpd_data->need_restore = genpd->status == GPD_STATE_POWER_OFF; gpd_data->td.constraint_changed = true; gpd_data->td.effective_constraint_ns = -1; - spin_unlock_irq(&dev->power.lock); mutex_unlock(&gpd_data->lock); - genpd_release_lock(genpd); - - return 0; - out: genpd_release_lock(genpd); - dev_pm_qos_remove_notifier(dev, &gpd_data->nb); - kfree(gpd_data); + if (gpd_data != gpd_data_new) + __pm_genpd_free_dev_data(dev, gpd_data_new); + return ret; } @@ -1348,6 +1403,7 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd, { struct generic_pm_domain_data *gpd_data; struct pm_domain_data *pdd; + bool remove = false; int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -1368,22 +1424,28 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd, genpd->max_off_time_changed = true; spin_lock_irq(&dev->power.lock); + dev->pm_domain = NULL; pdd = dev->power.subsys_data->domain_data; list_del_init(&pdd->list_node); - dev->power.subsys_data->domain_data = NULL; + gpd_data = to_gpd_data(pdd); + if (--gpd_data->refcount == 0) { + dev->power.subsys_data->domain_data = NULL; + remove = true; + } + spin_unlock_irq(&dev->power.lock); - gpd_data = to_gpd_data(pdd); mutex_lock(&gpd_data->lock); pdd->dev = NULL; mutex_unlock(&gpd_data->lock); genpd_release_lock(genpd); - dev_pm_qos_remove_notifier(dev, &gpd_data->nb); - kfree(gpd_data); dev_pm_put_subsys_data(dev); + if (remove) + __pm_genpd_free_dev_data(dev, gpd_data); + return 0; out: @@ -1541,33 +1603,52 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, * @dev: Device to add the callbacks to. * @ops: Set of callbacks to add. * @td: Timing data to add to the device along with the callbacks (optional). + * + * Every call to this routine should be balanced with a call to + * __pm_genpd_remove_callbacks() and they must not be nested. */ int pm_genpd_add_callbacks(struct device *dev, struct gpd_dev_ops *ops, struct gpd_timing_data *td) { - struct pm_domain_data *pdd; + struct generic_pm_domain_data *gpd_data_new, *gpd_data = NULL; int ret = 0; - if (!(dev && dev->power.subsys_data && ops)) + if (!(dev && ops)) return -EINVAL; + gpd_data_new = __pm_genpd_alloc_dev_data(dev); + if (!gpd_data_new) + return -ENOMEM; + pm_runtime_disable(dev); device_pm_lock(); - pdd = dev->power.subsys_data->domain_data; - if (pdd) { - struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); + ret = dev_pm_get_subsys_data(dev); + if (ret) + goto out; - gpd_data->ops = *ops; - if (td) - gpd_data->td = *td; + spin_lock_irq(&dev->power.lock); + + if (dev->power.subsys_data->domain_data) { + gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); } else { - ret = -EINVAL; + gpd_data = gpd_data_new; + dev->power.subsys_data->domain_data = &gpd_data->base; } + gpd_data->refcount++; + gpd_data->ops = *ops; + if (td) + gpd_data->td = *td; + + spin_unlock_irq(&dev->power.lock); + out: device_pm_unlock(); pm_runtime_enable(dev); + if (gpd_data != gpd_data_new) + __pm_genpd_free_dev_data(dev, gpd_data_new); + return ret; } EXPORT_SYMBOL_GPL(pm_genpd_add_callbacks); @@ -1576,10 +1657,13 @@ EXPORT_SYMBOL_GPL(pm_genpd_add_callbacks); * __pm_genpd_remove_callbacks - Remove PM domain callbacks from a given device. * @dev: Device to remove the callbacks from. * @clear_td: If set, clear the device's timing data too. + * + * This routine can only be called after pm_genpd_add_callbacks(). */ int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td) { - struct pm_domain_data *pdd; + struct generic_pm_domain_data *gpd_data = NULL; + bool remove = false; int ret = 0; if (!(dev && dev->power.subsys_data)) @@ -1588,24 +1672,118 @@ int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td) pm_runtime_disable(dev); device_pm_lock(); - pdd = dev->power.subsys_data->domain_data; - if (pdd) { - struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); + spin_lock_irq(&dev->power.lock); - gpd_data->ops = (struct gpd_dev_ops){ 0 }; + if (dev->power.subsys_data->domain_data) { + gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); + gpd_data->ops = (struct gpd_dev_ops){ NULL }; if (clear_td) gpd_data->td = (struct gpd_timing_data){ 0 }; + + if (--gpd_data->refcount == 0) { + dev->power.subsys_data->domain_data = NULL; + remove = true; + } } else { ret = -EINVAL; } + spin_unlock_irq(&dev->power.lock); + device_pm_unlock(); pm_runtime_enable(dev); - return ret; + if (ret) + return ret; + + dev_pm_put_subsys_data(dev); + if (remove) + __pm_genpd_free_dev_data(dev, gpd_data); + + return 0; } EXPORT_SYMBOL_GPL(__pm_genpd_remove_callbacks); +int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state) +{ + struct cpuidle_driver *cpuidle_drv; + struct gpd_cpu_data *cpu_data; + struct cpuidle_state *idle_state; + int ret = 0; + + if (IS_ERR_OR_NULL(genpd) || state < 0) + return -EINVAL; + + genpd_acquire_lock(genpd); + + if (genpd->cpu_data) { + ret = -EEXIST; + goto out; + } + cpu_data = kzalloc(sizeof(*cpu_data), GFP_KERNEL); + if (!cpu_data) { + ret = -ENOMEM; + goto out; + } + cpuidle_drv = cpuidle_driver_ref(); + if (!cpuidle_drv) { + ret = -ENODEV; + goto out; + } + if (cpuidle_drv->state_count <= state) { + ret = -EINVAL; + goto err; + } + idle_state = &cpuidle_drv->states[state]; + if (!idle_state->disabled) { + ret = -EAGAIN; + goto err; + } + cpu_data->idle_state = idle_state; + cpu_data->saved_exit_latency = idle_state->exit_latency; + genpd->cpu_data = cpu_data; + genpd_recalc_cpu_exit_latency(genpd); + + out: + genpd_release_lock(genpd); + return ret; + + err: + cpuidle_driver_unref(); + goto out; +} + +int genpd_detach_cpuidle(struct generic_pm_domain *genpd) +{ + struct gpd_cpu_data *cpu_data; + struct cpuidle_state *idle_state; + int ret = 0; + + if (IS_ERR_OR_NULL(genpd)) + return -EINVAL; + + genpd_acquire_lock(genpd); + + cpu_data = genpd->cpu_data; + if (!cpu_data) { + ret = -ENODEV; + goto out; + } + idle_state = cpu_data->idle_state; + if (!idle_state->disabled) { + ret = -EAGAIN; + goto out; + } + idle_state->exit_latency = cpu_data->saved_exit_latency; + cpuidle_driver_unref(); + genpd->cpu_data = NULL; + kfree(cpu_data); + + out: + genpd_release_lock(genpd); + return ret; +} + /* Default device callbacks for generic PM domains. */ /** @@ -1615,16 +1793,24 @@ EXPORT_SYMBOL_GPL(__pm_genpd_remove_callbacks); static int pm_genpd_default_save_state(struct device *dev) { int (*cb)(struct device *__dev); - struct device_driver *drv = dev->driver; cb = dev_gpd_data(dev)->ops.save_state; if (cb) return cb(dev); - if (drv && drv->pm && drv->pm->runtime_suspend) - return drv->pm->runtime_suspend(dev); + if (dev->type && dev->type->pm) + cb = dev->type->pm->runtime_suspend; + else if (dev->class && dev->class->pm) + cb = dev->class->pm->runtime_suspend; + else if (dev->bus && dev->bus->pm) + cb = dev->bus->pm->runtime_suspend; + else + cb = NULL; - return 0; + if (!cb && dev->driver && dev->driver->pm) + cb = dev->driver->pm->runtime_suspend; + + return cb ? cb(dev) : 0; } /** @@ -1634,16 +1820,24 @@ static int pm_genpd_default_save_state(struct device *dev) static int pm_genpd_default_restore_state(struct device *dev) { int (*cb)(struct device *__dev); - struct device_driver *drv = dev->driver; cb = dev_gpd_data(dev)->ops.restore_state; if (cb) return cb(dev); - if (drv && drv->pm && drv->pm->runtime_resume) - return drv->pm->runtime_resume(dev); + if (dev->type && dev->type->pm) + cb = dev->type->pm->runtime_resume; + else if (dev->class && dev->class->pm) + cb = dev->class->pm->runtime_resume; + else if (dev->bus && dev->bus->pm) + cb = dev->bus->pm->runtime_resume; + else + cb = NULL; - return 0; + if (!cb && dev->driver && dev->driver->pm) + cb = dev->driver->pm->runtime_resume; + + return cb ? cb(dev) : 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index efa9a2ca30e7..d6a533e68e0f 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -281,6 +281,7 @@ static void poll_idle_init(struct cpuidle_driver *drv) state->power_usage = -1; state->flags = 0; state->enter = poll_idle; + state->disabled = false; } #else static void poll_idle_init(struct cpuidle_driver *drv) {} diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 8391d93f57d5..5b1f2c372c1f 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -281,6 +281,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev) * unless the timer is happening really really soon. */ if (data->expected_us > 5 && + !drv->states[CPUIDLE_DRIVER_STATE_START].disabled && dev->states_usage[CPUIDLE_DRIVER_STATE_START].disable == 0) data->last_state_idx = CPUIDLE_DRIVER_STATE_START; @@ -292,7 +293,7 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev) struct cpuidle_state *s = &drv->states[i]; struct cpuidle_state_usage *su = &dev->states_usage[i]; - if (su->disable) + if (s->disabled || su->disable) continue; if (s->target_residency > data->predicted_us) continue; diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index ca6cdf55eb18..89dcd30ac8ea 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -47,6 +47,7 @@ struct cpuidle_state { unsigned int exit_latency; /* in US */ int power_usage; /* in mW */ unsigned int target_residency; /* in US */ + bool disabled; /* disabled on all CPUs */ int (*enter) (struct cpuidle_device *dev, struct cpuidle_driver *drv, diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 30f794eb3826..a7d6172922d4 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -15,6 +15,7 @@ #include <linux/err.h> #include <linux/of.h> #include <linux/notifier.h> +#include <linux/cpuidle.h> enum gpd_status { GPD_STATE_ACTIVE = 0, /* PM domain is active */ @@ -45,6 +46,11 @@ struct gpd_dev_ops { bool (*active_wakeup)(struct device *dev); }; +struct gpd_cpu_data { + unsigned int saved_exit_latency; + struct cpuidle_state *idle_state; +}; + struct generic_pm_domain { struct dev_pm_domain domain; /* PM domain operations */ struct list_head gpd_list_node; /* Node in the global PM domains list */ @@ -75,6 +81,7 @@ struct generic_pm_domain { bool max_off_time_changed; bool cached_power_down_ok; struct device_node *of_node; /* Node in device tree */ + struct gpd_cpu_data *cpu_data; }; static inline struct generic_pm_domain *pd_to_genpd(struct dev_pm_domain *pd) @@ -105,6 +112,7 @@ struct generic_pm_domain_data { struct gpd_timing_data td; struct notifier_block nb; struct mutex lock; + unsigned int refcount; bool need_restore; bool always_on; }; @@ -155,6 +163,8 @@ extern int pm_genpd_add_callbacks(struct device *dev, struct gpd_dev_ops *ops, struct gpd_timing_data *td); extern int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td); +extern int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state); +extern int genpd_detach_cpuidle(struct generic_pm_domain *genpd); extern void pm_genpd_init(struct generic_pm_domain *genpd, struct dev_power_governor *gov, bool is_off); @@ -211,6 +221,14 @@ static inline int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td) { return -ENOSYS; } +static inline int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int st) +{ + return -ENOSYS; +} +static inline int genpd_detach_cpuidle(struct generic_pm_domain *genpd) +{ + return -ENOSYS; +} static inline void pm_genpd_init(struct generic_pm_domain *genpd, struct dev_power_governor *gov, bool is_off) { |