From 1c33be57496d927ce05b2513ff0c108f69db4345 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 12 Apr 2012 02:56:10 -0400 Subject: ARM: b.L: core switcher code This is the core code implementing big.LITTLE switcher functionality. Rationale for this code is available here: http://lwn.net/Articles/481055/ The main entry point for a switch request is: void bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) If the calling CPU is not the wanted one, this wrapper takes care of sending the request to the appropriate CPU with schedule_work_on(). At the moment the core switch operation is handled by bL_switch_to() which must be called on the CPU for which a switch is requested. What this code does: * Return early if the current cluster is the wanted one. * Close the gate in the kernel entry vector for both the inbound and outbound CPUs. * Wake up the inbound CPU so it can perform its reset sequence in parallel up to the kernel entry vector gate. * Migrate all interrupts in the GIC targeting the outbound CPU interface to the inbound CPU interface, including SGIs. This is performed by gic_migrate_target() in drivers/irqchip/irq-gic.c. * Call cpu_pm_enter() which takes care of flushing the VFP state to RAM and save the CPU interface config from the GIC to RAM. * Modify the cpu_logical_map to refer to the inbound physical CPU. * Call cpu_suspend() which saves the CPU state (general purpose registers, page table address) onto the stack and store the resulting stack pointer in an array indexed by the updated cpu_logical_map, then call the provided shutdown function. This happens in arch/arm/kernel/sleep.S. At this point, the provided shutdown function executed by the outbound CPU ungates the inbound CPU. Therefore the inbound CPU: * Picks up the saved stack pointer in the array indexed by its MPIDR in arch/arm/kernel/sleep.S. * The MMU and caches are re-enabled using the saved state on the provided stack, just like if this was a resume operation from a suspended state. * Then cpu_suspend() returns, although this is on the inbound CPU rather than the outbound CPU which called it initially. * The function cpu_pm_exit() is called which effect is to restore the CPU interface state in the GIC using the state previously saved by the outbound CPU. * Exit of bL_switch_to() to resume normal kernel execution on the new CPU. However, the outbound CPU is potentially still running in parallel while the inbound CPU is resuming normal kernel execution, hence we need per CPU stack isolation to execute bL_do_switch(). After the outbound CPU has ungated the inbound CPU, it calls mcpm_cpu_power_down() to: * Clean its L1 cache. * If it is the last CPU still alive in its cluster (last man standing), it also cleans its L2 cache and disables cache snooping from the other cluster. * Power down the CPU (or whole cluster). Code called from bL_do_switch() might end up referencing 'current' for some reasons. However, 'current' is derived from the stack pointer. With any arbitrary stack, the returned value for 'current' and any dereferenced values through it are just random garbage which may lead to segmentation faults. The active page table during the execution of bL_do_switch() is also a problem. There is no guarantee that the inbound CPU won't destroy the corresponding task which would free the attached page table while the outbound CPU is still running and relying on it. To solve both issues, we borrow some of the task space belonging to the init/idle task which, by its nature, is lightly used and therefore is unlikely to clash with our usage. The init task is also never going away. Right now the logical CPU number is assumed to be equivalent to the physical CPU number within each cluster. The kernel should also be booted with only one cluster active. These limitations will be lifted eventually. Signed-off-by: Nicolas Pitre --- arch/arm/common/Makefile | 1 + arch/arm/common/bL_switcher.c | 241 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 arch/arm/common/bL_switcher.c (limited to 'arch/arm/common') diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 8c60f473e976..2586240d5a45 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -17,3 +17,4 @@ obj-$(CONFIG_MCPM) += mcpm_head.o mcpm_entry.o mcpm_platsmp.o vlock.o AFLAGS_mcpm_head.o := -march=armv7-a AFLAGS_vlock.o := -march=armv7-a obj-$(CONFIG_TI_PRIV_EDMA) += edma.o +obj-$(CONFIG_BL_SWITCHER) += bL_switcher.o diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c new file mode 100644 index 000000000000..f8f2e96b1466 --- /dev/null +++ b/arch/arm/common/bL_switcher.c @@ -0,0 +1,241 @@ +/* + * arch/arm/common/bL_switcher.c -- big.LITTLE cluster switcher core driver + * + * Created by: Nicolas Pitre, March 2012 + * Copyright: (C) 2012-2013 Linaro Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +/* + * Use our own MPIDR accessors as the generic ones in asm/cputype.h have + * __attribute_const__ and we don't want the compiler to assume any + * constness here as the value _does_ change along some code paths. + */ + +static int read_mpidr(void) +{ + unsigned int id; + asm volatile ("mrc p15, 0, %0, c0, c0, 5" : "=r" (id)); + return id & MPIDR_HWID_BITMASK; +} + +/* + * bL switcher core code. + */ + +static void bL_do_switch(void *_unused) +{ + unsigned mpidr, cpuid, clusterid, ob_cluster, ib_cluster; + + /* + * We now have a piece of stack borrowed from the init task's. + * Let's also switch to init_mm right away to match it. + */ + cpu_switch_mm(init_mm.pgd, &init_mm); + + pr_debug("%s\n", __func__); + + mpidr = read_mpidr(); + cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); + clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); + ob_cluster = clusterid; + ib_cluster = clusterid ^ 1; + + /* + * Our state has been saved at this point. Let's release our + * inbound CPU. + */ + mcpm_set_entry_vector(cpuid, ib_cluster, cpu_resume); + sev(); + + /* + * From this point, we must assume that our counterpart CPU might + * have taken over in its parallel world already, as if execution + * just returned from cpu_suspend(). It is therefore important to + * be very careful not to make any change the other guy is not + * expecting. This is why we need stack isolation. + * + * Fancy under cover tasks could be performed here. For now + * we have none. + */ + + /* Let's put ourself down. */ + mcpm_cpu_power_down(); + + /* should never get here */ + BUG(); +} + +/* + * Stack isolation. To ensure 'current' remains valid, we just borrow + * a slice of the init/idle task which should be fairly lightly used. + * The borrowed area starts just above the thread_info structure located + * at the very bottom of the stack, aligned to a cache line. + */ +#define STACK_SIZE 256 +extern void call_with_stack(void (*fn)(void *), void *arg, void *sp); +static int bL_switchpoint(unsigned long _arg) +{ + unsigned int mpidr = read_mpidr(); + unsigned int cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); + unsigned int clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); + unsigned int cpu_index = cpuid + clusterid * MAX_CPUS_PER_CLUSTER; + void *stack = &init_thread_info + 1; + stack = PTR_ALIGN(stack, L1_CACHE_BYTES); + stack += cpu_index * STACK_SIZE + STACK_SIZE; + call_with_stack(bL_do_switch, (void *)_arg, stack); + BUG(); +} + +/* + * Generic switcher interface + */ + +/* + * bL_switch_to - Switch to a specific cluster for the current CPU + * @new_cluster_id: the ID of the cluster to switch to. + * + * This function must be called on the CPU to be switched. + * Returns 0 on success, else a negative status code. + */ +static int bL_switch_to(unsigned int new_cluster_id) +{ + unsigned int mpidr, cpuid, clusterid, ob_cluster, ib_cluster, this_cpu; + int ret; + + mpidr = read_mpidr(); + cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); + clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); + ob_cluster = clusterid; + ib_cluster = clusterid ^ 1; + + if (new_cluster_id == clusterid) + return 0; + + pr_debug("before switch: CPU %d in cluster %d\n", cpuid, clusterid); + + /* Close the gate for our entry vectors */ + mcpm_set_entry_vector(cpuid, ob_cluster, NULL); + mcpm_set_entry_vector(cpuid, ib_cluster, NULL); + + /* + * Let's wake up the inbound CPU now in case it requires some delay + * to come online, but leave it gated in our entry vector code. + */ + ret = mcpm_cpu_power_up(cpuid, ib_cluster); + if (ret) { + pr_err("%s: mcpm_cpu_power_up() returned %d\n", __func__, ret); + return ret; + } + + /* + * From this point we are entering the switch critical zone + * and can't take any interrupts anymore. + */ + local_irq_disable(); + local_fiq_disable(); + + this_cpu = smp_processor_id(); + + /* redirect GIC's SGIs to our counterpart */ + gic_migrate_target(cpuid + ib_cluster*4); + + /* + * Raise a SGI on the inbound CPU to make sure it doesn't stall + * in a possible WFI, such as in mcpm_power_down(). + */ + arch_send_wakeup_ipi_mask(cpumask_of(this_cpu)); + + ret = cpu_pm_enter(); + + /* we can not tolerate errors at this point */ + if (ret) + panic("%s: cpu_pm_enter() returned %d\n", __func__, ret); + + /* Flip the cluster in the CPU logical map for this CPU. */ + cpu_logical_map(this_cpu) ^= (1 << 8); + + /* Let's do the actual CPU switch. */ + ret = cpu_suspend(0, bL_switchpoint); + if (ret > 0) + panic("%s: cpu_suspend() returned %d\n", __func__, ret); + + /* We are executing on the inbound CPU at this point */ + mpidr = read_mpidr(); + cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); + clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); + pr_debug("after switch: CPU %d in cluster %d\n", cpuid, clusterid); + BUG_ON(clusterid != ib_cluster); + + mcpm_cpu_powered_up(); + + ret = cpu_pm_exit(); + + local_fiq_enable(); + local_irq_enable(); + + if (ret) + pr_err("%s exiting with error %d\n", __func__, ret); + return ret; +} + +struct switch_args { + unsigned int cluster; + struct work_struct work; +}; + +static void __bL_switch_to(struct work_struct *work) +{ + struct switch_args *args = container_of(work, struct switch_args, work); + bL_switch_to(args->cluster); +} + +/* + * bL_switch_request - Switch to a specific cluster for the given CPU + * + * @cpu: the CPU to switch + * @new_cluster_id: the ID of the cluster to switch to. + * + * This function causes a cluster switch on the given CPU. If the given + * CPU is the same as the calling CPU then the switch happens right away. + * Otherwise the request is put on a work queue to be scheduled on the + * remote CPU. + */ +void bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) +{ + unsigned int this_cpu = get_cpu(); + struct switch_args args; + + if (cpu == this_cpu) { + bL_switch_to(new_cluster_id); + put_cpu(); + return; + } + put_cpu(); + + args.cluster = new_cluster_id; + INIT_WORK_ONSTACK(&args.work, __bL_switch_to); + schedule_work_on(cpu, &args.work); + flush_work(&args.work); +} +EXPORT_SYMBOL_GPL(bL_switch_request); -- cgit v1.2.3 From 3f09d4799ecc076cccc11ab2333a36ec849d24f5 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Wed, 16 May 2012 15:55:54 +0100 Subject: ARM: bL_switcher: add clockevent save/restore support Per-CPU timers that are shutdown when a CPU is switched over must be disabled upon switching and reprogrammed on the inbound CPU by relying on the clock events management API. save/restore sequence is executed with irqs disabled as mandated by the clock events API. The next_event is an absolute time, hence, when the inbound CPU resumes, if the timer has expired the min delta is forced into the tick device to fire after few cycles. This patch adds switching support for clock events that are per-CPU and have to be migrated when a switch takes place; the cpumask of the clock event device is checked against the cpumask of the current cpu, and if they match, the clockevent device mode is saved and it is put in shutdown mode. Resume code reprogrammes the tick device accordingly. Tested on A15/A7 fast models and architected timers. Signed-off-by: Lorenzo Pieralisi Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index f8f2e96b1466..ca04b5384bb0 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -15,7 +15,11 @@ #include #include #include +#include #include +#include +#include +#include #include #include #include @@ -121,6 +125,8 @@ static int bL_switchpoint(unsigned long _arg) static int bL_switch_to(unsigned int new_cluster_id) { unsigned int mpidr, cpuid, clusterid, ob_cluster, ib_cluster, this_cpu; + struct tick_device *tdev; + enum clock_event_mode tdev_mode; int ret; mpidr = read_mpidr(); @@ -166,6 +172,14 @@ static int bL_switch_to(unsigned int new_cluster_id) */ arch_send_wakeup_ipi_mask(cpumask_of(this_cpu)); + tdev = tick_get_device(this_cpu); + if (tdev && !cpumask_equal(tdev->evtdev->cpumask, cpumask_of(this_cpu))) + tdev = NULL; + if (tdev) { + tdev_mode = tdev->evtdev->mode; + clockevents_set_mode(tdev->evtdev, CLOCK_EVT_MODE_SHUTDOWN); + } + ret = cpu_pm_enter(); /* we can not tolerate errors at this point */ @@ -191,6 +205,12 @@ static int bL_switch_to(unsigned int new_cluster_id) ret = cpu_pm_exit(); + if (tdev) { + clockevents_set_mode(tdev->evtdev, tdev_mode); + clockevents_program_event(tdev->evtdev, + tdev->evtdev->next_event, 1); + } + local_fiq_enable(); local_irq_enable(); -- cgit v1.2.3 From 71ce1deeff8f9341ae3b21983e9bdde28e8c96fe Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Fri, 26 Oct 2012 02:36:17 -0400 Subject: ARM: bL_switcher: move to dedicated threads rather than workqueues The workqueues are problematic as they may be contended. They can't be scheduled with top priority either. Also the optimization in bL_switch_request() to skip the workqueue entirely when the target CPU and the calling CPU were the same didn't allow for bL_switch_request() to be called from atomic context, as might be the case for some cpufreq drivers. Let's move to dedicated kthreads instead. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 101 ++++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 23 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index ca04b5384bb0..407c4cc64c0b 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -15,8 +15,10 @@ #include #include #include +#include #include -#include +#include +#include #include #include #include @@ -219,15 +221,48 @@ static int bL_switch_to(unsigned int new_cluster_id) return ret; } -struct switch_args { - unsigned int cluster; - struct work_struct work; +struct bL_thread { + struct task_struct *task; + wait_queue_head_t wq; + int wanted_cluster; }; -static void __bL_switch_to(struct work_struct *work) +static struct bL_thread bL_threads[NR_CPUS]; + +static int bL_switcher_thread(void *arg) +{ + struct bL_thread *t = arg; + struct sched_param param = { .sched_priority = 1 }; + int cluster; + + sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); + + do { + if (signal_pending(current)) + flush_signals(current); + wait_event_interruptible(t->wq, + t->wanted_cluster != -1 || + kthread_should_stop()); + cluster = xchg(&t->wanted_cluster, -1); + if (cluster != -1) + bL_switch_to(cluster); + } while (!kthread_should_stop()); + + return 0; +} + +static struct task_struct * __init bL_switcher_thread_create(int cpu, void *arg) { - struct switch_args *args = container_of(work, struct switch_args, work); - bL_switch_to(args->cluster); + struct task_struct *task; + + task = kthread_create_on_node(bL_switcher_thread, arg, + cpu_to_node(cpu), "kswitcher_%d", cpu); + if (!IS_ERR(task)) { + kthread_bind(task, cpu); + wake_up_process(task); + } else + pr_err("%s failed for CPU %d\n", __func__, cpu); + return task; } /* @@ -236,26 +271,46 @@ static void __bL_switch_to(struct work_struct *work) * @cpu: the CPU to switch * @new_cluster_id: the ID of the cluster to switch to. * - * This function causes a cluster switch on the given CPU. If the given - * CPU is the same as the calling CPU then the switch happens right away. - * Otherwise the request is put on a work queue to be scheduled on the - * remote CPU. + * This function causes a cluster switch on the given CPU by waking up + * the appropriate switcher thread. This function may or may not return + * before the switch has occurred. */ -void bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) +int bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) { - unsigned int this_cpu = get_cpu(); - struct switch_args args; + struct bL_thread *t; - if (cpu == this_cpu) { - bL_switch_to(new_cluster_id); - put_cpu(); - return; + if (cpu >= ARRAY_SIZE(bL_threads)) { + pr_err("%s: cpu %d out of bounds\n", __func__, cpu); + return -EINVAL; } - put_cpu(); - args.cluster = new_cluster_id; - INIT_WORK_ONSTACK(&args.work, __bL_switch_to); - schedule_work_on(cpu, &args.work); - flush_work(&args.work); + t = &bL_threads[cpu]; + if (IS_ERR(t->task)) + return PTR_ERR(t->task); + if (!t->task) + return -ESRCH; + + t->wanted_cluster = new_cluster_id; + wake_up(&t->wq); + return 0; } EXPORT_SYMBOL_GPL(bL_switch_request); + +static int __init bL_switcher_init(void) +{ + int cpu; + + pr_info("big.LITTLE switcher initializing\n"); + + for_each_online_cpu(cpu) { + struct bL_thread *t = &bL_threads[cpu]; + init_waitqueue_head(&t->wq); + t->wanted_cluster = -1; + t->task = bL_switcher_thread_create(cpu, t); + } + + pr_info("big.LITTLE switcher initialized\n"); + return 0; +} + +late_initcall(bL_switcher_init); -- cgit v1.2.3 From c052de2693498bee6ff2e1b5bcd32309c4f8780b Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 27 Nov 2012 15:55:33 -0500 Subject: ARM: bL_switcher: simplify stack isolation We now have a dedicated thread for each logical CPU. That's plenty of stack space for our needs. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 407c4cc64c0b..4caca71c906f 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -53,12 +53,6 @@ static void bL_do_switch(void *_unused) { unsigned mpidr, cpuid, clusterid, ob_cluster, ib_cluster; - /* - * We now have a piece of stack borrowed from the init task's. - * Let's also switch to init_mm right away to match it. - */ - cpu_switch_mm(init_mm.pgd, &init_mm); - pr_debug("%s\n", __func__); mpidr = read_mpidr(); @@ -93,22 +87,21 @@ static void bL_do_switch(void *_unused) } /* - * Stack isolation. To ensure 'current' remains valid, we just borrow - * a slice of the init/idle task which should be fairly lightly used. - * The borrowed area starts just above the thread_info structure located - * at the very bottom of the stack, aligned to a cache line. + * Stack isolation. To ensure 'current' remains valid, we just use another + * piece of our thread's stack space which should be fairly lightly used. + * The selected area starts just above the thread_info structure located + * at the very bottom of the stack, aligned to a cache line, and indexed + * with the cluster number. */ -#define STACK_SIZE 256 +#define STACK_SIZE 512 extern void call_with_stack(void (*fn)(void *), void *arg, void *sp); static int bL_switchpoint(unsigned long _arg) { unsigned int mpidr = read_mpidr(); - unsigned int cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); unsigned int clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); - unsigned int cpu_index = cpuid + clusterid * MAX_CPUS_PER_CLUSTER; - void *stack = &init_thread_info + 1; + void *stack = current_thread_info() + 1; stack = PTR_ALIGN(stack, L1_CACHE_BYTES); - stack += cpu_index * STACK_SIZE + STACK_SIZE; + stack += clusterid * STACK_SIZE + STACK_SIZE; call_with_stack(bL_do_switch, (void *)_arg, stack); BUG(); } -- cgit v1.2.3 From 9797a0e95ead7bfe52260c369ee9fe6ba445afaf Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Wed, 21 Nov 2012 11:53:27 -0500 Subject: ARM: bL_switcher: hot-unplug half of the available CPUs In a regular kernel configuration, all the CPUs are initially available. But the switcher execution model uses half of them at any time. Instead of hacking the DTB to remove half of the CPUs, let's remove them at run time and make sure we still have a working switcher configuration. This way, the same DTB can be used whether or not the switcher is used. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 78 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 4caca71c906f..50e95d894e35 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -289,18 +289,94 @@ int bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) } EXPORT_SYMBOL_GPL(bL_switch_request); +/* + * Activation and configuration code. + */ + +static cpumask_t bL_switcher_removed_logical_cpus; + +static void __init bL_switcher_restore_cpus(void) +{ + int i; + + for_each_cpu(i, &bL_switcher_removed_logical_cpus) + cpu_up(i); +} + +static int __init bL_switcher_halve_cpus(void) +{ + int cpu, cluster, i, ret; + cpumask_t cluster_mask[2], common_mask; + + cpumask_clear(&bL_switcher_removed_logical_cpus); + cpumask_clear(&cluster_mask[0]); + cpumask_clear(&cluster_mask[1]); + + for_each_online_cpu(i) { + cpu = cpu_logical_map(i) & 0xff; + cluster = (cpu_logical_map(i) >> 8) & 0xff; + if (cluster >= 2) { + pr_err("%s: only dual cluster systems are supported\n", __func__); + return -EINVAL; + } + cpumask_set_cpu(cpu, &cluster_mask[cluster]); + } + + if (!cpumask_and(&common_mask, &cluster_mask[0], &cluster_mask[1])) { + pr_err("%s: no common set of CPUs\n", __func__); + return -EINVAL; + } + + for_each_online_cpu(i) { + cpu = cpu_logical_map(i) & 0xff; + cluster = (cpu_logical_map(i) >> 8) & 0xff; + + if (cpumask_test_cpu(cpu, &common_mask)) { + /* + * We keep only those logical CPUs which number + * is equal to their physical CPU number. This is + * not perfect but good enough for now. + */ + if (cpu == i) + continue; + } + + ret = cpu_down(i); + if (ret) { + bL_switcher_restore_cpus(); + return ret; + } + cpumask_set_cpu(i, &bL_switcher_removed_logical_cpus); + } + + return 0; +} + static int __init bL_switcher_init(void) { - int cpu; + int cpu, ret; pr_info("big.LITTLE switcher initializing\n"); + if (MAX_NR_CLUSTERS != 2) { + pr_err("%s: only dual cluster systems are supported\n", __func__); + return -EINVAL; + } + + cpu_hotplug_driver_lock(); + ret = bL_switcher_halve_cpus(); + if (ret) { + cpu_hotplug_driver_unlock(); + return ret; + } + for_each_online_cpu(cpu) { struct bL_thread *t = &bL_threads[cpu]; init_waitqueue_head(&t->wq); t->wanted_cluster = -1; t->task = bL_switcher_thread_create(cpu, t); } + cpu_hotplug_driver_unlock(); pr_info("big.LITTLE switcher initialized\n"); return 0; -- cgit v1.2.3 From ed96762e3241f57aa812977cf1920d3ee0363f4d Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 5 Jul 2012 21:33:26 -0400 Subject: ARM: bL_switcher: do not hardcode GIC IDs in the code Currently, GIC IDs are hardcoded making the code dependent on the 4+4 b.L configuration. Let's allow for GIC IDs to be discovered upon switcher initialization to support other b.L configurations such as the 1+1 one, or 2+3 as on the VExpress TC2. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 50e95d894e35..1c2e5bcfb1f7 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -110,6 +110,8 @@ static int bL_switchpoint(unsigned long _arg) * Generic switcher interface */ +static unsigned int bL_gic_id[MAX_CPUS_PER_CLUSTER][MAX_NR_CLUSTERS]; + /* * bL_switch_to - Switch to a specific cluster for the current CPU * @new_cluster_id: the ID of the cluster to switch to. @@ -159,7 +161,7 @@ static int bL_switch_to(unsigned int new_cluster_id) this_cpu = smp_processor_id(); /* redirect GIC's SGIs to our counterpart */ - gic_migrate_target(cpuid + ib_cluster*4); + gic_migrate_target(bL_gic_id[cpuid][ib_cluster]); /* * Raise a SGI on the inbound CPU to make sure it doesn't stall @@ -332,6 +334,16 @@ static int __init bL_switcher_halve_cpus(void) cluster = (cpu_logical_map(i) >> 8) & 0xff; if (cpumask_test_cpu(cpu, &common_mask)) { + /* Let's take note of the GIC ID for this CPU */ + int gic_id = gic_get_cpu_id(i); + if (gic_id < 0) { + pr_err("%s: bad GIC ID for CPU %d\n", __func__, i); + return -EINVAL; + } + bL_gic_id[cpu][cluster] = gic_id; + pr_info("GIC ID for CPU %u cluster %u is %u\n", + cpu, cluster, gic_id); + /* * We keep only those logical CPUs which number * is equal to their physical CPU number. This is -- cgit v1.2.3 From 6b7437aed1568076cefa4d42747b1515dcb848db Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 22 Nov 2012 00:05:07 -0500 Subject: ARM: bL_switcher: ability to enable and disable the switcher via sysfs The /sys/kernel/bL_switcher/enable file allows to enable or disable the switcher by writing 1 or 0 to it respectively. It is still enabled by default on boot. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 171 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 160 insertions(+), 11 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 1c2e5bcfb1f7..395f60f6292b 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -220,6 +221,7 @@ struct bL_thread { struct task_struct *task; wait_queue_head_t wq; int wanted_cluster; + struct completion started; }; static struct bL_thread bL_threads[NR_CPUS]; @@ -231,6 +233,7 @@ static int bL_switcher_thread(void *arg) int cluster; sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); + complete(&t->started); do { if (signal_pending(current)) @@ -246,7 +249,7 @@ static int bL_switcher_thread(void *arg) return 0; } -static struct task_struct * __init bL_switcher_thread_create(int cpu, void *arg) +static struct task_struct *bL_switcher_thread_create(int cpu, void *arg) { struct task_struct *task; @@ -295,9 +298,11 @@ EXPORT_SYMBOL_GPL(bL_switch_request); * Activation and configuration code. */ +static unsigned int bL_switcher_active; +static unsigned int bL_switcher_cpu_original_cluster[MAX_CPUS_PER_CLUSTER]; static cpumask_t bL_switcher_removed_logical_cpus; -static void __init bL_switcher_restore_cpus(void) +static void bL_switcher_restore_cpus(void) { int i; @@ -305,7 +310,7 @@ static void __init bL_switcher_restore_cpus(void) cpu_up(i); } -static int __init bL_switcher_halve_cpus(void) +static int bL_switcher_halve_cpus(void) { int cpu, cluster, i, ret; cpumask_t cluster_mask[2], common_mask; @@ -349,8 +354,10 @@ static int __init bL_switcher_halve_cpus(void) * is equal to their physical CPU number. This is * not perfect but good enough for now. */ - if (cpu == i) + if (cpu == i) { + bL_switcher_cpu_original_cluster[cpu] = cluster; continue; + } } ret = cpu_down(i); @@ -364,18 +371,18 @@ static int __init bL_switcher_halve_cpus(void) return 0; } -static int __init bL_switcher_init(void) +static int bL_switcher_enable(void) { int cpu, ret; - pr_info("big.LITTLE switcher initializing\n"); - - if (MAX_NR_CLUSTERS != 2) { - pr_err("%s: only dual cluster systems are supported\n", __func__); - return -EINVAL; + cpu_hotplug_driver_lock(); + if (bL_switcher_active) { + cpu_hotplug_driver_unlock(); + return 0; } - cpu_hotplug_driver_lock(); + pr_info("big.LITTLE switcher initializing\n"); + ret = bL_switcher_halve_cpus(); if (ret) { cpu_hotplug_driver_unlock(); @@ -385,13 +392,155 @@ static int __init bL_switcher_init(void) for_each_online_cpu(cpu) { struct bL_thread *t = &bL_threads[cpu]; init_waitqueue_head(&t->wq); + init_completion(&t->started); t->wanted_cluster = -1; t->task = bL_switcher_thread_create(cpu, t); } + + bL_switcher_active = 1; cpu_hotplug_driver_unlock(); pr_info("big.LITTLE switcher initialized\n"); return 0; } +#ifdef CONFIG_SYSFS + +static void bL_switcher_disable(void) +{ + unsigned int cpu, cluster, i; + struct bL_thread *t; + struct task_struct *task; + + cpu_hotplug_driver_lock(); + if (!bL_switcher_active) { + cpu_hotplug_driver_unlock(); + return; + } + bL_switcher_active = 0; + + /* + * To deactivate the switcher, we must shut down the switcher + * threads to prevent any other requests from being accepted. + * Then, if the final cluster for given logical CPU is not the + * same as the original one, we'll recreate a switcher thread + * just for the purpose of switching the CPU back without any + * possibility for interference from external requests. + */ + for_each_online_cpu(cpu) { + BUG_ON(cpu != (cpu_logical_map(cpu) & 0xff)); + t = &bL_threads[cpu]; + task = t->task; + t->task = NULL; + if (!task || IS_ERR(task)) + continue; + kthread_stop(task); + /* no more switch may happen on this CPU at this point */ + cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 1); + if (cluster == bL_switcher_cpu_original_cluster[cpu]) + continue; + init_completion(&t->started); + t->wanted_cluster = bL_switcher_cpu_original_cluster[cpu]; + task = bL_switcher_thread_create(cpu, t); + if (!IS_ERR(task)) { + wait_for_completion(&t->started); + kthread_stop(task); + cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 1); + if (cluster == bL_switcher_cpu_original_cluster[cpu]) + continue; + } + /* If execution gets here, we're in trouble. */ + pr_crit("%s: unable to restore original cluster for CPU %d\n", + __func__, cpu); + for_each_cpu(i, &bL_switcher_removed_logical_cpus) { + if ((cpu_logical_map(i) & 0xff) != cpu) + continue; + pr_crit("%s: CPU %d can't be restored\n", + __func__, i); + cpumask_clear_cpu(i, &bL_switcher_removed_logical_cpus); + break; + } + } + + bL_switcher_restore_cpus(); + cpu_hotplug_driver_unlock(); +} + +static ssize_t bL_switcher_active_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", bL_switcher_active); +} + +static ssize_t bL_switcher_active_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int ret; + + switch (buf[0]) { + case '0': + bL_switcher_disable(); + ret = 0; + break; + case '1': + ret = bL_switcher_enable(); + break; + default: + ret = -EINVAL; + } + + return (ret >= 0) ? count : ret; +} + +static struct kobj_attribute bL_switcher_active_attr = + __ATTR(active, 0644, bL_switcher_active_show, bL_switcher_active_store); + +static struct attribute *bL_switcher_attrs[] = { + &bL_switcher_active_attr.attr, + NULL, +}; + +static struct attribute_group bL_switcher_attr_group = { + .attrs = bL_switcher_attrs, +}; + +static struct kobject *bL_switcher_kobj; + +static int __init bL_switcher_sysfs_init(void) +{ + int ret; + + bL_switcher_kobj = kobject_create_and_add("bL_switcher", kernel_kobj); + if (!bL_switcher_kobj) + return -ENOMEM; + ret = sysfs_create_group(bL_switcher_kobj, &bL_switcher_attr_group); + if (ret) + kobject_put(bL_switcher_kobj); + return ret; +} + +#endif /* CONFIG_SYSFS */ + +static int __init bL_switcher_init(void) +{ + int ret; + + if (MAX_NR_CLUSTERS != 2) { + pr_err("%s: only dual cluster systems are supported\n", __func__); + return -EINVAL; + } + + ret = bL_switcher_enable(); + if (ret) + return ret; + +#ifdef CONFIG_SYSFS + ret = bL_switcher_sysfs_init(); + if (ret) + pr_err("%s: unable to create sysfs entry\n", __func__); +#endif + + return 0; +} + late_initcall(bL_switcher_init); -- cgit v1.2.3 From c4821c0575a3b1bf26f100230dc2938297d7043b Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 22 Nov 2012 13:33:35 -0500 Subject: ARM: bL_switcher: add kernel cmdline param to disable the switcher on boot By adding no_bL_switcher to the kernel cmdline string, the switcher won't be activated automatically at boot time. It is still possible to activate it later with: echo 1 > /sys/kernel/bL_switcher/active Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 395f60f6292b..cec825ef392b 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -521,6 +522,9 @@ static int __init bL_switcher_sysfs_init(void) #endif /* CONFIG_SYSFS */ +static bool no_bL_switcher; +core_param(no_bL_switcher, no_bL_switcher, bool, 0644); + static int __init bL_switcher_init(void) { int ret; @@ -530,9 +534,11 @@ static int __init bL_switcher_init(void) return -EINVAL; } - ret = bL_switcher_enable(); - if (ret) - return ret; + if (!no_bL_switcher) { + ret = bL_switcher_enable(); + if (ret) + return ret; + } #ifdef CONFIG_SYSFS ret = bL_switcher_sysfs_init(); -- cgit v1.2.3 From 38c35d4f2e408c369e3030f0717d35ad443d9223 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 13 Jun 2013 23:42:46 -0400 Subject: ARM: bL_switcher: remove assumptions between logical and physical CPUs Up to now, the logical CPU was somehow tied to the physical CPU number within a cluster. This causes problems when forcing the boot CPU to be different from the first enumerated CPU in the device tree creating a discrepancy between logical and physical CPU numbers. Let's make the pairing completely independent from physical CPU numbers. Let's keep only those logical CPUs with same initial CPU cluster to create a uniform scheduler profile without having to modify any of the probed topology and compute capacity data. This has the potential to create a non contiguous CPU numbering space when the switcher is active with potential impact on buggy user space tools. It is however better to fix those tools rather than making the switcher code more intrusive. Signed-off-by: Nicolas Pitre Reviewed-by: Lorenzo Pieralisi --- arch/arm/common/bL_switcher.c | 176 +++++++++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 72 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index cec825ef392b..0e4fb3e5e99c 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -53,21 +53,19 @@ static int read_mpidr(void) static void bL_do_switch(void *_unused) { - unsigned mpidr, cpuid, clusterid, ob_cluster, ib_cluster; + unsigned ib_mpidr, ib_cpu, ib_cluster; pr_debug("%s\n", __func__); - mpidr = read_mpidr(); - cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); - clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); - ob_cluster = clusterid; - ib_cluster = clusterid ^ 1; + ib_mpidr = cpu_logical_map(smp_processor_id()); + ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0); + ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1); /* * Our state has been saved at this point. Let's release our * inbound CPU. */ - mcpm_set_entry_vector(cpuid, ib_cluster, cpu_resume); + mcpm_set_entry_vector(ib_cpu, ib_cluster, cpu_resume); sev(); /* @@ -113,6 +111,7 @@ static int bL_switchpoint(unsigned long _arg) */ static unsigned int bL_gic_id[MAX_CPUS_PER_CLUSTER][MAX_NR_CLUSTERS]; +static int bL_switcher_cpu_pairing[NR_CPUS]; /* * bL_switch_to - Switch to a specific cluster for the current CPU @@ -123,31 +122,38 @@ static unsigned int bL_gic_id[MAX_CPUS_PER_CLUSTER][MAX_NR_CLUSTERS]; */ static int bL_switch_to(unsigned int new_cluster_id) { - unsigned int mpidr, cpuid, clusterid, ob_cluster, ib_cluster, this_cpu; + unsigned int mpidr, this_cpu, that_cpu; + unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster; struct tick_device *tdev; enum clock_event_mode tdev_mode; int ret; - mpidr = read_mpidr(); - cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); - clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); - ob_cluster = clusterid; - ib_cluster = clusterid ^ 1; + this_cpu = smp_processor_id(); + ob_mpidr = read_mpidr(); + ob_cpu = MPIDR_AFFINITY_LEVEL(ob_mpidr, 0); + ob_cluster = MPIDR_AFFINITY_LEVEL(ob_mpidr, 1); + BUG_ON(cpu_logical_map(this_cpu) != ob_mpidr); - if (new_cluster_id == clusterid) + if (new_cluster_id == ob_cluster) return 0; - pr_debug("before switch: CPU %d in cluster %d\n", cpuid, clusterid); + that_cpu = bL_switcher_cpu_pairing[this_cpu]; + ib_mpidr = cpu_logical_map(that_cpu); + ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0); + ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1); + + pr_debug("before switch: CPU %d MPIDR %#x -> %#x\n", + this_cpu, ob_mpidr, ib_mpidr); /* Close the gate for our entry vectors */ - mcpm_set_entry_vector(cpuid, ob_cluster, NULL); - mcpm_set_entry_vector(cpuid, ib_cluster, NULL); + mcpm_set_entry_vector(ob_cpu, ob_cluster, NULL); + mcpm_set_entry_vector(ib_cpu, ib_cluster, NULL); /* * Let's wake up the inbound CPU now in case it requires some delay * to come online, but leave it gated in our entry vector code. */ - ret = mcpm_cpu_power_up(cpuid, ib_cluster); + ret = mcpm_cpu_power_up(ib_cpu, ib_cluster); if (ret) { pr_err("%s: mcpm_cpu_power_up() returned %d\n", __func__, ret); return ret; @@ -160,10 +166,8 @@ static int bL_switch_to(unsigned int new_cluster_id) local_irq_disable(); local_fiq_disable(); - this_cpu = smp_processor_id(); - /* redirect GIC's SGIs to our counterpart */ - gic_migrate_target(bL_gic_id[cpuid][ib_cluster]); + gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]); /* * Raise a SGI on the inbound CPU to make sure it doesn't stall @@ -185,8 +189,9 @@ static int bL_switch_to(unsigned int new_cluster_id) if (ret) panic("%s: cpu_pm_enter() returned %d\n", __func__, ret); - /* Flip the cluster in the CPU logical map for this CPU. */ - cpu_logical_map(this_cpu) ^= (1 << 8); + /* Swap the physical CPUs in the logical map for this logical CPU. */ + cpu_logical_map(this_cpu) = ib_mpidr; + cpu_logical_map(that_cpu) = ob_mpidr; /* Let's do the actual CPU switch. */ ret = cpu_suspend(0, bL_switchpoint); @@ -195,10 +200,8 @@ static int bL_switch_to(unsigned int new_cluster_id) /* We are executing on the inbound CPU at this point */ mpidr = read_mpidr(); - cpuid = MPIDR_AFFINITY_LEVEL(mpidr, 0); - clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); - pr_debug("after switch: CPU %d in cluster %d\n", cpuid, clusterid); - BUG_ON(clusterid != ib_cluster); + pr_debug("after switch: CPU %d MPIDR %#x\n", this_cpu, mpidr); + BUG_ON(mpidr != ib_mpidr); mcpm_cpu_powered_up(); @@ -300,7 +303,7 @@ EXPORT_SYMBOL_GPL(bL_switch_request); */ static unsigned int bL_switcher_active; -static unsigned int bL_switcher_cpu_original_cluster[MAX_CPUS_PER_CLUSTER]; +static unsigned int bL_switcher_cpu_original_cluster[NR_CPUS]; static cpumask_t bL_switcher_removed_logical_cpus; static void bL_switcher_restore_cpus(void) @@ -313,52 +316,86 @@ static void bL_switcher_restore_cpus(void) static int bL_switcher_halve_cpus(void) { - int cpu, cluster, i, ret; - cpumask_t cluster_mask[2], common_mask; - - cpumask_clear(&bL_switcher_removed_logical_cpus); - cpumask_clear(&cluster_mask[0]); - cpumask_clear(&cluster_mask[1]); + int i, j, cluster_0, gic_id, ret; + unsigned int cpu, cluster, mask; + cpumask_t available_cpus; + /* First pass to validate what we have */ + mask = 0; for_each_online_cpu(i) { - cpu = cpu_logical_map(i) & 0xff; - cluster = (cpu_logical_map(i) >> 8) & 0xff; + cpu = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0); + cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1); if (cluster >= 2) { pr_err("%s: only dual cluster systems are supported\n", __func__); return -EINVAL; } - cpumask_set_cpu(cpu, &cluster_mask[cluster]); + if (WARN_ON(cpu >= MAX_CPUS_PER_CLUSTER)) + return -EINVAL; + mask |= (1 << cluster); } - - if (!cpumask_and(&common_mask, &cluster_mask[0], &cluster_mask[1])) { - pr_err("%s: no common set of CPUs\n", __func__); + if (mask != 3) { + pr_err("%s: no CPU pairing possible\n", __func__); return -EINVAL; } - for_each_online_cpu(i) { - cpu = cpu_logical_map(i) & 0xff; - cluster = (cpu_logical_map(i) >> 8) & 0xff; - - if (cpumask_test_cpu(cpu, &common_mask)) { - /* Let's take note of the GIC ID for this CPU */ - int gic_id = gic_get_cpu_id(i); - if (gic_id < 0) { - pr_err("%s: bad GIC ID for CPU %d\n", __func__, i); - return -EINVAL; - } - bL_gic_id[cpu][cluster] = gic_id; - pr_info("GIC ID for CPU %u cluster %u is %u\n", - cpu, cluster, gic_id); - + /* + * Now let's do the pairing. We match each CPU with another CPU + * from a different cluster. To get a uniform scheduling behavior + * without fiddling with CPU topology and compute capacity data, + * we'll use logical CPUs initially belonging to the same cluster. + */ + memset(bL_switcher_cpu_pairing, -1, sizeof(bL_switcher_cpu_pairing)); + cpumask_copy(&available_cpus, cpu_online_mask); + cluster_0 = -1; + for_each_cpu(i, &available_cpus) { + int match = -1; + cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1); + if (cluster_0 == -1) + cluster_0 = cluster; + if (cluster != cluster_0) + continue; + cpumask_clear_cpu(i, &available_cpus); + for_each_cpu(j, &available_cpus) { + cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(j), 1); /* - * We keep only those logical CPUs which number - * is equal to their physical CPU number. This is - * not perfect but good enough for now. + * Let's remember the last match to create "odd" + * pairings on purpose in order for other code not + * to assume any relation between physical and + * logical CPU numbers. */ - if (cpu == i) { - bL_switcher_cpu_original_cluster[cpu] = cluster; - continue; - } + if (cluster != cluster_0) + match = j; + } + if (match != -1) { + bL_switcher_cpu_pairing[i] = match; + cpumask_clear_cpu(match, &available_cpus); + pr_info("CPU%d paired with CPU%d\n", i, match); + } + } + + /* + * Now we disable the unwanted CPUs i.e. everything that has no + * pairing information (that includes the pairing counterparts). + */ + cpumask_clear(&bL_switcher_removed_logical_cpus); + for_each_online_cpu(i) { + cpu = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0); + cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1); + + /* Let's take note of the GIC ID for this CPU */ + gic_id = gic_get_cpu_id(i); + if (gic_id < 0) { + pr_err("%s: bad GIC ID for CPU %d\n", __func__, i); + bL_switcher_restore_cpus(); + return -EINVAL; + } + bL_gic_id[cpu][cluster] = gic_id; + pr_info("GIC ID for CPU %u cluster %u is %u\n", + cpu, cluster, gic_id); + + if (bL_switcher_cpu_pairing[i] != -1) { + bL_switcher_cpu_original_cluster[i] = cluster; + continue; } ret = cpu_down(i); @@ -409,7 +446,7 @@ static int bL_switcher_enable(void) static void bL_switcher_disable(void) { - unsigned int cpu, cluster, i; + unsigned int cpu, cluster; struct bL_thread *t; struct task_struct *task; @@ -429,7 +466,6 @@ static void bL_switcher_disable(void) * possibility for interference from external requests. */ for_each_online_cpu(cpu) { - BUG_ON(cpu != (cpu_logical_map(cpu) & 0xff)); t = &bL_threads[cpu]; task = t->task; t->task = NULL; @@ -453,14 +489,10 @@ static void bL_switcher_disable(void) /* If execution gets here, we're in trouble. */ pr_crit("%s: unable to restore original cluster for CPU %d\n", __func__, cpu); - for_each_cpu(i, &bL_switcher_removed_logical_cpus) { - if ((cpu_logical_map(i) & 0xff) != cpu) - continue; - pr_crit("%s: CPU %d can't be restored\n", - __func__, i); - cpumask_clear_cpu(i, &bL_switcher_removed_logical_cpus); - break; - } + pr_crit("%s: CPU %d can't be restored\n", + __func__, bL_switcher_cpu_pairing[cpu]); + cpumask_clear_cpu(bL_switcher_cpu_pairing[cpu], + &bL_switcher_removed_logical_cpus); } bL_switcher_restore_cpus(); -- cgit v1.2.3 From 272614351423ce8c37ff730efc130e5b73fe64f5 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 26 Nov 2012 22:48:55 -0500 Subject: ARM: bL_switcher: filter CPU hotplug requests when the switcher is active Trying to support both the switcher and CPU hotplug at the same time is tricky due to ambiguous semantics. So let's at least prevent users from messing around with those logical CPUs the switcher has removed and those which were not active when the switcher was activated. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 0e4fb3e5e99c..335ff76d4c5a 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -554,6 +554,26 @@ static int __init bL_switcher_sysfs_init(void) #endif /* CONFIG_SYSFS */ +/* + * Veto any CPU hotplug operation on those CPUs we've removed + * while the switcher is active. + * We're just not ready to deal with that given the trickery involved. + */ +static int bL_switcher_hotplug_callback(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + if (bL_switcher_active) { + int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu]; + switch (action & 0xf) { + case CPU_UP_PREPARE: + case CPU_DOWN_PREPARE: + if (pairing == -1) + return NOTIFY_BAD; + } + } + return NOTIFY_DONE; +} + static bool no_bL_switcher; core_param(no_bL_switcher, no_bL_switcher, bool, 0644); @@ -566,6 +586,8 @@ static int __init bL_switcher_init(void) return -EINVAL; } + cpu_notifier(bL_switcher_hotplug_callback, 0); + if (!no_bL_switcher) { ret = bL_switcher_enable(); if (ret) -- cgit v1.2.3 From b22537c682671de97c932d5addb6b7d087352aa1 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 12 Apr 2012 03:04:28 -0400 Subject: ARM: bL_switcher: add a simple /dev user interface for debugging purposes Only the basic call to aid debugging. *** NOT FOR PRODUCTION *** Usage: echo , > /dev/b.L_switcher where is the logical CPU number, and is 0 for the first cluster and 1 for the second cluster. Signed-off-by: nicolas Pitre --- arch/arm/common/Makefile | 1 + arch/arm/common/bL_switcher_dummy_if.c | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 arch/arm/common/bL_switcher_dummy_if.c (limited to 'arch/arm/common') diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile index 2586240d5a45..5c8584c4944d 100644 --- a/arch/arm/common/Makefile +++ b/arch/arm/common/Makefile @@ -18,3 +18,4 @@ AFLAGS_mcpm_head.o := -march=armv7-a AFLAGS_vlock.o := -march=armv7-a obj-$(CONFIG_TI_PRIV_EDMA) += edma.o obj-$(CONFIG_BL_SWITCHER) += bL_switcher.o +obj-$(CONFIG_BL_SWITCHER_DUMMY_IF) += bL_switcher_dummy_if.o diff --git a/arch/arm/common/bL_switcher_dummy_if.c b/arch/arm/common/bL_switcher_dummy_if.c new file mode 100644 index 000000000000..3f47f1203c6b --- /dev/null +++ b/arch/arm/common/bL_switcher_dummy_if.c @@ -0,0 +1,71 @@ +/* + * arch/arm/common/bL_switcher_dummy_if.c -- b.L switcher dummy interface + * + * Created by: Nicolas Pitre, November 2012 + * Copyright: (C) 2012-2013 Linaro Limited + * + * Dummy interface to user space for debugging purpose only. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +static ssize_t bL_switcher_write(struct file *file, const char __user *buf, + size_t len, loff_t *pos) +{ + unsigned char val[3]; + unsigned int cpu, cluster; + int ret; + + pr_debug("%s\n", __func__); + + if (len < 3) + return -EINVAL; + + if (copy_from_user(val, buf, 3)) + return -EFAULT; + + /* format: , */ + if (val[0] < '0' || val[0] > '9' || + val[1] != ',' || + val[2] < '0' || val[2] > '1') + return -EINVAL; + + cpu = val[0] - '0'; + cluster = val[2] - '0'; + ret = bL_switch_request(cpu, cluster); + + return ret ? : len; +} + +static const struct file_operations bL_switcher_fops = { + .write = bL_switcher_write, + .owner = THIS_MODULE, +}; + +static struct miscdevice bL_switcher_device = { + MISC_DYNAMIC_MINOR, + "b.L_switcher", + &bL_switcher_fops +}; + +static int __init bL_switcher_dummy_if_init(void) +{ + return misc_register(&bL_switcher_device); +} + +static void __exit bL_switcher_dummy_if_exit(void) +{ + misc_deregister(&bL_switcher_device); +} + +module_init(bL_switcher_dummy_if_init); +module_exit(bL_switcher_dummy_if_exit); -- cgit v1.2.3 From c0f4375146a738bae23e48fa8b5383abf02177cb Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Mon, 10 Dec 2012 17:19:57 +0000 Subject: ARM: bL_switcher: Add synchronous enable/disable interface Some subsystems will need to know for sure whether the switcher is enabled or disabled during certain critical regions. This patch provides a simple mutex-based mechanism to discover whether the switcher is enabled and temporarily lock out further enable/disable: * bL_switcher_get_enabled() returns true iff the switcher is enabled and temporarily inhibits enable/disable. * bL_switcher_put_enabled() permits enable/disable of the switcher again after a previous call to bL_switcher_get_enabled(). Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 335ff76d4c5a..7d98629aa446 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -302,6 +303,7 @@ EXPORT_SYMBOL_GPL(bL_switch_request); * Activation and configuration code. */ +static DEFINE_MUTEX(bL_switcher_activation_lock); static unsigned int bL_switcher_active; static unsigned int bL_switcher_cpu_original_cluster[NR_CPUS]; static cpumask_t bL_switcher_removed_logical_cpus; @@ -413,9 +415,11 @@ static int bL_switcher_enable(void) { int cpu, ret; + mutex_lock(&bL_switcher_activation_lock); cpu_hotplug_driver_lock(); if (bL_switcher_active) { cpu_hotplug_driver_unlock(); + mutex_unlock(&bL_switcher_activation_lock); return 0; } @@ -424,6 +428,7 @@ static int bL_switcher_enable(void) ret = bL_switcher_halve_cpus(); if (ret) { cpu_hotplug_driver_unlock(); + mutex_unlock(&bL_switcher_activation_lock); return ret; } @@ -436,9 +441,10 @@ static int bL_switcher_enable(void) } bL_switcher_active = 1; - cpu_hotplug_driver_unlock(); - pr_info("big.LITTLE switcher initialized\n"); + + cpu_hotplug_driver_unlock(); + mutex_unlock(&bL_switcher_activation_lock); return 0; } @@ -450,9 +456,11 @@ static void bL_switcher_disable(void) struct bL_thread *t; struct task_struct *task; + mutex_lock(&bL_switcher_activation_lock); cpu_hotplug_driver_lock(); if (!bL_switcher_active) { cpu_hotplug_driver_unlock(); + mutex_unlock(&bL_switcher_activation_lock); return; } bL_switcher_active = 0; @@ -497,6 +505,7 @@ static void bL_switcher_disable(void) bL_switcher_restore_cpus(); cpu_hotplug_driver_unlock(); + mutex_unlock(&bL_switcher_activation_lock); } static ssize_t bL_switcher_active_show(struct kobject *kobj, @@ -554,6 +563,20 @@ static int __init bL_switcher_sysfs_init(void) #endif /* CONFIG_SYSFS */ +bool bL_switcher_get_enabled(void) +{ + mutex_lock(&bL_switcher_activation_lock); + + return bL_switcher_active; +} +EXPORT_SYMBOL_GPL(bL_switcher_get_enabled); + +void bL_switcher_put_enabled(void) +{ + mutex_unlock(&bL_switcher_activation_lock); +} +EXPORT_SYMBOL_GPL(bL_switcher_put_enabled); + /* * Veto any CPU hotplug operation on those CPUs we've removed * while the switcher is active. -- cgit v1.2.3 From 491990e29f5d285a1b75e74785e3160716b79040 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Mon, 10 Dec 2012 17:19:58 +0000 Subject: ARM: bL_switcher: Add runtime control notifier Some subsystems will need to respond synchronously to runtime enabling and disabling of the switcher. This patch adds a dedicated notifier interface to support such subsystems. Pre- and post- enable/disable notifications are sent to registered callbacks, allowing safe transition of non-b.L- transparent subsystems across these control transitions. Notifier callbacks may veto switcher (de)activation on pre notifications only. Post notifications won't revert the action. If enabling or disabling of the switcher fails after the pre-change notification has been sent, subsystems which have registered notifiers can be left in an inappropriate state. This patch sends a suitable post-change notification on failure, indicating that the old state has been reestablished. For example, a failed initialisation will result in the following sequence: BL_NOTIFY_PRE_ENABLE /* switcher initialisation fails */ BL_NOTIFY_POST_DISABLE It is the responsibility of notified subsystems to respond in an appropriate way. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 60 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 10 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 7d98629aa446..016488730cb7 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -304,10 +305,34 @@ EXPORT_SYMBOL_GPL(bL_switch_request); */ static DEFINE_MUTEX(bL_switcher_activation_lock); +static BLOCKING_NOTIFIER_HEAD(bL_activation_notifier); static unsigned int bL_switcher_active; static unsigned int bL_switcher_cpu_original_cluster[NR_CPUS]; static cpumask_t bL_switcher_removed_logical_cpus; +int bL_switcher_register_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&bL_activation_notifier, nb); +} +EXPORT_SYMBOL_GPL(bL_switcher_register_notifier); + +int bL_switcher_unregister_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&bL_activation_notifier, nb); +} +EXPORT_SYMBOL_GPL(bL_switcher_unregister_notifier); + +static int bL_activation_notify(unsigned long val) +{ + int ret; + + ret = blocking_notifier_call_chain(&bL_activation_notifier, val, NULL); + if (ret & NOTIFY_STOP_MASK) + pr_err("%s: notifier chain failed with status 0x%x\n", + __func__, ret); + return notifier_to_errno(ret); +} + static void bL_switcher_restore_cpus(void) { int i; @@ -425,12 +450,13 @@ static int bL_switcher_enable(void) pr_info("big.LITTLE switcher initializing\n"); + ret = bL_activation_notify(BL_NOTIFY_PRE_ENABLE); + if (ret) + goto error; + ret = bL_switcher_halve_cpus(); - if (ret) { - cpu_hotplug_driver_unlock(); - mutex_unlock(&bL_switcher_activation_lock); - return ret; - } + if (ret) + goto error; for_each_online_cpu(cpu) { struct bL_thread *t = &bL_threads[cpu]; @@ -441,11 +467,18 @@ static int bL_switcher_enable(void) } bL_switcher_active = 1; + bL_activation_notify(BL_NOTIFY_POST_ENABLE); pr_info("big.LITTLE switcher initialized\n"); + goto out; + +error: + pr_warn("big.LITTLE switcher initialization failed\n"); + bL_activation_notify(BL_NOTIFY_POST_DISABLE); +out: cpu_hotplug_driver_unlock(); mutex_unlock(&bL_switcher_activation_lock); - return 0; + return ret; } #ifdef CONFIG_SYSFS @@ -458,11 +491,15 @@ static void bL_switcher_disable(void) mutex_lock(&bL_switcher_activation_lock); cpu_hotplug_driver_lock(); - if (!bL_switcher_active) { - cpu_hotplug_driver_unlock(); - mutex_unlock(&bL_switcher_activation_lock); - return; + + if (!bL_switcher_active) + goto out; + + if (bL_activation_notify(BL_NOTIFY_PRE_DISABLE) != 0) { + bL_activation_notify(BL_NOTIFY_POST_ENABLE); + goto out; } + bL_switcher_active = 0; /* @@ -504,6 +541,9 @@ static void bL_switcher_disable(void) } bL_switcher_restore_cpus(); + bL_activation_notify(BL_NOTIFY_POST_DISABLE); + +out: cpu_hotplug_driver_unlock(); mutex_unlock(&bL_switcher_activation_lock); } -- cgit v1.2.3 From 0577fee283fb385afbcdb78d1f4c398d7326b68f Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Wed, 22 May 2013 19:08:16 +0100 Subject: ARM: bL_switcher: Add switch completion callback for bL_switch_request() There is no explicit way to know when a switch started via bL_switch_request() is complete. This can lead to unpredictable behaviour when the switcher is controlled by a subsystem which makes dynamic decisions (such as cpufreq). The CPU PM notifier is not really suitable for signalling completion, because the CPU could get suspended and resumed for other, independent reasons while a switch request is in flight. Adding a whole new notifier for this seems excessive, and may tempt people to put heavyweight code on this path. This patch implements a new bL_switch_request_cb() function that allows for a per-request lightweight callback, private between the switcher and the caller of bL_switch_request_cb(). Overlapping switches on a single CPU are considered incorrect if they are requested via bL_switch_request_cb() with a callback (they will lead to an unpredictable final state without explicit external synchronisation to force the requests into a particular order). Queuing requests robustly would be overkill because only one subsystem should be attempting to control the switcher at any time. Overlapping requests of this kind will be failed with -EBUSY to indicate that the second request won't take effect and the completer will never be called for it. bL_switch_request() is retained as a wrapper round the new function, with the old, fire-and-forget semantics. In this case the last request will always win. The request may still be denied if a previous request with a completer is still pending. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 53 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 016488730cb7..34316be404d5 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -9,6 +9,7 @@ * published by the Free Software Foundation. */ +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -224,10 +226,13 @@ static int bL_switch_to(unsigned int new_cluster_id) } struct bL_thread { + spinlock_t lock; struct task_struct *task; wait_queue_head_t wq; int wanted_cluster; struct completion started; + bL_switch_completion_handler completer; + void *completer_cookie; }; static struct bL_thread bL_threads[NR_CPUS]; @@ -237,6 +242,8 @@ static int bL_switcher_thread(void *arg) struct bL_thread *t = arg; struct sched_param param = { .sched_priority = 1 }; int cluster; + bL_switch_completion_handler completer; + void *completer_cookie; sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); complete(&t->started); @@ -247,9 +254,21 @@ static int bL_switcher_thread(void *arg) wait_event_interruptible(t->wq, t->wanted_cluster != -1 || kthread_should_stop()); - cluster = xchg(&t->wanted_cluster, -1); - if (cluster != -1) + + spin_lock(&t->lock); + cluster = t->wanted_cluster; + completer = t->completer; + completer_cookie = t->completer_cookie; + t->wanted_cluster = -1; + t->completer = NULL; + spin_unlock(&t->lock); + + if (cluster != -1) { bL_switch_to(cluster); + + if (completer) + completer(completer_cookie); + } } while (!kthread_should_stop()); return 0; @@ -270,16 +289,30 @@ static struct task_struct *bL_switcher_thread_create(int cpu, void *arg) } /* - * bL_switch_request - Switch to a specific cluster for the given CPU + * bL_switch_request_cb - Switch to a specific cluster for the given CPU, + * with completion notification via a callback * * @cpu: the CPU to switch * @new_cluster_id: the ID of the cluster to switch to. + * @completer: switch completion callback. if non-NULL, + * @completer(@completer_cookie) will be called on completion of + * the switch, in non-atomic context. + * @completer_cookie: opaque context argument for @completer. * * This function causes a cluster switch on the given CPU by waking up * the appropriate switcher thread. This function may or may not return * before the switch has occurred. + * + * If a @completer callback function is supplied, it will be called when + * the switch is complete. This can be used to determine asynchronously + * when the switch is complete, regardless of when bL_switch_request() + * returns. When @completer is supplied, no new switch request is permitted + * for the affected CPU until after the switch is complete, and @completer + * has returned. */ -int bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) +int bL_switch_request_cb(unsigned int cpu, unsigned int new_cluster_id, + bL_switch_completion_handler completer, + void *completer_cookie) { struct bL_thread *t; @@ -289,16 +322,25 @@ int bL_switch_request(unsigned int cpu, unsigned int new_cluster_id) } t = &bL_threads[cpu]; + if (IS_ERR(t->task)) return PTR_ERR(t->task); if (!t->task) return -ESRCH; + spin_lock(&t->lock); + if (t->completer) { + spin_unlock(&t->lock); + return -EBUSY; + } + t->completer = completer; + t->completer_cookie = completer_cookie; t->wanted_cluster = new_cluster_id; + spin_unlock(&t->lock); wake_up(&t->wq); return 0; } -EXPORT_SYMBOL_GPL(bL_switch_request); +EXPORT_SYMBOL_GPL(bL_switch_request_cb); /* * Activation and configuration code. @@ -460,6 +502,7 @@ static int bL_switcher_enable(void) for_each_online_cpu(cpu) { struct bL_thread *t = &bL_threads[cpu]; + spin_lock_init(&t->lock); init_waitqueue_head(&t->wq); init_completion(&t->started); t->wanted_cluster = -1; -- cgit v1.2.3 From 108a9640abfada2599b6cb08c7cc00a4eebf8f8f Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 23 Oct 2012 01:39:08 -0400 Subject: ARM: bL_switcher: synchronize the outbound with the inbound Let's wait for the inbound CPU to come up and snoop some of the outbound CPU cache before bringing the outbound CPU down. That should be more efficient than going down right away. Possible improvements might involve some monitoring of the CCI event counters. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 34316be404d5..aab7c1274885 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -55,9 +55,10 @@ static int read_mpidr(void) * bL switcher core code. */ -static void bL_do_switch(void *_unused) +static void bL_do_switch(void *_arg) { unsigned ib_mpidr, ib_cpu, ib_cluster; + long volatile handshake, **handshake_ptr = _arg; pr_debug("%s\n", __func__); @@ -65,6 +66,13 @@ static void bL_do_switch(void *_unused) ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0); ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1); + /* Advertise our handshake location */ + if (handshake_ptr) { + handshake = 0; + *handshake_ptr = &handshake; + } else + handshake = -1; + /* * Our state has been saved at this point. Let's release our * inbound CPU. @@ -83,6 +91,14 @@ static void bL_do_switch(void *_unused) * we have none. */ + /* + * Let's wait until our inbound is alive. + */ + while (!handshake) { + wfe(); + smp_mb(); + } + /* Let's put ourself down. */ mcpm_cpu_power_down(); @@ -130,6 +146,7 @@ static int bL_switch_to(unsigned int new_cluster_id) unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster; struct tick_device *tdev; enum clock_event_mode tdev_mode; + long volatile *handshake_ptr; int ret; this_cpu = smp_processor_id(); @@ -198,7 +215,7 @@ static int bL_switch_to(unsigned int new_cluster_id) cpu_logical_map(that_cpu) = ob_mpidr; /* Let's do the actual CPU switch. */ - ret = cpu_suspend(0, bL_switchpoint); + ret = cpu_suspend((unsigned long)&handshake_ptr, bL_switchpoint); if (ret > 0) panic("%s: cpu_suspend() returned %d\n", __func__, ret); @@ -220,6 +237,9 @@ static int bL_switch_to(unsigned int new_cluster_id) local_fiq_enable(); local_irq_enable(); + *handshake_ptr = 1; + dsb_sev(); + if (ret) pr_err("%s exiting with error %d\n", __func__, ret); return ret; -- cgit v1.2.3 From de885d147ad2c4a66777e3557440247efde1cc8d Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 27 Nov 2012 23:11:20 -0500 Subject: ARM: mcpm: add a simple poke mechanism to the early entry code This allows to poke a predetermined value into a specific address upon entering the early boot code in bL_head.S. Signed-off-by: Nicolas Pitre --- arch/arm/common/mcpm_entry.c | 12 ++++++++++++ arch/arm/common/mcpm_head.S | 16 ++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/mcpm_entry.c b/arch/arm/common/mcpm_entry.c index 370236dd1a03..4a2b32fd53a1 100644 --- a/arch/arm/common/mcpm_entry.c +++ b/arch/arm/common/mcpm_entry.c @@ -27,6 +27,18 @@ void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr) sync_cache_w(&mcpm_entry_vectors[cluster][cpu]); } +extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2]; + +void mcpm_set_early_poke(unsigned cpu, unsigned cluster, + unsigned long poke_phys_addr, unsigned long poke_val) +{ + unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0]; + poke[0] = poke_phys_addr; + poke[1] = poke_val; + __cpuc_flush_dcache_area((void *)poke, 8); + outer_clean_range(__pa(poke), __pa(poke + 2)); +} + static const struct mcpm_platform_ops *platform_ops; int __init mcpm_platform_register(const struct mcpm_platform_ops *ops) diff --git a/arch/arm/common/mcpm_head.S b/arch/arm/common/mcpm_head.S index 39c96df3477a..49dd5352fe70 100644 --- a/arch/arm/common/mcpm_head.S +++ b/arch/arm/common/mcpm_head.S @@ -71,12 +71,19 @@ ENTRY(mcpm_entry_point) * position independent way. */ adr r5, 3f - ldmia r5, {r6, r7, r8, r11} + ldmia r5, {r0, r6, r7, r8, r11} + add r0, r5, r0 @ r0 = mcpm_entry_early_pokes add r6, r5, r6 @ r6 = mcpm_entry_vectors ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys add r8, r5, r8 @ r8 = mcpm_sync add r11, r5, r11 @ r11 = first_man_locks + @ Perform an early poke, if any + add r0, r0, r4, lsl #3 + ldmia r0, {r0, r1} + teq r0, #0 + strne r1, [r0] + mov r0, #MCPM_SYNC_CLUSTER_SIZE mla r8, r0, r10, r8 @ r8 = sync cluster base @@ -195,7 +202,8 @@ mcpm_entry_gated: .align 2 -3: .word mcpm_entry_vectors - . +3: .word mcpm_entry_early_pokes - . + .word mcpm_entry_vectors - 3b .word mcpm_power_up_setup_phys - 3b .word mcpm_sync - 3b .word first_man_locks - 3b @@ -214,6 +222,10 @@ first_man_locks: ENTRY(mcpm_entry_vectors) .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER + .type mcpm_entry_early_pokes, #object +ENTRY(mcpm_entry_early_pokes) + .space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER + .type mcpm_power_up_setup_phys, #object ENTRY(mcpm_power_up_setup_phys) .space 4 @ set by mcpm_sync_init() -- cgit v1.2.3 From 6137eba6c2b9bc2b7fd52e77741f50e43db4b5a6 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Thu, 13 Jun 2013 23:51:18 -0400 Subject: ARM: bL_switcher: wait until inbound is alive before performing a switch In some cases, a significant delay may be observed between the moment a request for a CPU to come up is made and the moment it is ready to start executing kernel code. This is especially true when a whole cluster has to be powered up which may take in the order of miliseconds. It is therefore a good idea to let the outbound CPU continue to execute code in the mean time, and be notified when the inbound is ready before performing the actual switch. This is achieved by registering a completion block with the appropriate IPI callback, and programming the sending of an IPI by the early assembly code prior to entering the main kernel code. Once the IPI is delivered to the outbound CPU, the completion block is "completed" and the switcher thread is resumed. Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index aab7c1274885..dc53eb8dcc81 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -144,10 +144,11 @@ static int bL_switch_to(unsigned int new_cluster_id) { unsigned int mpidr, this_cpu, that_cpu; unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster; + struct completion inbound_alive; struct tick_device *tdev; enum clock_event_mode tdev_mode; long volatile *handshake_ptr; - int ret; + int ipi_nr, ret; this_cpu = smp_processor_id(); ob_mpidr = read_mpidr(); @@ -166,10 +167,18 @@ static int bL_switch_to(unsigned int new_cluster_id) pr_debug("before switch: CPU %d MPIDR %#x -> %#x\n", this_cpu, ob_mpidr, ib_mpidr); + this_cpu = smp_processor_id(); + /* Close the gate for our entry vectors */ mcpm_set_entry_vector(ob_cpu, ob_cluster, NULL); mcpm_set_entry_vector(ib_cpu, ib_cluster, NULL); + /* Install our "inbound alive" notifier. */ + init_completion(&inbound_alive); + ipi_nr = register_ipi_completion(&inbound_alive, this_cpu); + ipi_nr |= ((1 << 16) << bL_gic_id[ob_cpu][ob_cluster]); + mcpm_set_early_poke(ib_cpu, ib_cluster, gic_get_sgir_physaddr(), ipi_nr); + /* * Let's wake up the inbound CPU now in case it requires some delay * to come online, but leave it gated in our entry vector code. @@ -180,6 +189,19 @@ static int bL_switch_to(unsigned int new_cluster_id) return ret; } + /* + * Raise a SGI on the inbound CPU to make sure it doesn't stall + * in a possible WFI, such as in bL_power_down(). + */ + gic_send_sgi(bL_gic_id[ib_cpu][ib_cluster], 0); + + /* + * Wait for the inbound to come up. This allows for other + * tasks to be scheduled in the mean time. + */ + wait_for_completion(&inbound_alive); + mcpm_set_early_poke(ib_cpu, ib_cluster, 0, 0); + /* * From this point we are entering the switch critical zone * and can't take any interrupts anymore. @@ -190,12 +212,6 @@ static int bL_switch_to(unsigned int new_cluster_id) /* redirect GIC's SGIs to our counterpart */ gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]); - /* - * Raise a SGI on the inbound CPU to make sure it doesn't stall - * in a possible WFI, such as in mcpm_power_down(). - */ - arch_send_wakeup_ipi_mask(cpumask_of(this_cpu)); - tdev = tick_get_device(this_cpu); if (tdev && !cpumask_equal(tdev->evtdev->cpumask, cpumask_of(this_cpu))) tdev = NULL; -- cgit v1.2.3 From 1bfbddb6f3a0dbb8c3996d1c4d4911d695737c15 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Mon, 14 May 2012 17:40:07 +0100 Subject: ARM: bL_switcher: Basic trace events support This patch adds simple trace events to the b.L switcher code to allow tracing of CPU migration events. To make use of the trace events, you will need: CONFIG_FTRACE=y CONFIG_ENABLE_DEFAULT_TRACERS=y The following events are added: * power:cpu_migrate_begin * power:cpu_migrate_finish each with the following data: u64 timestamp; u32 cpu_hwid; power:cpu_migrate_begin occurs immediately before the switcher-specific migration operations start. power:cpu_migrate_finish occurs immediately when migration is completed. The cpu_hwid field contains the ID fields of the MPIDR. * For power:cpu_migrate_begin, cpu_hwid is the ID of the outbound physical CPU (equivalent to (from_phys_cpu,from_phys_cluster)). * For power:cpu_migrate_finish, cpu_hwid is the ID of the inbound physical CPU (equivalent to (to_phys_cpu,to_phys_cluster)). By design, the cpu_hwid field is masked in the same way as the device tree cpu node reg property, allowing direct correlation to the DT description of the hardware. The timestamp is added in order to minimise timing noise. An accurate system-wide clock should be used for generating this (hopefully getnstimeofday is appropriate, but it could be changed). It could be any monotonic shared clock, since the aim is to allow accurate deltas to be computed. We don't necessarily care about accurate synchronisation with wall clock time. In practice, each switch takes place on a single logical CPU, and the trace infrastructure should guarantee that events are well-ordered with respect to a single logical CPU. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index dc53eb8dcc81..7002de360d23 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -33,10 +34,14 @@ #include #include +#include #include #include #include +#define CREATE_TRACE_POINTS +#include + /* * Use our own MPIDR accessors as the generic ones in asm/cputype.h have @@ -51,6 +56,16 @@ static int read_mpidr(void) return id & MPIDR_HWID_BITMASK; } +/* + * Get a global nanosecond time stamp for tracing. + */ +static s64 get_ns(void) +{ + struct timespec ts; + getnstimeofday(&ts); + return timespec_to_ns(&ts); +} + /* * bL switcher core code. */ @@ -208,6 +223,7 @@ static int bL_switch_to(unsigned int new_cluster_id) */ local_irq_disable(); local_fiq_disable(); + trace_cpu_migrate_begin(get_ns(), ob_mpidr); /* redirect GIC's SGIs to our counterpart */ gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]); @@ -250,6 +266,7 @@ static int bL_switch_to(unsigned int new_cluster_id) tdev->evtdev->next_event, 1); } + trace_cpu_migrate_finish(get_ns(), ib_mpidr); local_fiq_enable(); local_irq_enable(); -- cgit v1.2.3 From b09bbe5b1267b6af22a9584d614f5eec5d74f405 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Wed, 6 Feb 2013 15:45:23 +0000 Subject: ARM: bL_switcher/trace: Add trace trigger for trace bootstrapping When tracing switching, an external tracer needs a way to bootstrap its knowledge of the logical<->physical CPU mapping. This patch adds a sysfs attribute trace_trigger. A write to this attribute will generate a power:cpu_migrate_current event for each online CPU, indicating the current physical CPU for each logical CPU. Activating or deactivating the switcher also generates these events, so that the tracer knows about the resulting remapping of affected CPUs. Signed-off-by: Dave Martin --- arch/arm/common/bL_switcher.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 7002de360d23..f0dc025077d5 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -531,6 +532,25 @@ static int bL_switcher_halve_cpus(void) return 0; } +static void bL_switcher_trace_trigger_cpu(void *__always_unused info) +{ + trace_cpu_migrate_current(get_ns(), read_mpidr()); +} + +static int bL_switcher_trace_trigger(void) +{ + int ret; + + preempt_disable(); + + bL_switcher_trace_trigger_cpu(NULL); + ret = smp_call_function(bL_switcher_trace_trigger_cpu, NULL, true); + + preempt_enable(); + + return ret; +} + static int bL_switcher_enable(void) { int cpu, ret; @@ -553,6 +573,8 @@ static int bL_switcher_enable(void) if (ret) goto error; + bL_switcher_trace_trigger(); + for_each_online_cpu(cpu) { struct bL_thread *t = &bL_threads[cpu]; spin_lock_init(&t->lock); @@ -637,6 +659,8 @@ static void bL_switcher_disable(void) } bL_switcher_restore_cpus(); + bL_switcher_trace_trigger(); + bL_activation_notify(BL_NOTIFY_POST_DISABLE); out: @@ -670,11 +694,23 @@ static ssize_t bL_switcher_active_store(struct kobject *kobj, return (ret >= 0) ? count : ret; } +static ssize_t bL_switcher_trace_trigger_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, size_t count) +{ + int ret = bL_switcher_trace_trigger(); + + return ret ? ret : count; +} + static struct kobj_attribute bL_switcher_active_attr = __ATTR(active, 0644, bL_switcher_active_show, bL_switcher_active_store); +static struct kobj_attribute bL_switcher_trace_trigger_attr = + __ATTR(trace_trigger, 0200, NULL, bL_switcher_trace_trigger_store); + static struct attribute *bL_switcher_attrs[] = { &bL_switcher_active_attr.attr, + &bL_switcher_trace_trigger_attr.attr, NULL, }; -- cgit v1.2.3 From 29064b88466ee725613db16d8c05b0ec5443a309 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Mon, 11 Feb 2013 14:39:19 +0000 Subject: ARM: bL_switcher/trace: Add kernel trace trigger interface This patch exports a bL_switcher_trace_trigger() function to provide a means for drivers using the trace events to get the current status when starting a trace session. Calling this function is equivalent to pinging the trace_trigger file in sysfs. Signed-off-by: Dave Martin --- arch/arm/common/bL_switcher.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index f0dc025077d5..f4878a36047a 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -537,7 +537,7 @@ static void bL_switcher_trace_trigger_cpu(void *__always_unused info) trace_cpu_migrate_current(get_ns(), read_mpidr()); } -static int bL_switcher_trace_trigger(void) +int bL_switcher_trace_trigger(void) { int ret; @@ -550,6 +550,7 @@ static int bL_switcher_trace_trigger(void) return ret; } +EXPORT_SYMBOL_GPL(bL_switcher_trace_trigger); static int bL_switcher_enable(void) { -- cgit v1.2.3 From d08e2e09042bd3f7ef66a35cb4bb92794ab26bb2 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Wed, 13 Feb 2013 16:20:44 +0000 Subject: ARM: bL_switcher: Add query interface to discover CPU affinities When the switcher is active, there is no straightforward way to figure out which logical CPU a given physical CPU maps to. This patch provides a function bL_switcher_get_logical_index(mpidr), which is analogous to get_logical_index(). This function returns the logical CPU on which the specified physical CPU is grouped (or -EINVAL if unknown). If the switcher is inactive or not present, -EUNATCH is returned instead. Signed-off-by: Dave Martin Signed-off-by: Nicolas Pitre --- arch/arm/common/bL_switcher.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index f4878a36047a..63bbc4f70564 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -532,6 +532,26 @@ static int bL_switcher_halve_cpus(void) return 0; } +/* Determine the logical CPU a given physical CPU is grouped on. */ +int bL_switcher_get_logical_index(u32 mpidr) +{ + int cpu; + + if (!bL_switcher_active) + return -EUNATCH; + + mpidr &= MPIDR_HWID_BITMASK; + for_each_online_cpu(cpu) { + int pairing = bL_switcher_cpu_pairing[cpu]; + if (pairing == -1) + continue; + if ((mpidr == cpu_logical_map(cpu)) || + (mpidr == cpu_logical_map(pairing))) + return cpu; + } + return -EINVAL; +} + static void bL_switcher_trace_trigger_cpu(void *__always_unused info) { trace_cpu_migrate_current(get_ns(), read_mpidr()); -- cgit v1.2.3 From 519ceb9fd10cd7e836d0aa97b2068cc9e97f463b Mon Sep 17 00:00:00 2001 From: Victor Kamensky Date: Mon, 7 Oct 2013 21:37:19 -0700 Subject: ARM: mcpm: fix big endian issue in mcpm startup code In big endian mode mcpm_entry_point is first function that called on secondaries CPU. First it should switch CPU into big endian code. [ben.dooks@codethink.co.uk: merge fix patch from Victor into this] Signed-off-by: Victor Kamensky Acked-by: Nicolas Pitre Reviewed-by: Dave Martin Signed-off-by: Ben Dooks --- arch/arm/common/mcpm_head.S | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm/common') diff --git a/arch/arm/common/mcpm_head.S b/arch/arm/common/mcpm_head.S index 39c96df3477a..4f88f5e655be 100644 --- a/arch/arm/common/mcpm_head.S +++ b/arch/arm/common/mcpm_head.S @@ -15,6 +15,7 @@ #include #include +#include #include "vlock.h" @@ -47,6 +48,7 @@ ENTRY(mcpm_entry_point) + ARM_BE8(setend be) THUMB( adr r12, BSYM(1f) ) THUMB( bx r12 ) THUMB( .thumb ) -- cgit v1.2.3 From b0ced9d220f78f27b93e4d1024b3865ba172dbce Mon Sep 17 00:00:00 2001 From: Tushar Behera Date: Thu, 31 Oct 2013 06:46:14 +0100 Subject: ARM: 7874/2: bL_switcher: Remove cpu_hotplug_driver_{lock,unlock}() Commit 6dedcca610c6 ("hotplug, powerpc, x86: Remove cpu_hotplug_driver_lock())" removes the the definition of cpu_hotplug_driver_{lock,unlock} APIs, thereby causing a build error. Replace these calls with {lock,unlock}_device_hotplug(). Signed-off-by: Tushar Behera Signed-off-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/common/bL_switcher.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'arch/arm/common') diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 63bbc4f70564..5774b6ea7ad5 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -577,9 +577,9 @@ static int bL_switcher_enable(void) int cpu, ret; mutex_lock(&bL_switcher_activation_lock); - cpu_hotplug_driver_lock(); + lock_device_hotplug(); if (bL_switcher_active) { - cpu_hotplug_driver_unlock(); + unlock_device_hotplug(); mutex_unlock(&bL_switcher_activation_lock); return 0; } @@ -615,7 +615,7 @@ error: bL_activation_notify(BL_NOTIFY_POST_DISABLE); out: - cpu_hotplug_driver_unlock(); + unlock_device_hotplug(); mutex_unlock(&bL_switcher_activation_lock); return ret; } @@ -629,7 +629,7 @@ static void bL_switcher_disable(void) struct task_struct *task; mutex_lock(&bL_switcher_activation_lock); - cpu_hotplug_driver_lock(); + lock_device_hotplug(); if (!bL_switcher_active) goto out; @@ -685,7 +685,7 @@ static void bL_switcher_disable(void) bL_activation_notify(BL_NOTIFY_POST_DISABLE); out: - cpu_hotplug_driver_unlock(); + unlock_device_hotplug(); mutex_unlock(&bL_switcher_activation_lock); } -- cgit v1.2.3