From 651bb2e9dca6e6dbad3fba5f6e6086a23575b8b5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 19 Jan 2017 17:20:59 +0000 Subject: arm64: arch_timer: Add infrastructure for multiple erratum detection methods We're currently stuck with DT when it comes to handling errata, which is pretty restrictive. In order to make things more flexible, let's introduce an infrastructure that could support alternative discovery methods. No change in functionality. Acked-by: Thomas Gleixner Reviewed-by: Hanjun Guo Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 83 +++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 11 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 7a8a4117f123..5c5c2af74ad9 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -182,7 +182,9 @@ EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); static const struct arch_timer_erratum_workaround ool_workarounds[] = { #ifdef CONFIG_FSL_ERRATUM_A008585 { + .match_type = ate_match_dt, .id = "fsl,erratum-a008585", + .desc = "Freescale erratum a005858", .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0, .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0, .read_cntvct_el0 = fsl_a008585_read_cntvct_el0, @@ -190,13 +192,81 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { #endif #ifdef CONFIG_HISILICON_ERRATUM_161010101 { + .match_type = ate_match_dt, .id = "hisilicon,erratum-161010101", + .desc = "HiSilicon erratum 161010101", .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, }, #endif }; + +typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *, + const void *); + +static +bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa, + const void *arg) +{ + const struct device_node *np = arg; + + return of_property_read_bool(np, wa->id); +} + +static const struct arch_timer_erratum_workaround * +arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, + ate_match_fn_t match_fn, + void *arg) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) { + if (ool_workarounds[i].match_type != type) + continue; + + if (match_fn(&ool_workarounds[i], arg)) + return &ool_workarounds[i]; + } + + return NULL; +} + +static +void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa) +{ + timer_unstable_counter_workaround = wa; + static_branch_enable(&arch_timer_read_ool_enabled); +} + +static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type, + void *arg) +{ + const struct arch_timer_erratum_workaround *wa; + ate_match_fn_t match_fn = NULL; + + if (static_branch_unlikely(&arch_timer_read_ool_enabled)) + return; + + switch (type) { + case ate_match_dt: + match_fn = arch_timer_check_dt_erratum; + break; + default: + WARN_ON(1); + return; + } + + wa = arch_timer_iterate_errata(type, match_fn, arg); + if (!wa) + return; + + arch_timer_enable_workaround(wa); + pr_info("Enabling global workaround for %s\n", wa->desc); +} + +#else +#define arch_timer_check_ool_workaround(t,a) do { } while(0) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static __always_inline @@ -960,17 +1030,8 @@ static int __init arch_timer_of_init(struct device_node *np) arch_timer_c3stop = !of_property_read_bool(np, "always-on"); -#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND - for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) { - if (of_property_read_bool(np, ool_workarounds[i].id)) { - timer_unstable_counter_workaround = &ool_workarounds[i]; - static_branch_enable(&arch_timer_read_ool_enabled); - pr_info("arch_timer: Enabling workaround for %s\n", - timer_unstable_counter_workaround->id); - break; - } - } -#endif + /* Check for globally applicable workarounds */ + arch_timer_check_ool_workaround(ate_match_dt, np); /* * If we cannot rely on firmware initializing the timer registers then -- cgit v1.2.3 From 0064030c6fd4ca6cfab42de037b2a89445beeead Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 20 Mar 2017 16:47:59 +0000 Subject: arm64: arch_timer: Add erratum handler for CPU-specific capability Should we ever have a workaround for an erratum that is detected using a capability and affecting a particular CPU, it'd be nice to have a way to probe them directly. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 5c5c2af74ad9..532e47fa43b3 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -214,6 +214,13 @@ bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa, return of_property_read_bool(np, wa->id); } +static +bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa, + const void *arg) +{ + return this_cpu_has_cap((uintptr_t)wa->id); +} + static const struct arch_timer_erratum_workaround * arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, ate_match_fn_t match_fn, @@ -244,14 +251,16 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t { const struct arch_timer_erratum_workaround *wa; ate_match_fn_t match_fn = NULL; - - if (static_branch_unlikely(&arch_timer_read_ool_enabled)) - return; + bool local = false; switch (type) { case ate_match_dt: match_fn = arch_timer_check_dt_erratum; break; + case ate_match_local_cap_id: + match_fn = arch_timer_check_local_cap_erratum; + local = true; + break; default: WARN_ON(1); return; @@ -261,8 +270,17 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t if (!wa) return; + if (needs_unstable_timer_counter_workaround()) { + if (wa != timer_unstable_counter_workaround) + pr_warn("Can't enable workaround for %s (clashes with %s\n)", + wa->desc, + timer_unstable_counter_workaround->desc); + return; + } + arch_timer_enable_workaround(wa); - pr_info("Enabling global workaround for %s\n", wa->desc); + pr_info("Enabling %s workaround for %s\n", + local ? "local" : "global", wa->desc); } #else @@ -522,6 +540,8 @@ static void __arch_timer_setup(unsigned type, BUG(); } + arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); + erratum_workaround_set_sne(clk); } else { clk->features |= CLOCK_EVT_FEAT_DYNIRQ; -- cgit v1.2.3 From f4e00a1a55c7f9b08a48b57088ac5d044d5523d5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 20 Jan 2017 18:28:32 +0000 Subject: arm64: arch_timer: Move arch_timer_reg_read/write around As we're about to move things around, let's start with the low level read/write functions. This allows us to use these functions in the errata handling code without having to use forward declaration of static functions. Acked-by: Thomas Gleixner Acked-by: Mark Rutland Acked-by: Daniel Lezcano Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 124 +++++++++++++++++------------------ 1 file changed, 62 insertions(+), 62 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 532e47fa43b3..b5c83cc2a67a 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -96,6 +96,68 @@ early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); * Architected system timer support. */ +static __always_inline +void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, + struct clock_event_device *clk) +{ + if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { + struct arch_timer *timer = to_arch_timer(clk); + switch (reg) { + case ARCH_TIMER_REG_CTRL: + writel_relaxed(val, timer->base + CNTP_CTL); + break; + case ARCH_TIMER_REG_TVAL: + writel_relaxed(val, timer->base + CNTP_TVAL); + break; + } + } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { + struct arch_timer *timer = to_arch_timer(clk); + switch (reg) { + case ARCH_TIMER_REG_CTRL: + writel_relaxed(val, timer->base + CNTV_CTL); + break; + case ARCH_TIMER_REG_TVAL: + writel_relaxed(val, timer->base + CNTV_TVAL); + break; + } + } else { + arch_timer_reg_write_cp15(access, reg, val); + } +} + +static __always_inline +u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, + struct clock_event_device *clk) +{ + u32 val; + + if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { + struct arch_timer *timer = to_arch_timer(clk); + switch (reg) { + case ARCH_TIMER_REG_CTRL: + val = readl_relaxed(timer->base + CNTP_CTL); + break; + case ARCH_TIMER_REG_TVAL: + val = readl_relaxed(timer->base + CNTP_TVAL); + break; + } + } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { + struct arch_timer *timer = to_arch_timer(clk); + switch (reg) { + case ARCH_TIMER_REG_CTRL: + val = readl_relaxed(timer->base + CNTV_CTL); + break; + case ARCH_TIMER_REG_TVAL: + val = readl_relaxed(timer->base + CNTV_TVAL); + break; + } + } else { + val = arch_timer_reg_read_cp15(access, reg); + } + + return val; +} + #ifdef CONFIG_FSL_ERRATUM_A008585 /* * The number of retries is an arbitrary value well beyond the highest number @@ -287,68 +349,6 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t #define arch_timer_check_ool_workaround(t,a) do { } while(0) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ -static __always_inline -void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, - struct clock_event_device *clk) -{ - if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { - struct arch_timer *timer = to_arch_timer(clk); - switch (reg) { - case ARCH_TIMER_REG_CTRL: - writel_relaxed(val, timer->base + CNTP_CTL); - break; - case ARCH_TIMER_REG_TVAL: - writel_relaxed(val, timer->base + CNTP_TVAL); - break; - } - } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { - struct arch_timer *timer = to_arch_timer(clk); - switch (reg) { - case ARCH_TIMER_REG_CTRL: - writel_relaxed(val, timer->base + CNTV_CTL); - break; - case ARCH_TIMER_REG_TVAL: - writel_relaxed(val, timer->base + CNTV_TVAL); - break; - } - } else { - arch_timer_reg_write_cp15(access, reg, val); - } -} - -static __always_inline -u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, - struct clock_event_device *clk) -{ - u32 val; - - if (access == ARCH_TIMER_MEM_PHYS_ACCESS) { - struct arch_timer *timer = to_arch_timer(clk); - switch (reg) { - case ARCH_TIMER_REG_CTRL: - val = readl_relaxed(timer->base + CNTP_CTL); - break; - case ARCH_TIMER_REG_TVAL: - val = readl_relaxed(timer->base + CNTP_TVAL); - break; - } - } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { - struct arch_timer *timer = to_arch_timer(clk); - switch (reg) { - case ARCH_TIMER_REG_CTRL: - val = readl_relaxed(timer->base + CNTV_CTL); - break; - case ARCH_TIMER_REG_TVAL: - val = readl_relaxed(timer->base + CNTV_TVAL); - break; - } - } else { - val = arch_timer_reg_read_cp15(access, reg); - } - - return val; -} - static __always_inline irqreturn_t timer_handler(const int access, struct clock_event_device *evt) { -- cgit v1.2.3 From 8328089f0aeba39173eee3fc109b5c77efd64747 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 27 Jan 2017 10:27:09 +0000 Subject: arm64: arch_timer: Get rid of erratum_workaround_set_sne Let's move the handling of workarounds affecting set_next_event to the affected function, instead of overriding the pointers as an afterthough. Yes, this is an extra indirection on the erratum handling path, but the HW is busted anyway. This will allow for some more flexibility later. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 90 ++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 49 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index b5c83cc2a67a..2c02e25d1475 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -241,6 +241,38 @@ EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled); +static void erratum_set_next_event_tval_generic(const int access, unsigned long evt, + struct clock_event_device *clk) +{ + unsigned long ctrl; + u64 cval = evt + arch_counter_get_cntvct(); + + ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); + ctrl |= ARCH_TIMER_CTRL_ENABLE; + ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; + + if (access == ARCH_TIMER_PHYS_ACCESS) + write_sysreg(cval, cntp_cval_el0); + else + write_sysreg(cval, cntv_cval_el0); + + arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); +} + +static int erratum_set_next_event_tval_virt(unsigned long evt, + struct clock_event_device *clk) +{ + erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk); + return 0; +} + +static int erratum_set_next_event_tval_phys(unsigned long evt, + struct clock_event_device *clk) +{ + erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk); + return 0; +} + static const struct arch_timer_erratum_workaround ool_workarounds[] = { #ifdef CONFIG_FSL_ERRATUM_A008585 { @@ -347,6 +379,9 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t #else #define arch_timer_check_ool_workaround(t,a) do { } while(0) +#define erratum_set_next_event_tval_virt(...) ({BUG(); 0;}) +#define erratum_set_next_event_tval_phys(...) ({BUG(); 0;}) +#define needs_unstable_timer_counter_workaround() ({false;}) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static __always_inline irqreturn_t timer_handler(const int access, @@ -436,43 +471,12 @@ static __always_inline void set_next_event(const int access, unsigned long evt, arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); } -#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND -static __always_inline void erratum_set_next_event_generic(const int access, - unsigned long evt, struct clock_event_device *clk) -{ - unsigned long ctrl; - u64 cval = evt + arch_counter_get_cntvct(); - - ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); - ctrl |= ARCH_TIMER_CTRL_ENABLE; - ctrl &= ~ARCH_TIMER_CTRL_IT_MASK; - - if (access == ARCH_TIMER_PHYS_ACCESS) - write_sysreg(cval, cntp_cval_el0); - else if (access == ARCH_TIMER_VIRT_ACCESS) - write_sysreg(cval, cntv_cval_el0); - - arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); -} - -static int erratum_set_next_event_virt(unsigned long evt, - struct clock_event_device *clk) -{ - erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk); - return 0; -} - -static int erratum_set_next_event_phys(unsigned long evt, - struct clock_event_device *clk) -{ - erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk); - return 0; -} -#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ - static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk) { + if (needs_unstable_timer_counter_workaround()) + return erratum_set_next_event_tval_virt(evt, clk); + set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0; } @@ -480,6 +484,9 @@ static int arch_timer_set_next_event_virt(unsigned long evt, static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk) { + if (needs_unstable_timer_counter_workaround()) + return erratum_set_next_event_tval_phys(evt, clk); + set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0; } @@ -498,19 +505,6 @@ static int arch_timer_set_next_event_phys_mem(unsigned long evt, return 0; } -static void erratum_workaround_set_sne(struct clock_event_device *clk) -{ -#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND - if (!static_branch_unlikely(&arch_timer_read_ool_enabled)) - return; - - if (arch_timer_uses_ppi == VIRT_PPI) - clk->set_next_event = erratum_set_next_event_virt; - else - clk->set_next_event = erratum_set_next_event_phys; -#endif -} - static void __arch_timer_setup(unsigned type, struct clock_event_device *clk) { @@ -541,8 +535,6 @@ static void __arch_timer_setup(unsigned type, } arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL); - - erratum_workaround_set_sne(clk); } else { clk->features |= CLOCK_EVT_FEAT_DYNIRQ; clk->name = "arch_mem_timer"; -- cgit v1.2.3 From 01d3e3ff26080040cb02ec4989a0da36d069f1e9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 27 Jan 2017 10:27:09 +0000 Subject: arm64: arch_timer: Rework the set_next_event workarounds The way we work around errata affecting set_next_event is not very nice, at it imposes this workaround on errata that do not need it. Add new workaround hooks and let the existing workarounds use them. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 2c02e25d1475..a0c9ee80147e 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -282,6 +282,8 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0, .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0, .read_cntvct_el0 = fsl_a008585_read_cntvct_el0, + .set_next_event_phys = erratum_set_next_event_tval_phys, + .set_next_event_virt = erratum_set_next_event_tval_virt, }, #endif #ifdef CONFIG_HISILICON_ERRATUM_161010101 @@ -292,6 +294,8 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, + .set_next_event_phys = erratum_set_next_event_tval_phys, + .set_next_event_virt = erratum_set_next_event_tval_virt, }, #endif }; @@ -377,11 +381,24 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t local ? "local" : "global", wa->desc); } +#define erratum_handler(fn, r, ...) \ +({ \ + bool __val; \ + if (needs_unstable_timer_counter_workaround() && \ + timer_unstable_counter_workaround->fn) { \ + r = timer_unstable_counter_workaround->fn(__VA_ARGS__); \ + __val = true; \ + } else { \ + __val = false; \ + } \ + __val; \ +}) + #else #define arch_timer_check_ool_workaround(t,a) do { } while(0) #define erratum_set_next_event_tval_virt(...) ({BUG(); 0;}) #define erratum_set_next_event_tval_phys(...) ({BUG(); 0;}) -#define needs_unstable_timer_counter_workaround() ({false;}) +#define erratum_handler(fn, r, ...) ({false;}) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static __always_inline irqreturn_t timer_handler(const int access, @@ -474,8 +491,10 @@ static __always_inline void set_next_event(const int access, unsigned long evt, static int arch_timer_set_next_event_virt(unsigned long evt, struct clock_event_device *clk) { - if (needs_unstable_timer_counter_workaround()) - return erratum_set_next_event_tval_virt(evt, clk); + int ret; + + if (erratum_handler(set_next_event_virt, ret, evt, clk)) + return ret; set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk); return 0; @@ -484,8 +503,10 @@ static int arch_timer_set_next_event_virt(unsigned long evt, static int arch_timer_set_next_event_phys(unsigned long evt, struct clock_event_device *clk) { - if (needs_unstable_timer_counter_workaround()) - return erratum_set_next_event_tval_phys(evt, clk); + int ret; + + if (erratum_handler(set_next_event_phys, ret, evt, clk)) + return ret; set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk); return 0; -- cgit v1.2.3 From 6acc71ccac7187fc0ef85f10bd09c2058f21fab5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 20 Feb 2017 18:34:48 +0000 Subject: arm64: arch_timer: Allows a CPU-specific erratum to only affect a subset of CPUs Instead of applying a CPU-specific workaround to all CPUs in the system, allow it to only affect a subset of them (typical big-little case). This is done by turning the erratum pointer into a per-CPU variable. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 42 +++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index a0c9ee80147e..4551587bcb44 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -235,7 +235,8 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) #endif #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND -const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround = NULL; +DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, + timer_unstable_counter_workaround); EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); @@ -338,9 +339,18 @@ arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, } static -void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa) +void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa, + bool local) { - timer_unstable_counter_workaround = wa; + int i; + + if (local) { + __this_cpu_write(timer_unstable_counter_workaround, wa); + } else { + for_each_possible_cpu(i) + per_cpu(timer_unstable_counter_workaround, i) = wa; + } + static_branch_enable(&arch_timer_read_ool_enabled); } @@ -369,14 +379,17 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t return; if (needs_unstable_timer_counter_workaround()) { - if (wa != timer_unstable_counter_workaround) + const struct arch_timer_erratum_workaround *__wa; + __wa = __this_cpu_read(timer_unstable_counter_workaround); + if (__wa && wa != __wa) pr_warn("Can't enable workaround for %s (clashes with %s\n)", - wa->desc, - timer_unstable_counter_workaround->desc); - return; + wa->desc, __wa->desc); + + if (__wa) + return; } - arch_timer_enable_workaround(wa); + arch_timer_enable_workaround(wa, local); pr_info("Enabling %s workaround for %s\n", local ? "local" : "global", wa->desc); } @@ -384,10 +397,15 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t #define erratum_handler(fn, r, ...) \ ({ \ bool __val; \ - if (needs_unstable_timer_counter_workaround() && \ - timer_unstable_counter_workaround->fn) { \ - r = timer_unstable_counter_workaround->fn(__VA_ARGS__); \ - __val = true; \ + if (needs_unstable_timer_counter_workaround()) { \ + const struct arch_timer_erratum_workaround *__wa; \ + __wa = __this_cpu_read(timer_unstable_counter_workaround); \ + if (__wa && __wa->fn) { \ + r = __wa->fn(__VA_ARGS__); \ + __val = true; \ + } else { \ + __val = false; \ + } \ } else { \ __val = false; \ } \ -- cgit v1.2.3 From 992dd16f6528bd3f378dbcda43688d374be02943 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 1 Feb 2017 11:53:46 +0000 Subject: arm64: arch_timer: Move clocksource_counter and co around In order to access clocksource_counter from the errata handling code, move it (together with the related structures and functions) towards the top of the file. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 62 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 4551587bcb44..395f5d95a737 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -158,6 +158,37 @@ u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, return val; } +/* + * Default to cp15 based access because arm64 uses this function for + * sched_clock() before DT is probed and the cp15 method is guaranteed + * to exist on arm64. arm doesn't use this before DT is probed so even + * if we don't have the cp15 accessors we won't have a problem. + */ +u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; + +static u64 arch_counter_read(struct clocksource *cs) +{ + return arch_timer_read_counter(); +} + +static u64 arch_counter_read_cc(const struct cyclecounter *cc) +{ + return arch_timer_read_counter(); +} + +static struct clocksource clocksource_counter = { + .name = "arch_sys_counter", + .rating = 400, + .read = arch_counter_read, + .mask = CLOCKSOURCE_MASK(56), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +static struct cyclecounter cyclecounter __ro_after_init = { + .read = arch_counter_read_cc, + .mask = CLOCKSOURCE_MASK(56), +}; + #ifdef CONFIG_FSL_ERRATUM_A008585 /* * The number of retries is an arbitrary value well beyond the highest number @@ -742,37 +773,6 @@ static u64 arch_counter_get_cntvct_mem(void) return ((u64) vct_hi << 32) | vct_lo; } -/* - * Default to cp15 based access because arm64 uses this function for - * sched_clock() before DT is probed and the cp15 method is guaranteed - * to exist on arm64. arm doesn't use this before DT is probed so even - * if we don't have the cp15 accessors we won't have a problem. - */ -u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct; - -static u64 arch_counter_read(struct clocksource *cs) -{ - return arch_timer_read_counter(); -} - -static u64 arch_counter_read_cc(const struct cyclecounter *cc) -{ - return arch_timer_read_counter(); -} - -static struct clocksource clocksource_counter = { - .name = "arch_sys_counter", - .rating = 400, - .read = arch_counter_read, - .mask = CLOCKSOURCE_MASK(56), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -static struct cyclecounter cyclecounter __ro_after_init = { - .read = arch_counter_read_cc, - .mask = CLOCKSOURCE_MASK(56), -}; - static struct arch_timer_kvm_info arch_timer_kvm_info; struct arch_timer_kvm_info *arch_timer_get_kvm_info(void) -- cgit v1.2.3 From bee67c53a990452c4f3c7a0e9cebfecb5acfbde3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 4 Apr 2017 17:05:16 +0100 Subject: arm64: arch_timer: Save cntkctl_el1 as a per-cpu variable As we're about to allow per CPU cntkctl_el1 configuration, we cannot rely on the register value to be common when performing power management. Let's turn saved_cntkctl into a per-cpu variable. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 395f5d95a737..5c114da0ed71 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -839,14 +839,14 @@ static int arch_timer_dying_cpu(unsigned int cpu) } #ifdef CONFIG_CPU_PM -static unsigned int saved_cntkctl; +static DEFINE_PER_CPU(unsigned long, saved_cntkctl); static int arch_timer_cpu_pm_notify(struct notifier_block *self, unsigned long action, void *hcpu) { if (action == CPU_PM_ENTER) - saved_cntkctl = arch_timer_get_cntkctl(); + __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl()); else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) - arch_timer_set_cntkctl(saved_cntkctl); + arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl)); return NOTIFY_OK; } -- cgit v1.2.3 From a86bd139f2ae02f960caeb0c1dd5f871d3e087cd Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 1 Feb 2017 12:07:15 +0000 Subject: arm64: arch_timer: Enable CNTVCT_EL0 trap if workaround is enabled Userspace being allowed to use read CNTVCT_EL0 anytime (and not only in the VDSO), we need to enable trapping whenever a cntvct workaround is enabled on a given CPU. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 45 +++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 5c114da0ed71..f8adea258cf0 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -83,6 +83,7 @@ static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI; static bool arch_timer_c3stop; static bool arch_timer_mem_use_virtual; static bool arch_counter_suspend_stop; +static bool vdso_default = true; static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); @@ -383,6 +384,17 @@ void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa } static_branch_enable(&arch_timer_read_ool_enabled); + + /* + * Don't use the vdso fastpath if errata require using the + * out-of-line counter accessor. We may change our mind pretty + * late in the game (with a per-CPU erratum, for example), so + * change both the default value and the vdso itself. + */ + if (wa->read_cntvct_el0) { + clocksource_counter.archdata.vdso_direct = false; + vdso_default = false; + } } static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type, @@ -443,11 +455,19 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t __val; \ }) +static bool arch_timer_this_cpu_has_cntvct_wa(void) +{ + const struct arch_timer_erratum_workaround *wa; + + wa = __this_cpu_read(timer_unstable_counter_workaround); + return wa && wa->read_cntvct_el0; +} #else #define arch_timer_check_ool_workaround(t,a) do { } while(0) #define erratum_set_next_event_tval_virt(...) ({BUG(); 0;}) #define erratum_set_next_event_tval_phys(...) ({BUG(); 0;}) #define erratum_handler(fn, r, ...) ({false;}) +#define arch_timer_this_cpu_has_cntvct_wa() ({false;}) #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */ static __always_inline irqreturn_t timer_handler(const int access, @@ -660,15 +680,23 @@ static void arch_counter_set_user_access(void) { u32 cntkctl = arch_timer_get_cntkctl(); - /* Disable user access to the timers and the physical counter */ + /* Disable user access to the timers and both counters */ /* Also disable virtual event stream */ cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN | ARCH_TIMER_USR_VT_ACCESS_EN + | ARCH_TIMER_USR_VCT_ACCESS_EN | ARCH_TIMER_VIRT_EVT_EN | ARCH_TIMER_USR_PCT_ACCESS_EN); - /* Enable user access to the virtual counter */ - cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; + /* + * Enable user access to the virtual counter if it doesn't + * need to be workaround. The vdso may have been already + * disabled though. + */ + if (arch_timer_this_cpu_has_cntvct_wa()) + pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id()); + else + cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; arch_timer_set_cntkctl(cntkctl); } @@ -791,16 +819,7 @@ static void __init arch_counter_register(unsigned type) else arch_timer_read_counter = arch_counter_get_cntpct; - clocksource_counter.archdata.vdso_direct = true; - -#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND - /* - * Don't use the vdso fastpath if errata require using - * the out-of-line counter accessor. - */ - if (static_branch_unlikely(&arch_timer_read_ool_enabled)) - clocksource_counter.archdata.vdso_direct = false; -#endif + clocksource_counter.archdata.vdso_direct = vdso_default; } else { arch_timer_read_counter = arch_counter_get_cntvct_mem; } -- cgit v1.2.3 From fa8d815fac96e7c9247783d5a1f8fa4685b3c543 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 27 Jan 2017 12:52:31 +0000 Subject: arm64: arch_timer: Workaround for Cortex-A73 erratum 858921 Cortex-A73 (all versions) counter read can return a wrong value when the counter crosses a 32bit boundary. The workaround involves performing the read twice, and to return one or the other depending on whether a transition has taken place. Acked-by: Thomas Gleixner Signed-off-by: Marc Zyngier --- drivers/clocksource/Kconfig | 11 +++++++++++ drivers/clocksource/arm_arch_timer.c | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 3356ab821624..af80a7c44faf 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -366,6 +366,17 @@ config HISILICON_ERRATUM_161010101 161010101. The workaround will be active if the hisilicon,erratum-161010101 property is found in the timer node. +config ARM64_ERRATUM_858921 + bool "Workaround for Cortex-A73 erratum 858921" + default y + select ARM_ARCH_TIMER_OOL_WORKAROUND + depends on ARM_ARCH_TIMER && ARM64 + help + This option enables a workaround applicable to Cortex-A73 + (all versions), whose counter may return incorrect values. + The workaround will be dynamically enabled when an affected + core is detected. + config ARM_GLOBAL_TIMER bool "Support for the ARM global timer" if COMPILE_TEST select CLKSRC_OF if OF diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index f8adea258cf0..8459d19a84f5 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -266,6 +266,17 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) } #endif +#ifdef CONFIG_ARM64_ERRATUM_858921 +static u64 notrace arm64_858921_read_cntvct_el0(void) +{ + u64 old, new; + + old = read_sysreg(cntvct_el0); + new = read_sysreg(cntvct_el0); + return (((old ^ new) >> 32) & 1) ? old : new; +} +#endif + #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround); @@ -331,6 +342,14 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .set_next_event_virt = erratum_set_next_event_tval_virt, }, #endif +#ifdef CONFIG_ARM64_ERRATUM_858921 + { + .match_type = ate_match_local_cap_id, + .id = (void *)ARM64_WORKAROUND_858921, + .desc = "ARM erratum 858921", + .read_cntvct_el0 = arm64_858921_read_cntvct_el0, + }, +#endif }; typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *, -- cgit v1.2.3 From 5a38bcac1f2f0bd0d24700690e36a277ffd0396d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Feb 2017 14:37:30 +0000 Subject: arm64: arch_timer: Allow erratum matching with ACPI OEM information Just as we're able to identify a broken platform using some DT information, let's enable a way to spot the offenders with ACPI. The difference is that we can only match on some OEM info instead of implementation-specific properties. So in order to avoid the insane multiplication of errata structures, we allow an array of OEM descriptions to be attached to an erratum structure. Acked-by: Thomas Gleixner Tested-by: dann frazier Tested-by: Hanjun Guo Reviewed-by: Hanjun Guo Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 8459d19a84f5..887f6d00a71d 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -190,6 +190,12 @@ static struct cyclecounter cyclecounter __ro_after_init = { .mask = CLOCKSOURCE_MASK(56), }; +struct ate_acpi_oem_info { + char oem_id[ACPI_OEM_ID_SIZE + 1]; + char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; + u32 oem_revision; +}; + #ifdef CONFIG_FSL_ERRATUM_A008585 /* * The number of retries is an arbitrary value well beyond the highest number @@ -371,6 +377,28 @@ bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workarou return this_cpu_has_cap((uintptr_t)wa->id); } + +static +bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa, + const void *arg) +{ + static const struct ate_acpi_oem_info empty_oem_info = {}; + const struct ate_acpi_oem_info *info = wa->id; + const struct acpi_table_header *table = arg; + + /* Iterate over the ACPI OEM info array, looking for a match */ + while (memcmp(info, &empty_oem_info, sizeof(*info))) { + if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) && + !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && + info->oem_revision == table->oem_revision) + return true; + + info++; + } + + return false; +} + static const struct arch_timer_erratum_workaround * arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, ate_match_fn_t match_fn, @@ -431,6 +459,9 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t match_fn = arch_timer_check_local_cap_erratum; local = true; break; + case ate_match_acpi_oem_info: + match_fn = arch_timer_check_acpi_oem_erratum; + break; default: WARN_ON(1); return; @@ -1277,6 +1308,9 @@ static int __init arch_timer_acpi_init(struct acpi_table_header *table) /* Always-on capability */ arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON); + /* Check for globally applicable workarounds */ + arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table); + arch_timer_init(); return 0; } -- cgit v1.2.3 From d003d029cea8a28139b4f9b88a36b8fac864f45b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Feb 2017 15:04:27 +0000 Subject: arm64: arch_timer: Add HISILICON_ERRATUM_161010101 ACPI matching data In order to deal with ACPI enabled platforms suffering from the HISILICON_ERRATUM_161010101, let's add the required OEM data that allow the workaround to be enabled. Acked-by: Thomas Gleixner Tested-by: dann frazier Tested-by: Hanjun Guo Reviewed-by: Hanjun Guo Signed-off-by: Marc Zyngier --- drivers/clocksource/arm_arch_timer.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'drivers/clocksource') diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 887f6d00a71d..bf9e9d76375e 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -270,6 +270,29 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) { return __hisi_161010101_read_reg(cntvct_el0); } + +static struct ate_acpi_oem_info hisi_161010101_oem_info[] = { + /* + * Note that trailing spaces are required to properly match + * the OEM table information. + */ + { + .oem_id = "HISI ", + .oem_table_id = "HIP05 ", + .oem_revision = 0, + }, + { + .oem_id = "HISI ", + .oem_table_id = "HIP06 ", + .oem_revision = 0, + }, + { + .oem_id = "HISI ", + .oem_table_id = "HIP07 ", + .oem_revision = 0, + }, + { /* Sentinel indicating the end of the OEM array */ }, +}; #endif #ifdef CONFIG_ARM64_ERRATUM_858921 @@ -347,6 +370,16 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .set_next_event_phys = erratum_set_next_event_tval_phys, .set_next_event_virt = erratum_set_next_event_tval_virt, }, + { + .match_type = ate_match_acpi_oem_info, + .id = hisi_161010101_oem_info, + .desc = "HiSilicon erratum 161010101", + .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0, + .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0, + .read_cntvct_el0 = hisi_161010101_read_cntvct_el0, + .set_next_event_phys = erratum_set_next_event_tval_phys, + .set_next_event_virt = erratum_set_next_event_tval_virt, + }, #endif #ifdef CONFIG_ARM64_ERRATUM_858921 { -- cgit v1.2.3