summaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/cpu/aperfmperf.c
blob: b3fa61d45352e7120d7be7fb5cdb9d6f071c9729 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
// SPDX-License-Identifier: GPL-2.0-only
/*
 * x86 APERF/MPERF KHz calculation for
 * /sys/.../cpufreq/scaling_cur_freq
 *
 * Copyright (C) 2017 Intel Corp.
 * Author: Len Brown <len.brown@intel.com>
 */
#include <linux/cpufreq.h>
#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/math64.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/sched/isolation.h>
#include <linux/sched/topology.h>
#include <linux/smp.h>
#include <linux/syscore_ops.h>

#include <asm/cpu.h>
#include <asm/cpu_device_id.h>
#include <asm/intel-family.h>

#include "cpu.h"

struct aperfmperf {
	seqcount_t	seq;
	unsigned long	last_update;
	u64		acnt;
	u64		mcnt;
	u64		aperf;
	u64		mperf;
};

static DEFINE_PER_CPU_SHARED_ALIGNED(struct aperfmperf, cpu_samples) = {
	.seq = SEQCNT_ZERO(cpu_samples.seq)
};

static void init_counter_refs(void)
{
	u64 aperf, mperf;

	rdmsrl(MSR_IA32_APERF, aperf);
	rdmsrl(MSR_IA32_MPERF, mperf);

	this_cpu_write(cpu_samples.aperf, aperf);
	this_cpu_write(cpu_samples.mperf, mperf);
}

#if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
/*
 * APERF/MPERF frequency ratio computation.
 *
 * The scheduler wants to do frequency invariant accounting and needs a <1
 * ratio to account for the 'current' frequency, corresponding to
 * freq_curr / freq_max.
 *
 * Since the frequency freq_curr on x86 is controlled by micro-controller and
 * our P-state setting is little more than a request/hint, we need to observe
 * the effective frequency 'BusyMHz', i.e. the average frequency over a time
 * interval after discarding idle time. This is given by:
 *
 *   BusyMHz = delta_APERF / delta_MPERF * freq_base
 *
 * where freq_base is the max non-turbo P-state.
 *
 * The freq_max term has to be set to a somewhat arbitrary value, because we
 * can't know which turbo states will be available at a given point in time:
 * it all depends on the thermal headroom of the entire package. We set it to
 * the turbo level with 4 cores active.
 *
 * Benchmarks show that's a good compromise between the 1C turbo ratio
 * (freq_curr/freq_max would rarely reach 1) and something close to freq_base,
 * which would ignore the entire turbo range (a conspicuous part, making
 * freq_curr/freq_max always maxed out).
 *
 * An exception to the heuristic above is the Atom uarch, where we choose the
 * highest turbo level for freq_max since Atom's are generally oriented towards
 * power efficiency.
 *
 * Setting freq_max to anything less than the 1C turbo ratio makes the ratio
 * freq_curr / freq_max to eventually grow >1, in which case we clip it to 1.
 */

DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key);

static u64 arch_turbo_freq_ratio = SCHED_CAPACITY_SCALE;
static u64 arch_max_freq_ratio = SCHED_CAPACITY_SCALE;

void arch_set_max_freq_ratio(bool turbo_disabled)
{
	arch_max_freq_ratio = turbo_disabled ? SCHED_CAPACITY_SCALE :
					arch_turbo_freq_ratio;
}
EXPORT_SYMBOL_GPL(arch_set_max_freq_ratio);

static bool __init turbo_disabled(void)
{
	u64 misc_en;
	int err;

	err = rdmsrl_safe(MSR_IA32_MISC_ENABLE, &misc_en);
	if (err)
		return false;

	return (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
}

static bool __init slv_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
{
	int err;

	err = rdmsrl_safe(MSR_ATOM_CORE_RATIOS, base_freq);
	if (err)
		return false;

	err = rdmsrl_safe(MSR_ATOM_CORE_TURBO_RATIOS, turbo_freq);
	if (err)
		return false;

	*base_freq = (*base_freq >> 16) & 0x3F;     /* max P state */
	*turbo_freq = *turbo_freq & 0x3F;           /* 1C turbo    */

	return true;
}

#define X86_MATCH(vfm)						\
	X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_APERFMPERF, NULL)

static const struct x86_cpu_id has_knl_turbo_ratio_limits[] __initconst = {
	X86_MATCH(INTEL_XEON_PHI_KNL),
	X86_MATCH(INTEL_XEON_PHI_KNM),
	{}
};

static const struct x86_cpu_id has_skx_turbo_ratio_limits[] __initconst = {
	X86_MATCH(INTEL_SKYLAKE_X),
	{}
};

static const struct x86_cpu_id has_glm_turbo_ratio_limits[] __initconst = {
	X86_MATCH(INTEL_ATOM_GOLDMONT),
	X86_MATCH(INTEL_ATOM_GOLDMONT_D),
	X86_MATCH(INTEL_ATOM_GOLDMONT_PLUS),
	{}
};

static bool __init knl_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq,
					  int num_delta_fratio)
{
	int fratio, delta_fratio, found;
	int err, i;
	u64 msr;

	err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
	if (err)
		return false;

	*base_freq = (*base_freq >> 8) & 0xFF;	    /* max P state */

	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
	if (err)
		return false;

	fratio = (msr >> 8) & 0xFF;
	i = 16;
	found = 0;
	do {
		if (found >= num_delta_fratio) {
			*turbo_freq = fratio;
			return true;
		}

		delta_fratio = (msr >> (i + 5)) & 0x7;

		if (delta_fratio) {
			found += 1;
			fratio -= delta_fratio;
		}

		i += 8;
	} while (i < 64);

	return true;
}

static bool __init skx_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq, int size)
{
	u64 ratios, counts;
	u32 group_size;
	int err, i;

	err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
	if (err)
		return false;

	*base_freq = (*base_freq >> 8) & 0xFF;      /* max P state */

	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &ratios);
	if (err)
		return false;

	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT1, &counts);
	if (err)
		return false;

	for (i = 0; i < 64; i += 8) {
		group_size = (counts >> i) & 0xFF;
		if (group_size >= size) {
			*turbo_freq = (ratios >> i) & 0xFF;
			return true;
		}
	}

	return false;
}

static bool __init core_set_max_freq_ratio(u64 *base_freq, u64 *turbo_freq)
{
	u64 msr;
	int err;

	err = rdmsrl_safe(MSR_PLATFORM_INFO, base_freq);
	if (err)
		return false;

	err = rdmsrl_safe(MSR_TURBO_RATIO_LIMIT, &msr);
	if (err)
		return false;

	*base_freq = (*base_freq >> 8) & 0xFF;    /* max P state */
	*turbo_freq = (msr >> 24) & 0xFF;         /* 4C turbo    */

	/* The CPU may have less than 4 cores */
	if (!*turbo_freq)
		*turbo_freq = msr & 0xFF;         /* 1C turbo    */

	return true;
}

static bool __init intel_set_max_freq_ratio(void)
{
	u64 base_freq, turbo_freq;
	u64 turbo_ratio;

	if (slv_set_max_freq_ratio(&base_freq, &turbo_freq))
		goto out;

	if (x86_match_cpu(has_glm_turbo_ratio_limits) &&
	    skx_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
		goto out;

	if (x86_match_cpu(has_knl_turbo_ratio_limits) &&
	    knl_set_max_freq_ratio(&base_freq, &turbo_freq, 1))
		goto out;

	if (x86_match_cpu(has_skx_turbo_ratio_limits) &&
	    skx_set_max_freq_ratio(&base_freq, &turbo_freq, 4))
		goto out;

	if (core_set_max_freq_ratio(&base_freq, &turbo_freq))
		goto out;

	return false;

out:
	/*
	 * Some hypervisors advertise X86_FEATURE_APERFMPERF
	 * but then fill all MSR's with zeroes.
	 * Some CPUs have turbo boost but don't declare any turbo ratio
	 * in MSR_TURBO_RATIO_LIMIT.
	 */
	if (!base_freq || !turbo_freq) {
		pr_debug("Couldn't determine cpu base or turbo frequency, necessary for scale-invariant accounting.\n");
		return false;
	}

	turbo_ratio = div_u64(turbo_freq * SCHED_CAPACITY_SCALE, base_freq);
	if (!turbo_ratio) {
		pr_debug("Non-zero turbo and base frequencies led to a 0 ratio.\n");
		return false;
	}

	arch_turbo_freq_ratio = turbo_ratio;
	arch_set_max_freq_ratio(turbo_disabled());

	return true;
}

#ifdef CONFIG_PM_SLEEP
static struct syscore_ops freq_invariance_syscore_ops = {
	.resume = init_counter_refs,
};

static void register_freq_invariance_syscore_ops(void)
{
	register_syscore_ops(&freq_invariance_syscore_ops);
}
#else
static inline void register_freq_invariance_syscore_ops(void) {}
#endif

static void freq_invariance_enable(void)
{
	if (static_branch_unlikely(&arch_scale_freq_key)) {
		WARN_ON_ONCE(1);
		return;
	}
	static_branch_enable(&arch_scale_freq_key);
	register_freq_invariance_syscore_ops();
	pr_info("Estimated ratio of average max frequency by base frequency (times 1024): %llu\n", arch_max_freq_ratio);
}

void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled)
{
	arch_turbo_freq_ratio = ratio;
	arch_set_max_freq_ratio(turbo_disabled);
	freq_invariance_enable();
}

static void __init bp_init_freq_invariance(void)
{
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return;

	if (intel_set_max_freq_ratio())
		freq_invariance_enable();
}

static void disable_freq_invariance_workfn(struct work_struct *work)
{
	int cpu;

	static_branch_disable(&arch_scale_freq_key);

	/*
	 * Set arch_freq_scale to a default value on all cpus
	 * This negates the effect of scaling
	 */
	for_each_possible_cpu(cpu)
		per_cpu(arch_freq_scale, cpu) = SCHED_CAPACITY_SCALE;
}

static DECLARE_WORK(disable_freq_invariance_work,
		    disable_freq_invariance_workfn);

DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE;
EXPORT_PER_CPU_SYMBOL_GPL(arch_freq_scale);

static void scale_freq_tick(u64 acnt, u64 mcnt)
{
	u64 freq_scale;

	if (!arch_scale_freq_invariant())
		return;

	if (check_shl_overflow(acnt, 2*SCHED_CAPACITY_SHIFT, &acnt))
		goto error;

	if (check_mul_overflow(mcnt, arch_max_freq_ratio, &mcnt) || !mcnt)
		goto error;

	freq_scale = div64_u64(acnt, mcnt);
	if (!freq_scale)
		goto error;

	if (freq_scale > SCHED_CAPACITY_SCALE)
		freq_scale = SCHED_CAPACITY_SCALE;

	this_cpu_write(arch_freq_scale, freq_scale);
	return;

error:
	pr_warn("Scheduler frequency invariance went wobbly, disabling!\n");
	schedule_work(&disable_freq_invariance_work);
}
#else
static inline void bp_init_freq_invariance(void) { }
static inline void scale_freq_tick(u64 acnt, u64 mcnt) { }
#endif /* CONFIG_X86_64 && CONFIG_SMP */

void arch_scale_freq_tick(void)
{
	struct aperfmperf *s = this_cpu_ptr(&cpu_samples);
	u64 acnt, mcnt, aperf, mperf;

	if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
		return;

	rdmsrl(MSR_IA32_APERF, aperf);
	rdmsrl(MSR_IA32_MPERF, mperf);
	acnt = aperf - s->aperf;
	mcnt = mperf - s->mperf;

	s->aperf = aperf;
	s->mperf = mperf;

	raw_write_seqcount_begin(&s->seq);
	s->last_update = jiffies;
	s->acnt = acnt;
	s->mcnt = mcnt;
	raw_write_seqcount_end(&s->seq);

	scale_freq_tick(acnt, mcnt);
}

/*
 * Discard samples older than the define maximum sample age of 20ms. There
 * is no point in sending IPIs in such a case. If the scheduler tick was
 * not running then the CPU is either idle or isolated.
 */
#define MAX_SAMPLE_AGE	((unsigned long)HZ / 50)

unsigned int arch_freq_get_on_cpu(int cpu)
{
	struct aperfmperf *s = per_cpu_ptr(&cpu_samples, cpu);
	unsigned int seq, freq;
	unsigned long last;
	u64 acnt, mcnt;

	if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
		goto fallback;

	do {
		seq = raw_read_seqcount_begin(&s->seq);
		last = s->last_update;
		acnt = s->acnt;
		mcnt = s->mcnt;
	} while (read_seqcount_retry(&s->seq, seq));

	/*
	 * Bail on invalid count and when the last update was too long ago,
	 * which covers idle and NOHZ full CPUs.
	 */
	if (!mcnt || (jiffies - last) > MAX_SAMPLE_AGE)
		goto fallback;

	return div64_u64((cpu_khz * acnt), mcnt);

fallback:
	freq = cpufreq_quick_get(cpu);
	return freq ? freq : cpu_khz;
}

static int __init bp_init_aperfmperf(void)
{
	if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF))
		return 0;

	init_counter_refs();
	bp_init_freq_invariance();
	return 0;
}
early_initcall(bp_init_aperfmperf);

void ap_init_aperfmperf(void)
{
	if (cpu_feature_enabled(X86_FEATURE_APERFMPERF))
		init_counter_refs();
}