summaryrefslogtreecommitdiffstats
path: root/include/linux/padata.h
blob: 0146daf3443066d8097181a5cdf49e87af24070d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * padata.h - header for the padata parallelization interface
 *
 * Copyright (C) 2008, 2009 secunet Security Networks AG
 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
 *
 * Copyright (c) 2020 Oracle and/or its affiliates.
 * Author: Daniel Jordan <daniel.m.jordan@oracle.com>
 */

#ifndef PADATA_H
#define PADATA_H

#include <linux/refcount.h>
#include <linux/compiler_types.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/kobject.h>

#define PADATA_CPU_SERIAL   0x01
#define PADATA_CPU_PARALLEL 0x02

/**
 * struct padata_priv - Represents one job
 *
 * @list: List entry, to attach to the padata lists.
 * @pd: Pointer to the internal control structure.
 * @cb_cpu: Callback cpu for serializatioon.
 * @seq_nr: Sequence number of the parallelized data object.
 * @info: Used to pass information from the parallel to the serial function.
 * @parallel: Parallel execution function.
 * @serial: Serial complete function.
 */
struct padata_priv {
	struct list_head	list;
	struct parallel_data	*pd;
	int			cb_cpu;
	unsigned int		seq_nr;
	int			info;
	void                    (*parallel)(struct padata_priv *padata);
	void                    (*serial)(struct padata_priv *padata);
};

/**
 * struct padata_list - one per work type per CPU
 *
 * @list: List head.
 * @lock: List lock.
 */
struct padata_list {
	struct list_head        list;
	spinlock_t              lock;
};

/**
* struct padata_serial_queue - The percpu padata serial queue
*
* @serial: List to wait for serialization after reordering.
* @work: work struct for serialization.
* @pd: Backpointer to the internal control structure.
*/
struct padata_serial_queue {
       struct padata_list    serial;
       struct work_struct    work;
       struct parallel_data *pd;
};

/**
 * struct padata_cpumask - The cpumasks for the parallel/serial workers
 *
 * @pcpu: cpumask for the parallel workers.
 * @cbcpu: cpumask for the serial (callback) workers.
 */
struct padata_cpumask {
	cpumask_var_t	pcpu;
	cpumask_var_t	cbcpu;
};

/**
 * struct parallel_data - Internal control structure, covers everything
 * that depends on the cpumask in use.
 *
 * @ps: padata_shell object.
 * @reorder_list: percpu reorder lists
 * @squeue: percpu padata queues used for serialuzation.
 * @refcnt: Number of objects holding a reference on this parallel_data.
 * @seq_nr: Sequence number of the parallelized data object.
 * @processed: Number of already processed objects.
 * @cpu: Next CPU to be processed.
 * @cpumask: The cpumasks in use for parallel and serial workers.
 * @reorder_work: work struct for reordering.
 * @lock: Reorder lock.
 */
struct parallel_data {
	struct padata_shell		*ps;
	struct padata_list		__percpu *reorder_list;
	struct padata_serial_queue	__percpu *squeue;
	refcount_t			refcnt;
	unsigned int			seq_nr;
	unsigned int			processed;
	int				cpu;
	struct padata_cpumask		cpumask;
	struct work_struct		reorder_work;
	spinlock_t                      ____cacheline_aligned lock;
};

/**
 * struct padata_shell - Wrapper around struct parallel_data, its
 * purpose is to allow the underlying control structure to be replaced
 * on the fly using RCU.
 *
 * @pinst: padat instance.
 * @pd: Actual parallel_data structure which may be substituted on the fly.
 * @opd: Pointer to old pd to be freed by padata_replace.
 * @list: List entry in padata_instance list.
 */
struct padata_shell {
	struct padata_instance		*pinst;
	struct parallel_data __rcu	*pd;
	struct parallel_data		*opd;
	struct list_head		list;
};

/**
 * struct padata_mt_job - represents one multithreaded job
 *
 * @thread_fn: Called for each chunk of work that a padata thread does.
 * @fn_arg: The thread function argument.
 * @start: The start of the job (units are job-specific).
 * @size: size of this node's work (units are job-specific).
 * @align: Ranges passed to the thread function fall on this boundary, with the
 *         possible exceptions of the beginning and end of the job.
 * @min_chunk: The minimum chunk size in job-specific units.  This allows
 *             the client to communicate the minimum amount of work that's
 *             appropriate for one worker thread to do at once.
 * @max_threads: Max threads to use for the job, actual number may be less
 *               depending on task size and minimum chunk size.
 * @numa_aware: Distribute jobs to different nodes with CPU in a round robin fashion.
 */
struct padata_mt_job {
	void (*thread_fn)(unsigned long start, unsigned long end, void *arg);
	void			*fn_arg;
	unsigned long		start;
	unsigned long		size;
	unsigned long		align;
	unsigned long		min_chunk;
	int			max_threads;
	bool			numa_aware;
};

/**
 * struct padata_instance - The overall control structure.
 *
 * @cpu_online_node: Linkage for CPU online callback.
 * @cpu_dead_node: Linkage for CPU offline callback.
 * @parallel_wq: The workqueue used for parallel work.
 * @serial_wq: The workqueue used for serial work.
 * @pslist: List of padata_shell objects attached to this instance.
 * @cpumask: User supplied cpumasks for parallel and serial works.
 * @kobj: padata instance kernel object.
 * @lock: padata instance lock.
 * @flags: padata flags.
 */
struct padata_instance {
	struct hlist_node		cpu_online_node;
	struct hlist_node		cpu_dead_node;
	struct workqueue_struct		*parallel_wq;
	struct workqueue_struct		*serial_wq;
	struct list_head		pslist;
	struct padata_cpumask		cpumask;
	struct kobject                   kobj;
	struct mutex			 lock;
	u8				 flags;
#define	PADATA_INIT	1
#define	PADATA_RESET	2
#define	PADATA_INVALID	4
};

#ifdef CONFIG_PADATA
extern void __init padata_init(void);
extern struct padata_instance *padata_alloc(const char *name);
extern void padata_free(struct padata_instance *pinst);
extern struct padata_shell *padata_alloc_shell(struct padata_instance *pinst);
extern void padata_free_shell(struct padata_shell *ps);
extern int padata_do_parallel(struct padata_shell *ps,
			      struct padata_priv *padata, int *cb_cpu);
extern void padata_do_serial(struct padata_priv *padata);
extern void __init padata_do_multithreaded(struct padata_mt_job *job);
extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
			      cpumask_var_t cpumask);
#else
static inline void __init padata_init(void) {}
static inline void __init padata_do_multithreaded(struct padata_mt_job *job)
{
	job->thread_fn(job->start, job->start + job->size, job->fn_arg);
}
#endif

#endif