kthread: rename probe_kthread_data() to kthread_probe_data()
[linux-2.6-block.git] / kernel / workqueue.c
CommitLineData
1da177e4 1/*
c54fce6e 2 * kernel/workqueue.c - generic async execution with shared worker pool
1da177e4 3 *
c54fce6e 4 * Copyright (C) 2002 Ingo Molnar
1da177e4 5 *
c54fce6e
TH
6 * Derived from the taskqueue/keventd code by:
7 * David Woodhouse <dwmw2@infradead.org>
8 * Andrew Morton
9 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
10 * Theodore Ts'o <tytso@mit.edu>
1da177e4 11 *
c54fce6e 12 * Made to use alloc_percpu by Christoph Lameter.
1da177e4 13 *
c54fce6e
TH
14 * Copyright (C) 2010 SUSE Linux Products GmbH
15 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
89ada679 16 *
c54fce6e
TH
17 * This is the generic async execution mechanism. Work items as are
18 * executed in process context. The worker pool is shared and
b11895c4
L
19 * automatically managed. There are two worker pools for each CPU (one for
20 * normal work items and the other for high priority ones) and some extra
21 * pools for workqueues which are not bound to any specific CPU - the
22 * number of these backing pools is dynamic.
c54fce6e
TH
23 *
24 * Please read Documentation/workqueue.txt for details.
1da177e4
LT
25 */
26
9984de1a 27#include <linux/export.h>
1da177e4
LT
28#include <linux/kernel.h>
29#include <linux/sched.h>
30#include <linux/init.h>
31#include <linux/signal.h>
32#include <linux/completion.h>
33#include <linux/workqueue.h>
34#include <linux/slab.h>
35#include <linux/cpu.h>
36#include <linux/notifier.h>
37#include <linux/kthread.h>
1fa44eca 38#include <linux/hardirq.h>
46934023 39#include <linux/mempolicy.h>
341a5958 40#include <linux/freezer.h>
d5abe669
PZ
41#include <linux/kallsyms.h>
42#include <linux/debug_locks.h>
4e6045f1 43#include <linux/lockdep.h>
c34056a3 44#include <linux/idr.h>
29c91e99 45#include <linux/jhash.h>
42f8570f 46#include <linux/hashtable.h>
76af4d93 47#include <linux/rculist.h>
bce90380 48#include <linux/nodemask.h>
4c16bd32 49#include <linux/moduleparam.h>
3d1cb205 50#include <linux/uaccess.h>
e22bee78 51
ea138446 52#include "workqueue_internal.h"
1da177e4 53
c8e55f36 54enum {
24647570
TH
55 /*
56 * worker_pool flags
bc2ae0f5 57 *
24647570 58 * A bound pool is either associated or disassociated with its CPU.
bc2ae0f5
TH
59 * While associated (!DISASSOCIATED), all workers are bound to the
60 * CPU and none has %WORKER_UNBOUND set and concurrency management
61 * is in effect.
62 *
63 * While DISASSOCIATED, the cpu may be offline and all workers have
64 * %WORKER_UNBOUND set and concurrency management disabled, and may
24647570 65 * be executing on any CPU. The pool behaves as an unbound one.
bc2ae0f5 66 *
bc3a1afc 67 * Note that DISASSOCIATED should be flipped only while holding
92f9c5c4 68 * attach_mutex to avoid changing binding state while
4736cbf7 69 * worker_attach_to_pool() is in progress.
bc2ae0f5 70 */
24647570 71 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
db7bccf4 72
c8e55f36 73 /* worker flags */
c8e55f36
TH
74 WORKER_DIE = 1 << 1, /* die die die */
75 WORKER_IDLE = 1 << 2, /* is idle */
e22bee78 76 WORKER_PREP = 1 << 3, /* preparing to run works */
fb0e7beb 77 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
f3421797 78 WORKER_UNBOUND = 1 << 7, /* worker is unbound */
a9ab775b 79 WORKER_REBOUND = 1 << 8, /* worker was rebound */
e22bee78 80
a9ab775b
TH
81 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
82 WORKER_UNBOUND | WORKER_REBOUND,
db7bccf4 83
e34cdddb 84 NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
4ce62e9e 85
29c91e99 86 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
c8e55f36 87 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
db7bccf4 88
e22bee78
TH
89 MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
90 IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
91
3233cdbd
TH
92 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
93 /* call for help after 10ms
94 (min two ticks) */
e22bee78
TH
95 MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
96 CREATE_COOLDOWN = HZ, /* time to breath after fail */
e22bee78
TH
97
98 /*
99 * Rescue workers are used only on emergencies and shared by
8698a745 100 * all cpus. Give MIN_NICE.
e22bee78 101 */
8698a745
DY
102 RESCUER_NICE_LEVEL = MIN_NICE,
103 HIGHPRI_NICE_LEVEL = MIN_NICE,
ecf6881f
TH
104
105 WQ_NAME_LEN = 24,
c8e55f36 106};
1da177e4
LT
107
108/*
4690c4ab
TH
109 * Structure fields follow one of the following exclusion rules.
110 *
e41e704b
TH
111 * I: Modifiable by initialization/destruction paths and read-only for
112 * everyone else.
4690c4ab 113 *
e22bee78
TH
114 * P: Preemption protected. Disabling preemption is enough and should
115 * only be modified and accessed from the local cpu.
116 *
d565ed63 117 * L: pool->lock protected. Access with pool->lock held.
4690c4ab 118 *
d565ed63
TH
119 * X: During normal operation, modification requires pool->lock and should
120 * be done only from local cpu. Either disabling preemption on local
121 * cpu or grabbing pool->lock is enough for read access. If
122 * POOL_DISASSOCIATED is set, it's identical to L.
e22bee78 123 *
92f9c5c4 124 * A: pool->attach_mutex protected.
822d8405 125 *
68e13a67 126 * PL: wq_pool_mutex protected.
5bcab335 127 *
68e13a67 128 * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
76af4d93 129 *
5b95e1af
LJ
130 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
131 *
132 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
133 * sched-RCU for reads.
134 *
3c25a55d
LJ
135 * WQ: wq->mutex protected.
136 *
b5927605 137 * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
2e109a28
TH
138 *
139 * MD: wq_mayday_lock protected.
1da177e4 140 */
1da177e4 141
2eaebdb3 142/* struct worker is defined in workqueue_internal.h */
c34056a3 143
bd7bdd43 144struct worker_pool {
d565ed63 145 spinlock_t lock; /* the pool lock */
d84ff051 146 int cpu; /* I: the associated cpu */
f3f90ad4 147 int node; /* I: the associated node ID */
9daf9e67 148 int id; /* I: pool ID */
11ebea50 149 unsigned int flags; /* X: flags */
bd7bdd43 150
82607adc
TH
151 unsigned long watchdog_ts; /* L: watchdog timestamp */
152
bd7bdd43
TH
153 struct list_head worklist; /* L: list of pending works */
154 int nr_workers; /* L: total number of workers */
ea1abd61
LJ
155
156 /* nr_idle includes the ones off idle_list for rebinding */
bd7bdd43
TH
157 int nr_idle; /* L: currently idle ones */
158
159 struct list_head idle_list; /* X: list of idle workers */
160 struct timer_list idle_timer; /* L: worker idle timeout */
161 struct timer_list mayday_timer; /* L: SOS timer for workers */
162
c5aa87bb 163 /* a workers is either on busy_hash or idle_list, or the manager */
c9e7cf27
TH
164 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
165 /* L: hash of busy workers */
166
bc3a1afc 167 /* see manage_workers() for details on the two manager mutexes */
34a06bd6 168 struct mutex manager_arb; /* manager arbitration */
2607d7a6 169 struct worker *manager; /* L: purely informational */
92f9c5c4
LJ
170 struct mutex attach_mutex; /* attach/detach exclusion */
171 struct list_head workers; /* A: attached workers */
60f5a4bc 172 struct completion *detach_completion; /* all workers detached */
e19e397a 173
7cda9aae 174 struct ida worker_ida; /* worker IDs for task name */
e19e397a 175
7a4e344c 176 struct workqueue_attrs *attrs; /* I: worker attributes */
68e13a67
LJ
177 struct hlist_node hash_node; /* PL: unbound_pool_hash node */
178 int refcnt; /* PL: refcnt for unbound pools */
7a4e344c 179
e19e397a
TH
180 /*
181 * The current concurrency level. As it's likely to be accessed
182 * from other CPUs during try_to_wake_up(), put it in a separate
183 * cacheline.
184 */
185 atomic_t nr_running ____cacheline_aligned_in_smp;
29c91e99
TH
186
187 /*
188 * Destruction of pool is sched-RCU protected to allow dereferences
189 * from get_work_pool().
190 */
191 struct rcu_head rcu;
8b03ae3c
TH
192} ____cacheline_aligned_in_smp;
193
1da177e4 194/*
112202d9
TH
195 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
196 * of work_struct->data are used for flags and the remaining high bits
197 * point to the pwq; thus, pwqs need to be aligned at two's power of the
198 * number of flag bits.
1da177e4 199 */
112202d9 200struct pool_workqueue {
bd7bdd43 201 struct worker_pool *pool; /* I: the associated pool */
4690c4ab 202 struct workqueue_struct *wq; /* I: the owning workqueue */
73f53c4a
TH
203 int work_color; /* L: current color */
204 int flush_color; /* L: flushing color */
8864b4e5 205 int refcnt; /* L: reference count */
73f53c4a
TH
206 int nr_in_flight[WORK_NR_COLORS];
207 /* L: nr of in_flight works */
1e19ffc6 208 int nr_active; /* L: nr of active works */
a0a1a5fd 209 int max_active; /* L: max active works */
1e19ffc6 210 struct list_head delayed_works; /* L: delayed works */
3c25a55d 211 struct list_head pwqs_node; /* WR: node on wq->pwqs */
2e109a28 212 struct list_head mayday_node; /* MD: node on wq->maydays */
8864b4e5
TH
213
214 /*
215 * Release of unbound pwq is punted to system_wq. See put_pwq()
216 * and pwq_unbound_release_workfn() for details. pool_workqueue
217 * itself is also sched-RCU protected so that the first pwq can be
b09f4fd3 218 * determined without grabbing wq->mutex.
8864b4e5
TH
219 */
220 struct work_struct unbound_release_work;
221 struct rcu_head rcu;
e904e6c2 222} __aligned(1 << WORK_STRUCT_FLAG_BITS);
1da177e4 223
73f53c4a
TH
224/*
225 * Structure used to wait for workqueue flush.
226 */
227struct wq_flusher {
3c25a55d
LJ
228 struct list_head list; /* WQ: list of flushers */
229 int flush_color; /* WQ: flush color waiting for */
73f53c4a
TH
230 struct completion done; /* flush completion */
231};
232
226223ab
TH
233struct wq_device;
234
1da177e4 235/*
c5aa87bb
TH
236 * The externally visible workqueue. It relays the issued work items to
237 * the appropriate worker_pool through its pool_workqueues.
1da177e4
LT
238 */
239struct workqueue_struct {
3c25a55d 240 struct list_head pwqs; /* WR: all pwqs of this wq */
e2dca7ad 241 struct list_head list; /* PR: list of all workqueues */
73f53c4a 242
3c25a55d
LJ
243 struct mutex mutex; /* protects this wq */
244 int work_color; /* WQ: current work color */
245 int flush_color; /* WQ: current flush color */
112202d9 246 atomic_t nr_pwqs_to_flush; /* flush in progress */
3c25a55d
LJ
247 struct wq_flusher *first_flusher; /* WQ: first flusher */
248 struct list_head flusher_queue; /* WQ: flush waiters */
249 struct list_head flusher_overflow; /* WQ: flush overflow list */
73f53c4a 250
2e109a28 251 struct list_head maydays; /* MD: pwqs requesting rescue */
e22bee78
TH
252 struct worker *rescuer; /* I: rescue worker */
253
87fc741e 254 int nr_drainers; /* WQ: drain in progress */
a357fc03 255 int saved_max_active; /* WQ: saved pwq max_active */
226223ab 256
5b95e1af
LJ
257 struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
258 struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
6029a918 259
226223ab
TH
260#ifdef CONFIG_SYSFS
261 struct wq_device *wq_dev; /* I: for sysfs interface */
262#endif
4e6045f1 263#ifdef CONFIG_LOCKDEP
4690c4ab 264 struct lockdep_map lockdep_map;
4e6045f1 265#endif
ecf6881f 266 char name[WQ_NAME_LEN]; /* I: workqueue name */
2728fd2f 267
e2dca7ad
TH
268 /*
269 * Destruction of workqueue_struct is sched-RCU protected to allow
270 * walking the workqueues list without grabbing wq_pool_mutex.
271 * This is used to dump all workqueues from sysrq.
272 */
273 struct rcu_head rcu;
274
2728fd2f
TH
275 /* hot fields used during command issue, aligned to cacheline */
276 unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
277 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
5b95e1af 278 struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
1da177e4
LT
279};
280
e904e6c2
TH
281static struct kmem_cache *pwq_cache;
282
bce90380
TH
283static cpumask_var_t *wq_numa_possible_cpumask;
284 /* possible CPUs of each node */
285
d55262c4
TH
286static bool wq_disable_numa;
287module_param_named(disable_numa, wq_disable_numa, bool, 0444);
288
cee22a15 289/* see the comment above the definition of WQ_POWER_EFFICIENT */
552f530c 290static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
cee22a15
VK
291module_param_named(power_efficient, wq_power_efficient, bool, 0444);
292
bce90380
TH
293static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
294
4c16bd32
TH
295/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
296static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
297
68e13a67 298static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
2e109a28 299static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
5bcab335 300
e2dca7ad 301static LIST_HEAD(workqueues); /* PR: list of all workqueues */
68e13a67 302static bool workqueue_freezing; /* PL: have wqs started freezing? */
7d19c5ce 303
ef557180
MG
304/* PL: allowable cpus for unbound wqs and work items */
305static cpumask_var_t wq_unbound_cpumask;
306
307/* CPU where unbound work was last round robin scheduled from this CPU */
308static DEFINE_PER_CPU(int, wq_rr_cpu_last);
b05a7928 309
f303fccb
TH
310/*
311 * Local execution of unbound work items is no longer guaranteed. The
312 * following always forces round-robin CPU selection on unbound work items
313 * to uncover usages which depend on it.
314 */
315#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
316static bool wq_debug_force_rr_cpu = true;
317#else
318static bool wq_debug_force_rr_cpu = false;
319#endif
320module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
321
7d19c5ce 322/* the per-cpu worker pools */
25528213 323static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
7d19c5ce 324
68e13a67 325static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
7d19c5ce 326
68e13a67 327/* PL: hash of all unbound pools keyed by pool->attrs */
29c91e99
TH
328static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
329
c5aa87bb 330/* I: attributes used when instantiating standard unbound pools on demand */
29c91e99
TH
331static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
332
8a2b7538
TH
333/* I: attributes used when instantiating ordered pools on demand */
334static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
335
d320c038 336struct workqueue_struct *system_wq __read_mostly;
ad7b1f84 337EXPORT_SYMBOL(system_wq);
044c782c 338struct workqueue_struct *system_highpri_wq __read_mostly;
1aabe902 339EXPORT_SYMBOL_GPL(system_highpri_wq);
044c782c 340struct workqueue_struct *system_long_wq __read_mostly;
d320c038 341EXPORT_SYMBOL_GPL(system_long_wq);
044c782c 342struct workqueue_struct *system_unbound_wq __read_mostly;
f3421797 343EXPORT_SYMBOL_GPL(system_unbound_wq);
044c782c 344struct workqueue_struct *system_freezable_wq __read_mostly;
24d51add 345EXPORT_SYMBOL_GPL(system_freezable_wq);
0668106c
VK
346struct workqueue_struct *system_power_efficient_wq __read_mostly;
347EXPORT_SYMBOL_GPL(system_power_efficient_wq);
348struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
349EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
d320c038 350
7d19c5ce 351static int worker_thread(void *__worker);
6ba94429 352static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
7d19c5ce 353
97bd2347
TH
354#define CREATE_TRACE_POINTS
355#include <trace/events/workqueue.h>
356
68e13a67 357#define assert_rcu_or_pool_mutex() \
f78f5b90
PM
358 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
359 !lockdep_is_held(&wq_pool_mutex), \
360 "sched RCU or wq_pool_mutex should be held")
5bcab335 361
b09f4fd3 362#define assert_rcu_or_wq_mutex(wq) \
f78f5b90
PM
363 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
364 !lockdep_is_held(&wq->mutex), \
365 "sched RCU or wq->mutex should be held")
76af4d93 366
5b95e1af 367#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
f78f5b90
PM
368 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
369 !lockdep_is_held(&wq->mutex) && \
370 !lockdep_is_held(&wq_pool_mutex), \
371 "sched RCU, wq->mutex or wq_pool_mutex should be held")
5b95e1af 372
f02ae73a
TH
373#define for_each_cpu_worker_pool(pool, cpu) \
374 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
375 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
7a62c2c8 376 (pool)++)
4ce62e9e 377
17116969
TH
378/**
379 * for_each_pool - iterate through all worker_pools in the system
380 * @pool: iteration cursor
611c92a0 381 * @pi: integer used for iteration
fa1b54e6 382 *
68e13a67
LJ
383 * This must be called either with wq_pool_mutex held or sched RCU read
384 * locked. If the pool needs to be used beyond the locking in effect, the
385 * caller is responsible for guaranteeing that the pool stays online.
fa1b54e6
TH
386 *
387 * The if/else clause exists only for the lockdep assertion and can be
388 * ignored.
17116969 389 */
611c92a0
TH
390#define for_each_pool(pool, pi) \
391 idr_for_each_entry(&worker_pool_idr, pool, pi) \
68e13a67 392 if (({ assert_rcu_or_pool_mutex(); false; })) { } \
fa1b54e6 393 else
17116969 394
822d8405
TH
395/**
396 * for_each_pool_worker - iterate through all workers of a worker_pool
397 * @worker: iteration cursor
822d8405
TH
398 * @pool: worker_pool to iterate workers of
399 *
92f9c5c4 400 * This must be called with @pool->attach_mutex.
822d8405
TH
401 *
402 * The if/else clause exists only for the lockdep assertion and can be
403 * ignored.
404 */
da028469
LJ
405#define for_each_pool_worker(worker, pool) \
406 list_for_each_entry((worker), &(pool)->workers, node) \
92f9c5c4 407 if (({ lockdep_assert_held(&pool->attach_mutex); false; })) { } \
822d8405
TH
408 else
409
49e3cf44
TH
410/**
411 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
412 * @pwq: iteration cursor
413 * @wq: the target workqueue
76af4d93 414 *
b09f4fd3 415 * This must be called either with wq->mutex held or sched RCU read locked.
794b18bc
TH
416 * If the pwq needs to be used beyond the locking in effect, the caller is
417 * responsible for guaranteeing that the pwq stays online.
76af4d93
TH
418 *
419 * The if/else clause exists only for the lockdep assertion and can be
420 * ignored.
49e3cf44
TH
421 */
422#define for_each_pwq(pwq, wq) \
76af4d93 423 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
b09f4fd3 424 if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
76af4d93 425 else
f3421797 426
dc186ad7
TG
427#ifdef CONFIG_DEBUG_OBJECTS_WORK
428
429static struct debug_obj_descr work_debug_descr;
430
99777288
SG
431static void *work_debug_hint(void *addr)
432{
433 return ((struct work_struct *) addr)->func;
434}
435
b9fdac7f
DC
436static bool work_is_static_object(void *addr)
437{
438 struct work_struct *work = addr;
439
440 return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
441}
442
dc186ad7
TG
443/*
444 * fixup_init is called when:
445 * - an active object is initialized
446 */
02a982a6 447static bool work_fixup_init(void *addr, enum debug_obj_state state)
dc186ad7
TG
448{
449 struct work_struct *work = addr;
450
451 switch (state) {
452 case ODEBUG_STATE_ACTIVE:
453 cancel_work_sync(work);
454 debug_object_init(work, &work_debug_descr);
02a982a6 455 return true;
dc186ad7 456 default:
02a982a6 457 return false;
dc186ad7
TG
458 }
459}
460
dc186ad7
TG
461/*
462 * fixup_free is called when:
463 * - an active object is freed
464 */
02a982a6 465static bool work_fixup_free(void *addr, enum debug_obj_state state)
dc186ad7
TG
466{
467 struct work_struct *work = addr;
468
469 switch (state) {
470 case ODEBUG_STATE_ACTIVE:
471 cancel_work_sync(work);
472 debug_object_free(work, &work_debug_descr);
02a982a6 473 return true;
dc186ad7 474 default:
02a982a6 475 return false;
dc186ad7
TG
476 }
477}
478
479static struct debug_obj_descr work_debug_descr = {
480 .name = "work_struct",
99777288 481 .debug_hint = work_debug_hint,
b9fdac7f 482 .is_static_object = work_is_static_object,
dc186ad7 483 .fixup_init = work_fixup_init,
dc186ad7
TG
484 .fixup_free = work_fixup_free,
485};
486
487static inline void debug_work_activate(struct work_struct *work)
488{
489 debug_object_activate(work, &work_debug_descr);
490}
491
492static inline void debug_work_deactivate(struct work_struct *work)
493{
494 debug_object_deactivate(work, &work_debug_descr);
495}
496
497void __init_work(struct work_struct *work, int onstack)
498{
499 if (onstack)
500 debug_object_init_on_stack(work, &work_debug_descr);
501 else
502 debug_object_init(work, &work_debug_descr);
503}
504EXPORT_SYMBOL_GPL(__init_work);
505
506void destroy_work_on_stack(struct work_struct *work)
507{
508 debug_object_free(work, &work_debug_descr);
509}
510EXPORT_SYMBOL_GPL(destroy_work_on_stack);
511
ea2e64f2
TG
512void destroy_delayed_work_on_stack(struct delayed_work *work)
513{
514 destroy_timer_on_stack(&work->timer);
515 debug_object_free(&work->work, &work_debug_descr);
516}
517EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
518
dc186ad7
TG
519#else
520static inline void debug_work_activate(struct work_struct *work) { }
521static inline void debug_work_deactivate(struct work_struct *work) { }
522#endif
523
4e8b22bd
LB
524/**
525 * worker_pool_assign_id - allocate ID and assing it to @pool
526 * @pool: the pool pointer of interest
527 *
528 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
529 * successfully, -errno on failure.
530 */
9daf9e67
TH
531static int worker_pool_assign_id(struct worker_pool *pool)
532{
533 int ret;
534
68e13a67 535 lockdep_assert_held(&wq_pool_mutex);
5bcab335 536
4e8b22bd
LB
537 ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
538 GFP_KERNEL);
229641a6 539 if (ret >= 0) {
e68035fb 540 pool->id = ret;
229641a6
TH
541 return 0;
542 }
fa1b54e6 543 return ret;
7c3eed5c
TH
544}
545
df2d5ae4
TH
546/**
547 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
548 * @wq: the target workqueue
549 * @node: the node ID
550 *
5b95e1af
LJ
551 * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
552 * read locked.
df2d5ae4
TH
553 * If the pwq needs to be used beyond the locking in effect, the caller is
554 * responsible for guaranteeing that the pwq stays online.
d185af30
YB
555 *
556 * Return: The unbound pool_workqueue for @node.
df2d5ae4
TH
557 */
558static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
559 int node)
560{
5b95e1af 561 assert_rcu_or_wq_mutex_or_pool_mutex(wq);
d6e022f1
TH
562
563 /*
564 * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
565 * delayed item is pending. The plan is to keep CPU -> NODE
566 * mapping valid and stable across CPU on/offlines. Once that
567 * happens, this workaround can be removed.
568 */
569 if (unlikely(node == NUMA_NO_NODE))
570 return wq->dfl_pwq;
571
df2d5ae4
TH
572 return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
573}
574
73f53c4a
TH
575static unsigned int work_color_to_flags(int color)
576{
577 return color << WORK_STRUCT_COLOR_SHIFT;
578}
579
580static int get_work_color(struct work_struct *work)
581{
582 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
583 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
584}
585
586static int work_next_color(int color)
587{
588 return (color + 1) % WORK_NR_COLORS;
589}
1da177e4 590
14441960 591/*
112202d9
TH
592 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
593 * contain the pointer to the queued pwq. Once execution starts, the flag
7c3eed5c 594 * is cleared and the high bits contain OFFQ flags and pool ID.
7a22ad75 595 *
112202d9
TH
596 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
597 * and clear_work_data() can be used to set the pwq, pool or clear
bbb68dfa
TH
598 * work->data. These functions should only be called while the work is
599 * owned - ie. while the PENDING bit is set.
7a22ad75 600 *
112202d9 601 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
7c3eed5c 602 * corresponding to a work. Pool is available once the work has been
112202d9 603 * queued anywhere after initialization until it is sync canceled. pwq is
7c3eed5c 604 * available only while the work item is queued.
7a22ad75 605 *
bbb68dfa
TH
606 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
607 * canceled. While being canceled, a work item may have its PENDING set
608 * but stay off timer and worklist for arbitrarily long and nobody should
609 * try to steal the PENDING bit.
14441960 610 */
7a22ad75
TH
611static inline void set_work_data(struct work_struct *work, unsigned long data,
612 unsigned long flags)
365970a1 613{
6183c009 614 WARN_ON_ONCE(!work_pending(work));
7a22ad75
TH
615 atomic_long_set(&work->data, data | flags | work_static(work));
616}
365970a1 617
112202d9 618static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
7a22ad75
TH
619 unsigned long extra_flags)
620{
112202d9
TH
621 set_work_data(work, (unsigned long)pwq,
622 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
365970a1
DH
623}
624
4468a00f
LJ
625static void set_work_pool_and_keep_pending(struct work_struct *work,
626 int pool_id)
627{
628 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
629 WORK_STRUCT_PENDING);
630}
631
7c3eed5c
TH
632static void set_work_pool_and_clear_pending(struct work_struct *work,
633 int pool_id)
7a22ad75 634{
23657bb1
TH
635 /*
636 * The following wmb is paired with the implied mb in
637 * test_and_set_bit(PENDING) and ensures all updates to @work made
638 * here are visible to and precede any updates by the next PENDING
639 * owner.
640 */
641 smp_wmb();
7c3eed5c 642 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
346c09f8
RP
643 /*
644 * The following mb guarantees that previous clear of a PENDING bit
645 * will not be reordered with any speculative LOADS or STORES from
646 * work->current_func, which is executed afterwards. This possible
647 * reordering can lead to a missed execution on attempt to qeueue
648 * the same @work. E.g. consider this case:
649 *
650 * CPU#0 CPU#1
651 * ---------------------------- --------------------------------
652 *
653 * 1 STORE event_indicated
654 * 2 queue_work_on() {
655 * 3 test_and_set_bit(PENDING)
656 * 4 } set_..._and_clear_pending() {
657 * 5 set_work_data() # clear bit
658 * 6 smp_mb()
659 * 7 work->current_func() {
660 * 8 LOAD event_indicated
661 * }
662 *
663 * Without an explicit full barrier speculative LOAD on line 8 can
664 * be executed before CPU#0 does STORE on line 1. If that happens,
665 * CPU#0 observes the PENDING bit is still set and new execution of
666 * a @work is not queued in a hope, that CPU#1 will eventually
667 * finish the queued @work. Meanwhile CPU#1 does not see
668 * event_indicated is set, because speculative LOAD was executed
669 * before actual STORE.
670 */
671 smp_mb();
7a22ad75 672}
f756d5e2 673
7a22ad75 674static void clear_work_data(struct work_struct *work)
1da177e4 675{
7c3eed5c
TH
676 smp_wmb(); /* see set_work_pool_and_clear_pending() */
677 set_work_data(work, WORK_STRUCT_NO_POOL, 0);
1da177e4
LT
678}
679
112202d9 680static struct pool_workqueue *get_work_pwq(struct work_struct *work)
b1f4ec17 681{
e120153d 682 unsigned long data = atomic_long_read(&work->data);
7a22ad75 683
112202d9 684 if (data & WORK_STRUCT_PWQ)
e120153d
TH
685 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
686 else
687 return NULL;
4d707b9f
ON
688}
689
7c3eed5c
TH
690/**
691 * get_work_pool - return the worker_pool a given work was associated with
692 * @work: the work item of interest
693 *
68e13a67
LJ
694 * Pools are created and destroyed under wq_pool_mutex, and allows read
695 * access under sched-RCU read lock. As such, this function should be
696 * called under wq_pool_mutex or with preemption disabled.
fa1b54e6
TH
697 *
698 * All fields of the returned pool are accessible as long as the above
699 * mentioned locking is in effect. If the returned pool needs to be used
700 * beyond the critical section, the caller is responsible for ensuring the
701 * returned pool is and stays online.
d185af30
YB
702 *
703 * Return: The worker_pool @work was last associated with. %NULL if none.
7c3eed5c
TH
704 */
705static struct worker_pool *get_work_pool(struct work_struct *work)
365970a1 706{
e120153d 707 unsigned long data = atomic_long_read(&work->data);
7c3eed5c 708 int pool_id;
7a22ad75 709
68e13a67 710 assert_rcu_or_pool_mutex();
fa1b54e6 711
112202d9
TH
712 if (data & WORK_STRUCT_PWQ)
713 return ((struct pool_workqueue *)
7c3eed5c 714 (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
7a22ad75 715
7c3eed5c
TH
716 pool_id = data >> WORK_OFFQ_POOL_SHIFT;
717 if (pool_id == WORK_OFFQ_POOL_NONE)
7a22ad75
TH
718 return NULL;
719
fa1b54e6 720 return idr_find(&worker_pool_idr, pool_id);
7c3eed5c
TH
721}
722
723/**
724 * get_work_pool_id - return the worker pool ID a given work is associated with
725 * @work: the work item of interest
726 *
d185af30 727 * Return: The worker_pool ID @work was last associated with.
7c3eed5c
TH
728 * %WORK_OFFQ_POOL_NONE if none.
729 */
730static int get_work_pool_id(struct work_struct *work)
731{
54d5b7d0
LJ
732 unsigned long data = atomic_long_read(&work->data);
733
112202d9
TH
734 if (data & WORK_STRUCT_PWQ)
735 return ((struct pool_workqueue *)
54d5b7d0 736 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
7c3eed5c 737
54d5b7d0 738 return data >> WORK_OFFQ_POOL_SHIFT;
7c3eed5c
TH
739}
740
bbb68dfa
TH
741static void mark_work_canceling(struct work_struct *work)
742{
7c3eed5c 743 unsigned long pool_id = get_work_pool_id(work);
bbb68dfa 744
7c3eed5c
TH
745 pool_id <<= WORK_OFFQ_POOL_SHIFT;
746 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
bbb68dfa
TH
747}
748
749static bool work_is_canceling(struct work_struct *work)
750{
751 unsigned long data = atomic_long_read(&work->data);
752
112202d9 753 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
bbb68dfa
TH
754}
755
e22bee78 756/*
3270476a
TH
757 * Policy functions. These define the policies on how the global worker
758 * pools are managed. Unless noted otherwise, these functions assume that
d565ed63 759 * they're being called with pool->lock held.
e22bee78
TH
760 */
761
63d95a91 762static bool __need_more_worker(struct worker_pool *pool)
a848e3b6 763{
e19e397a 764 return !atomic_read(&pool->nr_running);
a848e3b6
ON
765}
766
4594bf15 767/*
e22bee78
TH
768 * Need to wake up a worker? Called from anything but currently
769 * running workers.
974271c4
TH
770 *
771 * Note that, because unbound workers never contribute to nr_running, this
706026c2 772 * function will always return %true for unbound pools as long as the
974271c4 773 * worklist isn't empty.
4594bf15 774 */
63d95a91 775static bool need_more_worker(struct worker_pool *pool)
365970a1 776{
63d95a91 777 return !list_empty(&pool->worklist) && __need_more_worker(pool);
e22bee78 778}
4594bf15 779
e22bee78 780/* Can I start working? Called from busy but !running workers. */
63d95a91 781static bool may_start_working(struct worker_pool *pool)
e22bee78 782{
63d95a91 783 return pool->nr_idle;
e22bee78
TH
784}
785
786/* Do I need to keep working? Called from currently running workers. */
63d95a91 787static bool keep_working(struct worker_pool *pool)
e22bee78 788{
e19e397a
TH
789 return !list_empty(&pool->worklist) &&
790 atomic_read(&pool->nr_running) <= 1;
e22bee78
TH
791}
792
793/* Do we need a new worker? Called from manager. */
63d95a91 794static bool need_to_create_worker(struct worker_pool *pool)
e22bee78 795{
63d95a91 796 return need_more_worker(pool) && !may_start_working(pool);
e22bee78 797}
365970a1 798
e22bee78 799/* Do we have too many workers and should some go away? */
63d95a91 800static bool too_many_workers(struct worker_pool *pool)
e22bee78 801{
34a06bd6 802 bool managing = mutex_is_locked(&pool->manager_arb);
63d95a91
TH
803 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
804 int nr_busy = pool->nr_workers - nr_idle;
e22bee78
TH
805
806 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
365970a1
DH
807}
808
4d707b9f 809/*
e22bee78
TH
810 * Wake up functions.
811 */
812
1037de36
LJ
813/* Return the first idle worker. Safe with preemption disabled */
814static struct worker *first_idle_worker(struct worker_pool *pool)
7e11629d 815{
63d95a91 816 if (unlikely(list_empty(&pool->idle_list)))
7e11629d
TH
817 return NULL;
818
63d95a91 819 return list_first_entry(&pool->idle_list, struct worker, entry);
7e11629d
TH
820}
821
822/**
823 * wake_up_worker - wake up an idle worker
63d95a91 824 * @pool: worker pool to wake worker from
7e11629d 825 *
63d95a91 826 * Wake up the first idle worker of @pool.
7e11629d
TH
827 *
828 * CONTEXT:
d565ed63 829 * spin_lock_irq(pool->lock).
7e11629d 830 */
63d95a91 831static void wake_up_worker(struct worker_pool *pool)
7e11629d 832{
1037de36 833 struct worker *worker = first_idle_worker(pool);
7e11629d
TH
834
835 if (likely(worker))
836 wake_up_process(worker->task);
837}
838
d302f017 839/**
e22bee78
TH
840 * wq_worker_waking_up - a worker is waking up
841 * @task: task waking up
842 * @cpu: CPU @task is waking up to
843 *
844 * This function is called during try_to_wake_up() when a worker is
845 * being awoken.
846 *
847 * CONTEXT:
848 * spin_lock_irq(rq->lock)
849 */
d84ff051 850void wq_worker_waking_up(struct task_struct *task, int cpu)
e22bee78
TH
851{
852 struct worker *worker = kthread_data(task);
853
36576000 854 if (!(worker->flags & WORKER_NOT_RUNNING)) {
ec22ca5e 855 WARN_ON_ONCE(worker->pool->cpu != cpu);
e19e397a 856 atomic_inc(&worker->pool->nr_running);
36576000 857 }
e22bee78
TH
858}
859
860/**
861 * wq_worker_sleeping - a worker is going to sleep
862 * @task: task going to sleep
e22bee78
TH
863 *
864 * This function is called during schedule() when a busy worker is
865 * going to sleep. Worker on the same cpu can be woken up by
866 * returning pointer to its task.
867 *
868 * CONTEXT:
869 * spin_lock_irq(rq->lock)
870 *
d185af30 871 * Return:
e22bee78
TH
872 * Worker task on @cpu to wake up, %NULL if none.
873 */
9b7f6597 874struct task_struct *wq_worker_sleeping(struct task_struct *task)
e22bee78
TH
875{
876 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
111c225a 877 struct worker_pool *pool;
e22bee78 878
111c225a
TH
879 /*
880 * Rescuers, which may not have all the fields set up like normal
881 * workers, also reach here, let's not access anything before
882 * checking NOT_RUNNING.
883 */
2d64672e 884 if (worker->flags & WORKER_NOT_RUNNING)
e22bee78
TH
885 return NULL;
886
111c225a 887 pool = worker->pool;
111c225a 888
e22bee78 889 /* this can only happen on the local cpu */
9b7f6597 890 if (WARN_ON_ONCE(pool->cpu != raw_smp_processor_id()))
6183c009 891 return NULL;
e22bee78
TH
892
893 /*
894 * The counterpart of the following dec_and_test, implied mb,
895 * worklist not empty test sequence is in insert_work().
896 * Please read comment there.
897 *
628c78e7
TH
898 * NOT_RUNNING is clear. This means that we're bound to and
899 * running on the local cpu w/ rq lock held and preemption
900 * disabled, which in turn means that none else could be
d565ed63 901 * manipulating idle_list, so dereferencing idle_list without pool
628c78e7 902 * lock is safe.
e22bee78 903 */
e19e397a
TH
904 if (atomic_dec_and_test(&pool->nr_running) &&
905 !list_empty(&pool->worklist))
1037de36 906 to_wakeup = first_idle_worker(pool);
e22bee78
TH
907 return to_wakeup ? to_wakeup->task : NULL;
908}
909
910/**
911 * worker_set_flags - set worker flags and adjust nr_running accordingly
cb444766 912 * @worker: self
d302f017 913 * @flags: flags to set
d302f017 914 *
228f1d00 915 * Set @flags in @worker->flags and adjust nr_running accordingly.
d302f017 916 *
cb444766 917 * CONTEXT:
d565ed63 918 * spin_lock_irq(pool->lock)
d302f017 919 */
228f1d00 920static inline void worker_set_flags(struct worker *worker, unsigned int flags)
d302f017 921{
bd7bdd43 922 struct worker_pool *pool = worker->pool;
e22bee78 923
cb444766
TH
924 WARN_ON_ONCE(worker->task != current);
925
228f1d00 926 /* If transitioning into NOT_RUNNING, adjust nr_running. */
e22bee78
TH
927 if ((flags & WORKER_NOT_RUNNING) &&
928 !(worker->flags & WORKER_NOT_RUNNING)) {
228f1d00 929 atomic_dec(&pool->nr_running);
e22bee78
TH
930 }
931
d302f017
TH
932 worker->flags |= flags;
933}
934
935/**
e22bee78 936 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
cb444766 937 * @worker: self
d302f017
TH
938 * @flags: flags to clear
939 *
e22bee78 940 * Clear @flags in @worker->flags and adjust nr_running accordingly.
d302f017 941 *
cb444766 942 * CONTEXT:
d565ed63 943 * spin_lock_irq(pool->lock)
d302f017
TH
944 */
945static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
946{
63d95a91 947 struct worker_pool *pool = worker->pool;
e22bee78
TH
948 unsigned int oflags = worker->flags;
949
cb444766
TH
950 WARN_ON_ONCE(worker->task != current);
951
d302f017 952 worker->flags &= ~flags;
e22bee78 953
42c025f3
TH
954 /*
955 * If transitioning out of NOT_RUNNING, increment nr_running. Note
956 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
957 * of multiple flags, not a single flag.
958 */
e22bee78
TH
959 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
960 if (!(worker->flags & WORKER_NOT_RUNNING))
e19e397a 961 atomic_inc(&pool->nr_running);
d302f017
TH
962}
963
8cca0eea
TH
964/**
965 * find_worker_executing_work - find worker which is executing a work
c9e7cf27 966 * @pool: pool of interest
8cca0eea
TH
967 * @work: work to find worker for
968 *
c9e7cf27
TH
969 * Find a worker which is executing @work on @pool by searching
970 * @pool->busy_hash which is keyed by the address of @work. For a worker
a2c1c57b
TH
971 * to match, its current execution should match the address of @work and
972 * its work function. This is to avoid unwanted dependency between
973 * unrelated work executions through a work item being recycled while still
974 * being executed.
975 *
976 * This is a bit tricky. A work item may be freed once its execution
977 * starts and nothing prevents the freed area from being recycled for
978 * another work item. If the same work item address ends up being reused
979 * before the original execution finishes, workqueue will identify the
980 * recycled work item as currently executing and make it wait until the
981 * current execution finishes, introducing an unwanted dependency.
982 *
c5aa87bb
TH
983 * This function checks the work item address and work function to avoid
984 * false positives. Note that this isn't complete as one may construct a
985 * work function which can introduce dependency onto itself through a
986 * recycled work item. Well, if somebody wants to shoot oneself in the
987 * foot that badly, there's only so much we can do, and if such deadlock
988 * actually occurs, it should be easy to locate the culprit work function.
8cca0eea
TH
989 *
990 * CONTEXT:
d565ed63 991 * spin_lock_irq(pool->lock).
8cca0eea 992 *
d185af30
YB
993 * Return:
994 * Pointer to worker which is executing @work if found, %NULL
8cca0eea 995 * otherwise.
4d707b9f 996 */
c9e7cf27 997static struct worker *find_worker_executing_work(struct worker_pool *pool,
8cca0eea 998 struct work_struct *work)
4d707b9f 999{
42f8570f 1000 struct worker *worker;
42f8570f 1001
b67bfe0d 1002 hash_for_each_possible(pool->busy_hash, worker, hentry,
a2c1c57b
TH
1003 (unsigned long)work)
1004 if (worker->current_work == work &&
1005 worker->current_func == work->func)
42f8570f
SL
1006 return worker;
1007
1008 return NULL;
4d707b9f
ON
1009}
1010
bf4ede01
TH
1011/**
1012 * move_linked_works - move linked works to a list
1013 * @work: start of series of works to be scheduled
1014 * @head: target list to append @work to
402dd89d 1015 * @nextp: out parameter for nested worklist walking
bf4ede01
TH
1016 *
1017 * Schedule linked works starting from @work to @head. Work series to
1018 * be scheduled starts at @work and includes any consecutive work with
1019 * WORK_STRUCT_LINKED set in its predecessor.
1020 *
1021 * If @nextp is not NULL, it's updated to point to the next work of
1022 * the last scheduled work. This allows move_linked_works() to be
1023 * nested inside outer list_for_each_entry_safe().
1024 *
1025 * CONTEXT:
d565ed63 1026 * spin_lock_irq(pool->lock).
bf4ede01
TH
1027 */
1028static void move_linked_works(struct work_struct *work, struct list_head *head,
1029 struct work_struct **nextp)
1030{
1031 struct work_struct *n;
1032
1033 /*
1034 * Linked worklist will always end before the end of the list,
1035 * use NULL for list head.
1036 */
1037 list_for_each_entry_safe_from(work, n, NULL, entry) {
1038 list_move_tail(&work->entry, head);
1039 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1040 break;
1041 }
1042
1043 /*
1044 * If we're already inside safe list traversal and have moved
1045 * multiple works to the scheduled queue, the next position
1046 * needs to be updated.
1047 */
1048 if (nextp)
1049 *nextp = n;
1050}
1051
8864b4e5
TH
1052/**
1053 * get_pwq - get an extra reference on the specified pool_workqueue
1054 * @pwq: pool_workqueue to get
1055 *
1056 * Obtain an extra reference on @pwq. The caller should guarantee that
1057 * @pwq has positive refcnt and be holding the matching pool->lock.
1058 */
1059static void get_pwq(struct pool_workqueue *pwq)
1060{
1061 lockdep_assert_held(&pwq->pool->lock);
1062 WARN_ON_ONCE(pwq->refcnt <= 0);
1063 pwq->refcnt++;
1064}
1065
1066/**
1067 * put_pwq - put a pool_workqueue reference
1068 * @pwq: pool_workqueue to put
1069 *
1070 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1071 * destruction. The caller should be holding the matching pool->lock.
1072 */
1073static void put_pwq(struct pool_workqueue *pwq)
1074{
1075 lockdep_assert_held(&pwq->pool->lock);
1076 if (likely(--pwq->refcnt))
1077 return;
1078 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1079 return;
1080 /*
1081 * @pwq can't be released under pool->lock, bounce to
1082 * pwq_unbound_release_workfn(). This never recurses on the same
1083 * pool->lock as this path is taken only for unbound workqueues and
1084 * the release work item is scheduled on a per-cpu workqueue. To
1085 * avoid lockdep warning, unbound pool->locks are given lockdep
1086 * subclass of 1 in get_unbound_pool().
1087 */
1088 schedule_work(&pwq->unbound_release_work);
1089}
1090
dce90d47
TH
1091/**
1092 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1093 * @pwq: pool_workqueue to put (can be %NULL)
1094 *
1095 * put_pwq() with locking. This function also allows %NULL @pwq.
1096 */
1097static void put_pwq_unlocked(struct pool_workqueue *pwq)
1098{
1099 if (pwq) {
1100 /*
1101 * As both pwqs and pools are sched-RCU protected, the
1102 * following lock operations are safe.
1103 */
1104 spin_lock_irq(&pwq->pool->lock);
1105 put_pwq(pwq);
1106 spin_unlock_irq(&pwq->pool->lock);
1107 }
1108}
1109
112202d9 1110static void pwq_activate_delayed_work(struct work_struct *work)
bf4ede01 1111{
112202d9 1112 struct pool_workqueue *pwq = get_work_pwq(work);
bf4ede01
TH
1113
1114 trace_workqueue_activate_work(work);
82607adc
TH
1115 if (list_empty(&pwq->pool->worklist))
1116 pwq->pool->watchdog_ts = jiffies;
112202d9 1117 move_linked_works(work, &pwq->pool->worklist, NULL);
bf4ede01 1118 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
112202d9 1119 pwq->nr_active++;
bf4ede01
TH
1120}
1121
112202d9 1122static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
3aa62497 1123{
112202d9 1124 struct work_struct *work = list_first_entry(&pwq->delayed_works,
3aa62497
LJ
1125 struct work_struct, entry);
1126
112202d9 1127 pwq_activate_delayed_work(work);
3aa62497
LJ
1128}
1129
bf4ede01 1130/**
112202d9
TH
1131 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1132 * @pwq: pwq of interest
bf4ede01 1133 * @color: color of work which left the queue
bf4ede01
TH
1134 *
1135 * A work either has completed or is removed from pending queue,
112202d9 1136 * decrement nr_in_flight of its pwq and handle workqueue flushing.
bf4ede01
TH
1137 *
1138 * CONTEXT:
d565ed63 1139 * spin_lock_irq(pool->lock).
bf4ede01 1140 */
112202d9 1141static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
bf4ede01 1142{
8864b4e5 1143 /* uncolored work items don't participate in flushing or nr_active */
bf4ede01 1144 if (color == WORK_NO_COLOR)
8864b4e5 1145 goto out_put;
bf4ede01 1146
112202d9 1147 pwq->nr_in_flight[color]--;
bf4ede01 1148
112202d9
TH
1149 pwq->nr_active--;
1150 if (!list_empty(&pwq->delayed_works)) {
b3f9f405 1151 /* one down, submit a delayed one */
112202d9
TH
1152 if (pwq->nr_active < pwq->max_active)
1153 pwq_activate_first_delayed(pwq);
bf4ede01
TH
1154 }
1155
1156 /* is flush in progress and are we at the flushing tip? */
112202d9 1157 if (likely(pwq->flush_color != color))
8864b4e5 1158 goto out_put;
bf4ede01
TH
1159
1160 /* are there still in-flight works? */
112202d9 1161 if (pwq->nr_in_flight[color])
8864b4e5 1162 goto out_put;
bf4ede01 1163
112202d9
TH
1164 /* this pwq is done, clear flush_color */
1165 pwq->flush_color = -1;
bf4ede01
TH
1166
1167 /*
112202d9 1168 * If this was the last pwq, wake up the first flusher. It
bf4ede01
TH
1169 * will handle the rest.
1170 */
112202d9
TH
1171 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1172 complete(&pwq->wq->first_flusher->done);
8864b4e5
TH
1173out_put:
1174 put_pwq(pwq);
bf4ede01
TH
1175}
1176
36e227d2 1177/**
bbb68dfa 1178 * try_to_grab_pending - steal work item from worklist and disable irq
36e227d2
TH
1179 * @work: work item to steal
1180 * @is_dwork: @work is a delayed_work
bbb68dfa 1181 * @flags: place to store irq state
36e227d2
TH
1182 *
1183 * Try to grab PENDING bit of @work. This function can handle @work in any
d185af30 1184 * stable state - idle, on timer or on worklist.
36e227d2 1185 *
d185af30 1186 * Return:
36e227d2
TH
1187 * 1 if @work was pending and we successfully stole PENDING
1188 * 0 if @work was idle and we claimed PENDING
1189 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
bbb68dfa
TH
1190 * -ENOENT if someone else is canceling @work, this state may persist
1191 * for arbitrarily long
36e227d2 1192 *
d185af30 1193 * Note:
bbb68dfa 1194 * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
e0aecdd8
TH
1195 * interrupted while holding PENDING and @work off queue, irq must be
1196 * disabled on entry. This, combined with delayed_work->timer being
1197 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
bbb68dfa
TH
1198 *
1199 * On successful return, >= 0, irq is disabled and the caller is
1200 * responsible for releasing it using local_irq_restore(*@flags).
1201 *
e0aecdd8 1202 * This function is safe to call from any context including IRQ handler.
bf4ede01 1203 */
bbb68dfa
TH
1204static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1205 unsigned long *flags)
bf4ede01 1206{
d565ed63 1207 struct worker_pool *pool;
112202d9 1208 struct pool_workqueue *pwq;
bf4ede01 1209
bbb68dfa
TH
1210 local_irq_save(*flags);
1211
36e227d2
TH
1212 /* try to steal the timer if it exists */
1213 if (is_dwork) {
1214 struct delayed_work *dwork = to_delayed_work(work);
1215
e0aecdd8
TH
1216 /*
1217 * dwork->timer is irqsafe. If del_timer() fails, it's
1218 * guaranteed that the timer is not queued anywhere and not
1219 * running on the local CPU.
1220 */
36e227d2
TH
1221 if (likely(del_timer(&dwork->timer)))
1222 return 1;
1223 }
1224
1225 /* try to claim PENDING the normal way */
bf4ede01
TH
1226 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1227 return 0;
1228
1229 /*
1230 * The queueing is in progress, or it is already queued. Try to
1231 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1232 */
d565ed63
TH
1233 pool = get_work_pool(work);
1234 if (!pool)
bbb68dfa 1235 goto fail;
bf4ede01 1236
d565ed63 1237 spin_lock(&pool->lock);
0b3dae68 1238 /*
112202d9
TH
1239 * work->data is guaranteed to point to pwq only while the work
1240 * item is queued on pwq->wq, and both updating work->data to point
1241 * to pwq on queueing and to pool on dequeueing are done under
1242 * pwq->pool->lock. This in turn guarantees that, if work->data
1243 * points to pwq which is associated with a locked pool, the work
0b3dae68
LJ
1244 * item is currently queued on that pool.
1245 */
112202d9
TH
1246 pwq = get_work_pwq(work);
1247 if (pwq && pwq->pool == pool) {
16062836
TH
1248 debug_work_deactivate(work);
1249
1250 /*
1251 * A delayed work item cannot be grabbed directly because
1252 * it might have linked NO_COLOR work items which, if left
112202d9 1253 * on the delayed_list, will confuse pwq->nr_active
16062836
TH
1254 * management later on and cause stall. Make sure the work
1255 * item is activated before grabbing.
1256 */
1257 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
112202d9 1258 pwq_activate_delayed_work(work);
16062836
TH
1259
1260 list_del_init(&work->entry);
9c34a704 1261 pwq_dec_nr_in_flight(pwq, get_work_color(work));
16062836 1262
112202d9 1263 /* work->data points to pwq iff queued, point to pool */
16062836
TH
1264 set_work_pool_and_keep_pending(work, pool->id);
1265
1266 spin_unlock(&pool->lock);
1267 return 1;
bf4ede01 1268 }
d565ed63 1269 spin_unlock(&pool->lock);
bbb68dfa
TH
1270fail:
1271 local_irq_restore(*flags);
1272 if (work_is_canceling(work))
1273 return -ENOENT;
1274 cpu_relax();
36e227d2 1275 return -EAGAIN;
bf4ede01
TH
1276}
1277
4690c4ab 1278/**
706026c2 1279 * insert_work - insert a work into a pool
112202d9 1280 * @pwq: pwq @work belongs to
4690c4ab
TH
1281 * @work: work to insert
1282 * @head: insertion point
1283 * @extra_flags: extra WORK_STRUCT_* flags to set
1284 *
112202d9 1285 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
706026c2 1286 * work_struct flags.
4690c4ab
TH
1287 *
1288 * CONTEXT:
d565ed63 1289 * spin_lock_irq(pool->lock).
4690c4ab 1290 */
112202d9
TH
1291static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1292 struct list_head *head, unsigned int extra_flags)
b89deed3 1293{
112202d9 1294 struct worker_pool *pool = pwq->pool;
e22bee78 1295
4690c4ab 1296 /* we own @work, set data and link */
112202d9 1297 set_work_pwq(work, pwq, extra_flags);
1a4d9b0a 1298 list_add_tail(&work->entry, head);
8864b4e5 1299 get_pwq(pwq);
e22bee78
TH
1300
1301 /*
c5aa87bb
TH
1302 * Ensure either wq_worker_sleeping() sees the above
1303 * list_add_tail() or we see zero nr_running to avoid workers lying
1304 * around lazily while there are works to be processed.
e22bee78
TH
1305 */
1306 smp_mb();
1307
63d95a91
TH
1308 if (__need_more_worker(pool))
1309 wake_up_worker(pool);
b89deed3
ON
1310}
1311
c8efcc25
TH
1312/*
1313 * Test whether @work is being queued from another work executing on the
8d03ecfe 1314 * same workqueue.
c8efcc25
TH
1315 */
1316static bool is_chained_work(struct workqueue_struct *wq)
1317{
8d03ecfe
TH
1318 struct worker *worker;
1319
1320 worker = current_wq_worker();
1321 /*
1322 * Return %true iff I'm a worker execuing a work item on @wq. If
1323 * I'm @worker, it's safe to dereference it without locking.
1324 */
112202d9 1325 return worker && worker->current_pwq->wq == wq;
c8efcc25
TH
1326}
1327
ef557180
MG
1328/*
1329 * When queueing an unbound work item to a wq, prefer local CPU if allowed
1330 * by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
1331 * avoid perturbing sensitive tasks.
1332 */
1333static int wq_select_unbound_cpu(int cpu)
1334{
f303fccb 1335 static bool printed_dbg_warning;
ef557180
MG
1336 int new_cpu;
1337
f303fccb
TH
1338 if (likely(!wq_debug_force_rr_cpu)) {
1339 if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
1340 return cpu;
1341 } else if (!printed_dbg_warning) {
1342 pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n");
1343 printed_dbg_warning = true;
1344 }
1345
ef557180
MG
1346 if (cpumask_empty(wq_unbound_cpumask))
1347 return cpu;
1348
1349 new_cpu = __this_cpu_read(wq_rr_cpu_last);
1350 new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
1351 if (unlikely(new_cpu >= nr_cpu_ids)) {
1352 new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
1353 if (unlikely(new_cpu >= nr_cpu_ids))
1354 return cpu;
1355 }
1356 __this_cpu_write(wq_rr_cpu_last, new_cpu);
1357
1358 return new_cpu;
1359}
1360
d84ff051 1361static void __queue_work(int cpu, struct workqueue_struct *wq,
1da177e4
LT
1362 struct work_struct *work)
1363{
112202d9 1364 struct pool_workqueue *pwq;
c9178087 1365 struct worker_pool *last_pool;
1e19ffc6 1366 struct list_head *worklist;
8a2e8e5d 1367 unsigned int work_flags;
b75cac93 1368 unsigned int req_cpu = cpu;
8930caba
TH
1369
1370 /*
1371 * While a work item is PENDING && off queue, a task trying to
1372 * steal the PENDING will busy-loop waiting for it to either get
1373 * queued or lose PENDING. Grabbing PENDING and queueing should
1374 * happen with IRQ disabled.
1375 */
1376 WARN_ON_ONCE(!irqs_disabled());
1da177e4 1377
dc186ad7 1378 debug_work_activate(work);
1e19ffc6 1379
9ef28a73 1380 /* if draining, only works from the same workqueue are allowed */
618b01eb 1381 if (unlikely(wq->flags & __WQ_DRAINING) &&
c8efcc25 1382 WARN_ON_ONCE(!is_chained_work(wq)))
e41e704b 1383 return;
9e8cd2f5 1384retry:
df2d5ae4 1385 if (req_cpu == WORK_CPU_UNBOUND)
ef557180 1386 cpu = wq_select_unbound_cpu(raw_smp_processor_id());
df2d5ae4 1387
c9178087 1388 /* pwq which will be used unless @work is executing elsewhere */
df2d5ae4 1389 if (!(wq->flags & WQ_UNBOUND))
7fb98ea7 1390 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
df2d5ae4
TH
1391 else
1392 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
dbf2576e 1393
c9178087
TH
1394 /*
1395 * If @work was previously on a different pool, it might still be
1396 * running there, in which case the work needs to be queued on that
1397 * pool to guarantee non-reentrancy.
1398 */
1399 last_pool = get_work_pool(work);
1400 if (last_pool && last_pool != pwq->pool) {
1401 struct worker *worker;
18aa9eff 1402
c9178087 1403 spin_lock(&last_pool->lock);
18aa9eff 1404
c9178087 1405 worker = find_worker_executing_work(last_pool, work);
18aa9eff 1406
c9178087
TH
1407 if (worker && worker->current_pwq->wq == wq) {
1408 pwq = worker->current_pwq;
8930caba 1409 } else {
c9178087
TH
1410 /* meh... not running there, queue here */
1411 spin_unlock(&last_pool->lock);
112202d9 1412 spin_lock(&pwq->pool->lock);
8930caba 1413 }
f3421797 1414 } else {
112202d9 1415 spin_lock(&pwq->pool->lock);
502ca9d8
TH
1416 }
1417
9e8cd2f5
TH
1418 /*
1419 * pwq is determined and locked. For unbound pools, we could have
1420 * raced with pwq release and it could already be dead. If its
1421 * refcnt is zero, repeat pwq selection. Note that pwqs never die
df2d5ae4
TH
1422 * without another pwq replacing it in the numa_pwq_tbl or while
1423 * work items are executing on it, so the retrying is guaranteed to
9e8cd2f5
TH
1424 * make forward-progress.
1425 */
1426 if (unlikely(!pwq->refcnt)) {
1427 if (wq->flags & WQ_UNBOUND) {
1428 spin_unlock(&pwq->pool->lock);
1429 cpu_relax();
1430 goto retry;
1431 }
1432 /* oops */
1433 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1434 wq->name, cpu);
1435 }
1436
112202d9
TH
1437 /* pwq determined, queue */
1438 trace_workqueue_queue_work(req_cpu, pwq, work);
502ca9d8 1439
f5b2552b 1440 if (WARN_ON(!list_empty(&work->entry))) {
112202d9 1441 spin_unlock(&pwq->pool->lock);
f5b2552b
DC
1442 return;
1443 }
1e19ffc6 1444
112202d9
TH
1445 pwq->nr_in_flight[pwq->work_color]++;
1446 work_flags = work_color_to_flags(pwq->work_color);
1e19ffc6 1447
112202d9 1448 if (likely(pwq->nr_active < pwq->max_active)) {
cdadf009 1449 trace_workqueue_activate_work(work);
112202d9
TH
1450 pwq->nr_active++;
1451 worklist = &pwq->pool->worklist;
82607adc
TH
1452 if (list_empty(worklist))
1453 pwq->pool->watchdog_ts = jiffies;
8a2e8e5d
TH
1454 } else {
1455 work_flags |= WORK_STRUCT_DELAYED;
112202d9 1456 worklist = &pwq->delayed_works;
8a2e8e5d 1457 }
1e19ffc6 1458
112202d9 1459 insert_work(pwq, work, worklist, work_flags);
1e19ffc6 1460
112202d9 1461 spin_unlock(&pwq->pool->lock);
1da177e4
LT
1462}
1463
0fcb78c2 1464/**
c1a220e7
ZR
1465 * queue_work_on - queue work on specific cpu
1466 * @cpu: CPU number to execute work on
0fcb78c2
REB
1467 * @wq: workqueue to use
1468 * @work: work to queue
1469 *
c1a220e7
ZR
1470 * We queue the work to a specific CPU, the caller must ensure it
1471 * can't go away.
d185af30
YB
1472 *
1473 * Return: %false if @work was already on a queue, %true otherwise.
1da177e4 1474 */
d4283e93
TH
1475bool queue_work_on(int cpu, struct workqueue_struct *wq,
1476 struct work_struct *work)
1da177e4 1477{
d4283e93 1478 bool ret = false;
8930caba 1479 unsigned long flags;
ef1ca236 1480
8930caba 1481 local_irq_save(flags);
c1a220e7 1482
22df02bb 1483 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
4690c4ab 1484 __queue_work(cpu, wq, work);
d4283e93 1485 ret = true;
c1a220e7 1486 }
ef1ca236 1487
8930caba 1488 local_irq_restore(flags);
1da177e4
LT
1489 return ret;
1490}
ad7b1f84 1491EXPORT_SYMBOL(queue_work_on);
1da177e4 1492
d8e794df 1493void delayed_work_timer_fn(unsigned long __data)
1da177e4 1494{
52bad64d 1495 struct delayed_work *dwork = (struct delayed_work *)__data;
1da177e4 1496
e0aecdd8 1497 /* should have been called from irqsafe timer with irq already off */
60c057bc 1498 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1da177e4 1499}
1438ade5 1500EXPORT_SYMBOL(delayed_work_timer_fn);
1da177e4 1501
7beb2edf
TH
1502static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1503 struct delayed_work *dwork, unsigned long delay)
1da177e4 1504{
7beb2edf
TH
1505 struct timer_list *timer = &dwork->timer;
1506 struct work_struct *work = &dwork->work;
7beb2edf
TH
1507
1508 WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1509 timer->data != (unsigned long)dwork);
fc4b514f
TH
1510 WARN_ON_ONCE(timer_pending(timer));
1511 WARN_ON_ONCE(!list_empty(&work->entry));
7beb2edf 1512
8852aac2
TH
1513 /*
1514 * If @delay is 0, queue @dwork->work immediately. This is for
1515 * both optimization and correctness. The earliest @timer can
1516 * expire is on the closest next tick and delayed_work users depend
1517 * on that there's no such delay when @delay is 0.
1518 */
1519 if (!delay) {
1520 __queue_work(cpu, wq, &dwork->work);
1521 return;
1522 }
1523
7beb2edf 1524 timer_stats_timer_set_start_info(&dwork->timer);
1da177e4 1525
60c057bc 1526 dwork->wq = wq;
1265057f 1527 dwork->cpu = cpu;
7beb2edf
TH
1528 timer->expires = jiffies + delay;
1529
041bd12e
TH
1530 if (unlikely(cpu != WORK_CPU_UNBOUND))
1531 add_timer_on(timer, cpu);
1532 else
1533 add_timer(timer);
1da177e4
LT
1534}
1535
0fcb78c2
REB
1536/**
1537 * queue_delayed_work_on - queue work on specific CPU after delay
1538 * @cpu: CPU number to execute work on
1539 * @wq: workqueue to use
af9997e4 1540 * @dwork: work to queue
0fcb78c2
REB
1541 * @delay: number of jiffies to wait before queueing
1542 *
d185af30 1543 * Return: %false if @work was already on a queue, %true otherwise. If
715f1300
TH
1544 * @delay is zero and @dwork is idle, it will be scheduled for immediate
1545 * execution.
0fcb78c2 1546 */
d4283e93
TH
1547bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1548 struct delayed_work *dwork, unsigned long delay)
7a6bc1cd 1549{
52bad64d 1550 struct work_struct *work = &dwork->work;
d4283e93 1551 bool ret = false;
8930caba 1552 unsigned long flags;
7a6bc1cd 1553
8930caba
TH
1554 /* read the comment in __queue_work() */
1555 local_irq_save(flags);
7a6bc1cd 1556
22df02bb 1557 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
7beb2edf 1558 __queue_delayed_work(cpu, wq, dwork, delay);
d4283e93 1559 ret = true;
7a6bc1cd 1560 }
8a3e77cc 1561
8930caba 1562 local_irq_restore(flags);
7a6bc1cd
VP
1563 return ret;
1564}
ad7b1f84 1565EXPORT_SYMBOL(queue_delayed_work_on);
c7fc77f7 1566
8376fe22
TH
1567/**
1568 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1569 * @cpu: CPU number to execute work on
1570 * @wq: workqueue to use
1571 * @dwork: work to queue
1572 * @delay: number of jiffies to wait before queueing
1573 *
1574 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1575 * modify @dwork's timer so that it expires after @delay. If @delay is
1576 * zero, @work is guaranteed to be scheduled immediately regardless of its
1577 * current state.
1578 *
d185af30 1579 * Return: %false if @dwork was idle and queued, %true if @dwork was
8376fe22
TH
1580 * pending and its timer was modified.
1581 *
e0aecdd8 1582 * This function is safe to call from any context including IRQ handler.
8376fe22
TH
1583 * See try_to_grab_pending() for details.
1584 */
1585bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1586 struct delayed_work *dwork, unsigned long delay)
1587{
1588 unsigned long flags;
1589 int ret;
c7fc77f7 1590
8376fe22
TH
1591 do {
1592 ret = try_to_grab_pending(&dwork->work, true, &flags);
1593 } while (unlikely(ret == -EAGAIN));
63bc0362 1594
8376fe22
TH
1595 if (likely(ret >= 0)) {
1596 __queue_delayed_work(cpu, wq, dwork, delay);
1597 local_irq_restore(flags);
7a6bc1cd 1598 }
8376fe22
TH
1599
1600 /* -ENOENT from try_to_grab_pending() becomes %true */
7a6bc1cd
VP
1601 return ret;
1602}
8376fe22
TH
1603EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1604
c8e55f36
TH
1605/**
1606 * worker_enter_idle - enter idle state
1607 * @worker: worker which is entering idle state
1608 *
1609 * @worker is entering idle state. Update stats and idle timer if
1610 * necessary.
1611 *
1612 * LOCKING:
d565ed63 1613 * spin_lock_irq(pool->lock).
c8e55f36
TH
1614 */
1615static void worker_enter_idle(struct worker *worker)
1da177e4 1616{
bd7bdd43 1617 struct worker_pool *pool = worker->pool;
c8e55f36 1618
6183c009
TH
1619 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1620 WARN_ON_ONCE(!list_empty(&worker->entry) &&
1621 (worker->hentry.next || worker->hentry.pprev)))
1622 return;
c8e55f36 1623
051e1850 1624 /* can't use worker_set_flags(), also called from create_worker() */
cb444766 1625 worker->flags |= WORKER_IDLE;
bd7bdd43 1626 pool->nr_idle++;
e22bee78 1627 worker->last_active = jiffies;
c8e55f36
TH
1628
1629 /* idle_list is LIFO */
bd7bdd43 1630 list_add(&worker->entry, &pool->idle_list);
db7bccf4 1631
628c78e7
TH
1632 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1633 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
cb444766 1634
544ecf31 1635 /*
706026c2 1636 * Sanity check nr_running. Because wq_unbind_fn() releases
d565ed63 1637 * pool->lock between setting %WORKER_UNBOUND and zapping
628c78e7
TH
1638 * nr_running, the warning may trigger spuriously. Check iff
1639 * unbind is not in progress.
544ecf31 1640 */
24647570 1641 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
bd7bdd43 1642 pool->nr_workers == pool->nr_idle &&
e19e397a 1643 atomic_read(&pool->nr_running));
c8e55f36
TH
1644}
1645
1646/**
1647 * worker_leave_idle - leave idle state
1648 * @worker: worker which is leaving idle state
1649 *
1650 * @worker is leaving idle state. Update stats.
1651 *
1652 * LOCKING:
d565ed63 1653 * spin_lock_irq(pool->lock).
c8e55f36
TH
1654 */
1655static void worker_leave_idle(struct worker *worker)
1656{
bd7bdd43 1657 struct worker_pool *pool = worker->pool;
c8e55f36 1658
6183c009
TH
1659 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1660 return;
d302f017 1661 worker_clr_flags(worker, WORKER_IDLE);
bd7bdd43 1662 pool->nr_idle--;
c8e55f36
TH
1663 list_del_init(&worker->entry);
1664}
1665
f7537df5 1666static struct worker *alloc_worker(int node)
c34056a3
TH
1667{
1668 struct worker *worker;
1669
f7537df5 1670 worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
c8e55f36
TH
1671 if (worker) {
1672 INIT_LIST_HEAD(&worker->entry);
affee4b2 1673 INIT_LIST_HEAD(&worker->scheduled);
da028469 1674 INIT_LIST_HEAD(&worker->node);
e22bee78
TH
1675 /* on creation a worker is in !idle && prep state */
1676 worker->flags = WORKER_PREP;
c8e55f36 1677 }
c34056a3
TH
1678 return worker;
1679}
1680
4736cbf7
LJ
1681/**
1682 * worker_attach_to_pool() - attach a worker to a pool
1683 * @worker: worker to be attached
1684 * @pool: the target pool
1685 *
1686 * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1687 * cpu-binding of @worker are kept coordinated with the pool across
1688 * cpu-[un]hotplugs.
1689 */
1690static void worker_attach_to_pool(struct worker *worker,
1691 struct worker_pool *pool)
1692{
1693 mutex_lock(&pool->attach_mutex);
1694
1695 /*
1696 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1697 * online CPUs. It'll be re-applied when any of the CPUs come up.
1698 */
1699 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1700
1701 /*
1702 * The pool->attach_mutex ensures %POOL_DISASSOCIATED remains
1703 * stable across this function. See the comments above the
1704 * flag definition for details.
1705 */
1706 if (pool->flags & POOL_DISASSOCIATED)
1707 worker->flags |= WORKER_UNBOUND;
1708
1709 list_add_tail(&worker->node, &pool->workers);
1710
1711 mutex_unlock(&pool->attach_mutex);
1712}
1713
60f5a4bc
LJ
1714/**
1715 * worker_detach_from_pool() - detach a worker from its pool
1716 * @worker: worker which is attached to its pool
1717 * @pool: the pool @worker is attached to
1718 *
4736cbf7
LJ
1719 * Undo the attaching which had been done in worker_attach_to_pool(). The
1720 * caller worker shouldn't access to the pool after detached except it has
1721 * other reference to the pool.
60f5a4bc
LJ
1722 */
1723static void worker_detach_from_pool(struct worker *worker,
1724 struct worker_pool *pool)
1725{
1726 struct completion *detach_completion = NULL;
1727
92f9c5c4 1728 mutex_lock(&pool->attach_mutex);
da028469
LJ
1729 list_del(&worker->node);
1730 if (list_empty(&pool->workers))
60f5a4bc 1731 detach_completion = pool->detach_completion;
92f9c5c4 1732 mutex_unlock(&pool->attach_mutex);
60f5a4bc 1733
b62c0751
LJ
1734 /* clear leftover flags without pool->lock after it is detached */
1735 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
1736
60f5a4bc
LJ
1737 if (detach_completion)
1738 complete(detach_completion);
1739}
1740
c34056a3
TH
1741/**
1742 * create_worker - create a new workqueue worker
63d95a91 1743 * @pool: pool the new worker will belong to
c34056a3 1744 *
051e1850 1745 * Create and start a new worker which is attached to @pool.
c34056a3
TH
1746 *
1747 * CONTEXT:
1748 * Might sleep. Does GFP_KERNEL allocations.
1749 *
d185af30 1750 * Return:
c34056a3
TH
1751 * Pointer to the newly created worker.
1752 */
bc2ae0f5 1753static struct worker *create_worker(struct worker_pool *pool)
c34056a3 1754{
c34056a3 1755 struct worker *worker = NULL;
f3421797 1756 int id = -1;
e3c916a4 1757 char id_buf[16];
c34056a3 1758
7cda9aae
LJ
1759 /* ID is needed to determine kthread name */
1760 id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);
822d8405
TH
1761 if (id < 0)
1762 goto fail;
c34056a3 1763
f7537df5 1764 worker = alloc_worker(pool->node);
c34056a3
TH
1765 if (!worker)
1766 goto fail;
1767
bd7bdd43 1768 worker->pool = pool;
c34056a3
TH
1769 worker->id = id;
1770
29c91e99 1771 if (pool->cpu >= 0)
e3c916a4
TH
1772 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1773 pool->attrs->nice < 0 ? "H" : "");
f3421797 1774 else
e3c916a4
TH
1775 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1776
f3f90ad4 1777 worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
e3c916a4 1778 "kworker/%s", id_buf);
c34056a3
TH
1779 if (IS_ERR(worker->task))
1780 goto fail;
1781
91151228 1782 set_user_nice(worker->task, pool->attrs->nice);
25834c73 1783 kthread_bind_mask(worker->task, pool->attrs->cpumask);
91151228 1784
da028469 1785 /* successful, attach the worker to the pool */
4736cbf7 1786 worker_attach_to_pool(worker, pool);
822d8405 1787
051e1850
LJ
1788 /* start the newly created worker */
1789 spin_lock_irq(&pool->lock);
1790 worker->pool->nr_workers++;
1791 worker_enter_idle(worker);
1792 wake_up_process(worker->task);
1793 spin_unlock_irq(&pool->lock);
1794
c34056a3 1795 return worker;
822d8405 1796
c34056a3 1797fail:
9625ab17 1798 if (id >= 0)
7cda9aae 1799 ida_simple_remove(&pool->worker_ida, id);
c34056a3
TH
1800 kfree(worker);
1801 return NULL;
1802}
1803
c34056a3
TH
1804/**
1805 * destroy_worker - destroy a workqueue worker
1806 * @worker: worker to be destroyed
1807 *
73eb7fe7
LJ
1808 * Destroy @worker and adjust @pool stats accordingly. The worker should
1809 * be idle.
c8e55f36
TH
1810 *
1811 * CONTEXT:
60f5a4bc 1812 * spin_lock_irq(pool->lock).
c34056a3
TH
1813 */
1814static void destroy_worker(struct worker *worker)
1815{
bd7bdd43 1816 struct worker_pool *pool = worker->pool;
c34056a3 1817
cd549687
TH
1818 lockdep_assert_held(&pool->lock);
1819
c34056a3 1820 /* sanity check frenzy */
6183c009 1821 if (WARN_ON(worker->current_work) ||
73eb7fe7
LJ
1822 WARN_ON(!list_empty(&worker->scheduled)) ||
1823 WARN_ON(!(worker->flags & WORKER_IDLE)))
6183c009 1824 return;
c34056a3 1825
73eb7fe7
LJ
1826 pool->nr_workers--;
1827 pool->nr_idle--;
5bdfff96 1828
c8e55f36 1829 list_del_init(&worker->entry);
cb444766 1830 worker->flags |= WORKER_DIE;
60f5a4bc 1831 wake_up_process(worker->task);
c34056a3
TH
1832}
1833
63d95a91 1834static void idle_worker_timeout(unsigned long __pool)
e22bee78 1835{
63d95a91 1836 struct worker_pool *pool = (void *)__pool;
e22bee78 1837
d565ed63 1838 spin_lock_irq(&pool->lock);
e22bee78 1839
3347fc9f 1840 while (too_many_workers(pool)) {
e22bee78
TH
1841 struct worker *worker;
1842 unsigned long expires;
1843
1844 /* idle_list is kept in LIFO order, check the last one */
63d95a91 1845 worker = list_entry(pool->idle_list.prev, struct worker, entry);
e22bee78
TH
1846 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1847
3347fc9f 1848 if (time_before(jiffies, expires)) {
63d95a91 1849 mod_timer(&pool->idle_timer, expires);
3347fc9f 1850 break;
d5abe669 1851 }
3347fc9f
LJ
1852
1853 destroy_worker(worker);
e22bee78
TH
1854 }
1855
d565ed63 1856 spin_unlock_irq(&pool->lock);
e22bee78 1857}
d5abe669 1858
493a1724 1859static void send_mayday(struct work_struct *work)
e22bee78 1860{
112202d9
TH
1861 struct pool_workqueue *pwq = get_work_pwq(work);
1862 struct workqueue_struct *wq = pwq->wq;
493a1724 1863
2e109a28 1864 lockdep_assert_held(&wq_mayday_lock);
e22bee78 1865
493008a8 1866 if (!wq->rescuer)
493a1724 1867 return;
e22bee78
TH
1868
1869 /* mayday mayday mayday */
493a1724 1870 if (list_empty(&pwq->mayday_node)) {
77668c8b
LJ
1871 /*
1872 * If @pwq is for an unbound wq, its base ref may be put at
1873 * any time due to an attribute change. Pin @pwq until the
1874 * rescuer is done with it.
1875 */
1876 get_pwq(pwq);
493a1724 1877 list_add_tail(&pwq->mayday_node, &wq->maydays);
e22bee78 1878 wake_up_process(wq->rescuer->task);
493a1724 1879 }
e22bee78
TH
1880}
1881
706026c2 1882static void pool_mayday_timeout(unsigned long __pool)
e22bee78 1883{
63d95a91 1884 struct worker_pool *pool = (void *)__pool;
e22bee78
TH
1885 struct work_struct *work;
1886
b2d82909
TH
1887 spin_lock_irq(&pool->lock);
1888 spin_lock(&wq_mayday_lock); /* for wq->maydays */
e22bee78 1889
63d95a91 1890 if (need_to_create_worker(pool)) {
e22bee78
TH
1891 /*
1892 * We've been trying to create a new worker but
1893 * haven't been successful. We might be hitting an
1894 * allocation deadlock. Send distress signals to
1895 * rescuers.
1896 */
63d95a91 1897 list_for_each_entry(work, &pool->worklist, entry)
e22bee78 1898 send_mayday(work);
1da177e4 1899 }
e22bee78 1900
b2d82909
TH
1901 spin_unlock(&wq_mayday_lock);
1902 spin_unlock_irq(&pool->lock);
e22bee78 1903
63d95a91 1904 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1da177e4
LT
1905}
1906
e22bee78
TH
1907/**
1908 * maybe_create_worker - create a new worker if necessary
63d95a91 1909 * @pool: pool to create a new worker for
e22bee78 1910 *
63d95a91 1911 * Create a new worker for @pool if necessary. @pool is guaranteed to
e22bee78
TH
1912 * have at least one idle worker on return from this function. If
1913 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
63d95a91 1914 * sent to all rescuers with works scheduled on @pool to resolve
e22bee78
TH
1915 * possible allocation deadlock.
1916 *
c5aa87bb
TH
1917 * On return, need_to_create_worker() is guaranteed to be %false and
1918 * may_start_working() %true.
e22bee78
TH
1919 *
1920 * LOCKING:
d565ed63 1921 * spin_lock_irq(pool->lock) which may be released and regrabbed
e22bee78
TH
1922 * multiple times. Does GFP_KERNEL allocations. Called only from
1923 * manager.
e22bee78 1924 */
29187a9e 1925static void maybe_create_worker(struct worker_pool *pool)
d565ed63
TH
1926__releases(&pool->lock)
1927__acquires(&pool->lock)
1da177e4 1928{
e22bee78 1929restart:
d565ed63 1930 spin_unlock_irq(&pool->lock);
9f9c2364 1931
e22bee78 1932 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
63d95a91 1933 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
e22bee78
TH
1934
1935 while (true) {
051e1850 1936 if (create_worker(pool) || !need_to_create_worker(pool))
e22bee78 1937 break;
1da177e4 1938
e212f361 1939 schedule_timeout_interruptible(CREATE_COOLDOWN);
9f9c2364 1940
63d95a91 1941 if (!need_to_create_worker(pool))
e22bee78
TH
1942 break;
1943 }
1944
63d95a91 1945 del_timer_sync(&pool->mayday_timer);
d565ed63 1946 spin_lock_irq(&pool->lock);
051e1850
LJ
1947 /*
1948 * This is necessary even after a new worker was just successfully
1949 * created as @pool->lock was dropped and the new worker might have
1950 * already become busy.
1951 */
63d95a91 1952 if (need_to_create_worker(pool))
e22bee78 1953 goto restart;
e22bee78
TH
1954}
1955
73f53c4a 1956/**
e22bee78
TH
1957 * manage_workers - manage worker pool
1958 * @worker: self
73f53c4a 1959 *
706026c2 1960 * Assume the manager role and manage the worker pool @worker belongs
e22bee78 1961 * to. At any given time, there can be only zero or one manager per
706026c2 1962 * pool. The exclusion is handled automatically by this function.
e22bee78
TH
1963 *
1964 * The caller can safely start processing works on false return. On
1965 * true return, it's guaranteed that need_to_create_worker() is false
1966 * and may_start_working() is true.
73f53c4a
TH
1967 *
1968 * CONTEXT:
d565ed63 1969 * spin_lock_irq(pool->lock) which may be released and regrabbed
e22bee78
TH
1970 * multiple times. Does GFP_KERNEL allocations.
1971 *
d185af30 1972 * Return:
29187a9e
TH
1973 * %false if the pool doesn't need management and the caller can safely
1974 * start processing works, %true if management function was performed and
1975 * the conditions that the caller verified before calling the function may
1976 * no longer be true.
73f53c4a 1977 */
e22bee78 1978static bool manage_workers(struct worker *worker)
73f53c4a 1979{
63d95a91 1980 struct worker_pool *pool = worker->pool;
73f53c4a 1981
bc3a1afc 1982 /*
bc3a1afc
TH
1983 * Anyone who successfully grabs manager_arb wins the arbitration
1984 * and becomes the manager. mutex_trylock() on pool->manager_arb
1985 * failure while holding pool->lock reliably indicates that someone
1986 * else is managing the pool and the worker which failed trylock
1987 * can proceed to executing work items. This means that anyone
1988 * grabbing manager_arb is responsible for actually performing
1989 * manager duties. If manager_arb is grabbed and released without
1990 * actual management, the pool may stall indefinitely.
bc3a1afc 1991 */
34a06bd6 1992 if (!mutex_trylock(&pool->manager_arb))
29187a9e 1993 return false;
2607d7a6 1994 pool->manager = worker;
1e19ffc6 1995
29187a9e 1996 maybe_create_worker(pool);
e22bee78 1997
2607d7a6 1998 pool->manager = NULL;
34a06bd6 1999 mutex_unlock(&pool->manager_arb);
29187a9e 2000 return true;
73f53c4a
TH
2001}
2002
a62428c0
TH
2003/**
2004 * process_one_work - process single work
c34056a3 2005 * @worker: self
a62428c0
TH
2006 * @work: work to process
2007 *
2008 * Process @work. This function contains all the logics necessary to
2009 * process a single work including synchronization against and
2010 * interaction with other workers on the same cpu, queueing and
2011 * flushing. As long as context requirement is met, any worker can
2012 * call this function to process a work.
2013 *
2014 * CONTEXT:
d565ed63 2015 * spin_lock_irq(pool->lock) which is released and regrabbed.
a62428c0 2016 */
c34056a3 2017static void process_one_work(struct worker *worker, struct work_struct *work)
d565ed63
TH
2018__releases(&pool->lock)
2019__acquires(&pool->lock)
a62428c0 2020{
112202d9 2021 struct pool_workqueue *pwq = get_work_pwq(work);
bd7bdd43 2022 struct worker_pool *pool = worker->pool;
112202d9 2023 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
73f53c4a 2024 int work_color;
7e11629d 2025 struct worker *collision;
a62428c0
TH
2026#ifdef CONFIG_LOCKDEP
2027 /*
2028 * It is permissible to free the struct work_struct from
2029 * inside the function that is called from it, this we need to
2030 * take into account for lockdep too. To avoid bogus "held
2031 * lock freed" warnings as well as problems when looking into
2032 * work->lockdep_map, make a copy and use that here.
2033 */
4d82a1de
PZ
2034 struct lockdep_map lockdep_map;
2035
2036 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
a62428c0 2037#endif
807407c0 2038 /* ensure we're on the correct CPU */
85327af6 2039 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
ec22ca5e 2040 raw_smp_processor_id() != pool->cpu);
25511a47 2041
7e11629d
TH
2042 /*
2043 * A single work shouldn't be executed concurrently by
2044 * multiple workers on a single cpu. Check whether anyone is
2045 * already processing the work. If so, defer the work to the
2046 * currently executing one.
2047 */
c9e7cf27 2048 collision = find_worker_executing_work(pool, work);
7e11629d
TH
2049 if (unlikely(collision)) {
2050 move_linked_works(work, &collision->scheduled, NULL);
2051 return;
2052 }
2053
8930caba 2054 /* claim and dequeue */
a62428c0 2055 debug_work_deactivate(work);
c9e7cf27 2056 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
c34056a3 2057 worker->current_work = work;
a2c1c57b 2058 worker->current_func = work->func;
112202d9 2059 worker->current_pwq = pwq;
73f53c4a 2060 work_color = get_work_color(work);
7a22ad75 2061
a62428c0
TH
2062 list_del_init(&work->entry);
2063
fb0e7beb 2064 /*
228f1d00
LJ
2065 * CPU intensive works don't participate in concurrency management.
2066 * They're the scheduler's responsibility. This takes @worker out
2067 * of concurrency management and the next code block will chain
2068 * execution of the pending work items.
fb0e7beb
TH
2069 */
2070 if (unlikely(cpu_intensive))
228f1d00 2071 worker_set_flags(worker, WORKER_CPU_INTENSIVE);
fb0e7beb 2072
974271c4 2073 /*
a489a03e
LJ
2074 * Wake up another worker if necessary. The condition is always
2075 * false for normal per-cpu workers since nr_running would always
2076 * be >= 1 at this point. This is used to chain execution of the
2077 * pending work items for WORKER_NOT_RUNNING workers such as the
228f1d00 2078 * UNBOUND and CPU_INTENSIVE ones.
974271c4 2079 */
a489a03e 2080 if (need_more_worker(pool))
63d95a91 2081 wake_up_worker(pool);
974271c4 2082
8930caba 2083 /*
7c3eed5c 2084 * Record the last pool and clear PENDING which should be the last
d565ed63 2085 * update to @work. Also, do this inside @pool->lock so that
23657bb1
TH
2086 * PENDING and queued state changes happen together while IRQ is
2087 * disabled.
8930caba 2088 */
7c3eed5c 2089 set_work_pool_and_clear_pending(work, pool->id);
a62428c0 2090
d565ed63 2091 spin_unlock_irq(&pool->lock);
a62428c0 2092
112202d9 2093 lock_map_acquire_read(&pwq->wq->lockdep_map);
a62428c0 2094 lock_map_acquire(&lockdep_map);
e36c886a 2095 trace_workqueue_execute_start(work);
a2c1c57b 2096 worker->current_func(work);
e36c886a
AV
2097 /*
2098 * While we must be careful to not use "work" after this, the trace
2099 * point will only record its address.
2100 */
2101 trace_workqueue_execute_end(work);
a62428c0 2102 lock_map_release(&lockdep_map);
112202d9 2103 lock_map_release(&pwq->wq->lockdep_map);
a62428c0
TH
2104
2105 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
044c782c
VI
2106 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2107 " last function: %pf\n",
a2c1c57b
TH
2108 current->comm, preempt_count(), task_pid_nr(current),
2109 worker->current_func);
a62428c0
TH
2110 debug_show_held_locks(current);
2111 dump_stack();
2112 }
2113
b22ce278
TH
2114 /*
2115 * The following prevents a kworker from hogging CPU on !PREEMPT
2116 * kernels, where a requeueing work item waiting for something to
2117 * happen could deadlock with stop_machine as such work item could
2118 * indefinitely requeue itself while all other CPUs are trapped in
789cbbec
JL
2119 * stop_machine. At the same time, report a quiescent RCU state so
2120 * the same condition doesn't freeze RCU.
b22ce278 2121 */
3e28e377 2122 cond_resched_rcu_qs();
b22ce278 2123
d565ed63 2124 spin_lock_irq(&pool->lock);
a62428c0 2125
fb0e7beb
TH
2126 /* clear cpu intensive status */
2127 if (unlikely(cpu_intensive))
2128 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2129
a62428c0 2130 /* we're done with it, release */
42f8570f 2131 hash_del(&worker->hentry);
c34056a3 2132 worker->current_work = NULL;
a2c1c57b 2133 worker->current_func = NULL;
112202d9 2134 worker->current_pwq = NULL;
3d1cb205 2135 worker->desc_valid = false;
112202d9 2136 pwq_dec_nr_in_flight(pwq, work_color);
a62428c0
TH
2137}
2138
affee4b2
TH
2139/**
2140 * process_scheduled_works - process scheduled works
2141 * @worker: self
2142 *
2143 * Process all scheduled works. Please note that the scheduled list
2144 * may change while processing a work, so this function repeatedly
2145 * fetches a work from the top and executes it.
2146 *
2147 * CONTEXT:
d565ed63 2148 * spin_lock_irq(pool->lock) which may be released and regrabbed
affee4b2
TH
2149 * multiple times.
2150 */
2151static void process_scheduled_works(struct worker *worker)
1da177e4 2152{
affee4b2
TH
2153 while (!list_empty(&worker->scheduled)) {
2154 struct work_struct *work = list_first_entry(&worker->scheduled,
1da177e4 2155 struct work_struct, entry);
c34056a3 2156 process_one_work(worker, work);
1da177e4 2157 }
1da177e4
LT
2158}
2159
4690c4ab
TH
2160/**
2161 * worker_thread - the worker thread function
c34056a3 2162 * @__worker: self
4690c4ab 2163 *
c5aa87bb
TH
2164 * The worker thread function. All workers belong to a worker_pool -
2165 * either a per-cpu one or dynamic unbound one. These workers process all
2166 * work items regardless of their specific target workqueue. The only
2167 * exception is work items which belong to workqueues with a rescuer which
2168 * will be explained in rescuer_thread().
d185af30
YB
2169 *
2170 * Return: 0
4690c4ab 2171 */
c34056a3 2172static int worker_thread(void *__worker)
1da177e4 2173{
c34056a3 2174 struct worker *worker = __worker;
bd7bdd43 2175 struct worker_pool *pool = worker->pool;
1da177e4 2176
e22bee78
TH
2177 /* tell the scheduler that this is a workqueue worker */
2178 worker->task->flags |= PF_WQ_WORKER;
c8e55f36 2179woke_up:
d565ed63 2180 spin_lock_irq(&pool->lock);
1da177e4 2181
a9ab775b
TH
2182 /* am I supposed to die? */
2183 if (unlikely(worker->flags & WORKER_DIE)) {
d565ed63 2184 spin_unlock_irq(&pool->lock);
a9ab775b
TH
2185 WARN_ON_ONCE(!list_empty(&worker->entry));
2186 worker->task->flags &= ~PF_WQ_WORKER;
60f5a4bc
LJ
2187
2188 set_task_comm(worker->task, "kworker/dying");
7cda9aae 2189 ida_simple_remove(&pool->worker_ida, worker->id);
60f5a4bc
LJ
2190 worker_detach_from_pool(worker, pool);
2191 kfree(worker);
a9ab775b 2192 return 0;
c8e55f36 2193 }
affee4b2 2194
c8e55f36 2195 worker_leave_idle(worker);
db7bccf4 2196recheck:
e22bee78 2197 /* no more worker necessary? */
63d95a91 2198 if (!need_more_worker(pool))
e22bee78
TH
2199 goto sleep;
2200
2201 /* do we need to manage? */
63d95a91 2202 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
e22bee78
TH
2203 goto recheck;
2204
c8e55f36
TH
2205 /*
2206 * ->scheduled list can only be filled while a worker is
2207 * preparing to process a work or actually processing it.
2208 * Make sure nobody diddled with it while I was sleeping.
2209 */
6183c009 2210 WARN_ON_ONCE(!list_empty(&worker->scheduled));
c8e55f36 2211
e22bee78 2212 /*
a9ab775b
TH
2213 * Finish PREP stage. We're guaranteed to have at least one idle
2214 * worker or that someone else has already assumed the manager
2215 * role. This is where @worker starts participating in concurrency
2216 * management if applicable and concurrency management is restored
2217 * after being rebound. See rebind_workers() for details.
e22bee78 2218 */
a9ab775b 2219 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
e22bee78
TH
2220
2221 do {
c8e55f36 2222 struct work_struct *work =
bd7bdd43 2223 list_first_entry(&pool->worklist,
c8e55f36
TH
2224 struct work_struct, entry);
2225
82607adc
TH
2226 pool->watchdog_ts = jiffies;
2227
c8e55f36
TH
2228 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2229 /* optimization path, not strictly necessary */
2230 process_one_work(worker, work);
2231 if (unlikely(!list_empty(&worker->scheduled)))
affee4b2 2232 process_scheduled_works(worker);
c8e55f36
TH
2233 } else {
2234 move_linked_works(work, &worker->scheduled, NULL);
2235 process_scheduled_works(worker);
affee4b2 2236 }
63d95a91 2237 } while (keep_working(pool));
e22bee78 2238
228f1d00 2239 worker_set_flags(worker, WORKER_PREP);
d313dd85 2240sleep:
c8e55f36 2241 /*
d565ed63
TH
2242 * pool->lock is held and there's no work to process and no need to
2243 * manage, sleep. Workers are woken up only while holding
2244 * pool->lock or from local cpu, so setting the current state
2245 * before releasing pool->lock is enough to prevent losing any
2246 * event.
c8e55f36
TH
2247 */
2248 worker_enter_idle(worker);
2249 __set_current_state(TASK_INTERRUPTIBLE);
d565ed63 2250 spin_unlock_irq(&pool->lock);
c8e55f36
TH
2251 schedule();
2252 goto woke_up;
1da177e4
LT
2253}
2254
e22bee78
TH
2255/**
2256 * rescuer_thread - the rescuer thread function
111c225a 2257 * @__rescuer: self
e22bee78
TH
2258 *
2259 * Workqueue rescuer thread function. There's one rescuer for each
493008a8 2260 * workqueue which has WQ_MEM_RECLAIM set.
e22bee78 2261 *
706026c2 2262 * Regular work processing on a pool may block trying to create a new
e22bee78
TH
2263 * worker which uses GFP_KERNEL allocation which has slight chance of
2264 * developing into deadlock if some works currently on the same queue
2265 * need to be processed to satisfy the GFP_KERNEL allocation. This is
2266 * the problem rescuer solves.
2267 *
706026c2
TH
2268 * When such condition is possible, the pool summons rescuers of all
2269 * workqueues which have works queued on the pool and let them process
e22bee78
TH
2270 * those works so that forward progress can be guaranteed.
2271 *
2272 * This should happen rarely.
d185af30
YB
2273 *
2274 * Return: 0
e22bee78 2275 */
111c225a 2276static int rescuer_thread(void *__rescuer)
e22bee78 2277{
111c225a
TH
2278 struct worker *rescuer = __rescuer;
2279 struct workqueue_struct *wq = rescuer->rescue_wq;
e22bee78 2280 struct list_head *scheduled = &rescuer->scheduled;
4d595b86 2281 bool should_stop;
e22bee78
TH
2282
2283 set_user_nice(current, RESCUER_NICE_LEVEL);
111c225a
TH
2284
2285 /*
2286 * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2287 * doesn't participate in concurrency management.
2288 */
2289 rescuer->task->flags |= PF_WQ_WORKER;
e22bee78
TH
2290repeat:
2291 set_current_state(TASK_INTERRUPTIBLE);
2292
4d595b86
LJ
2293 /*
2294 * By the time the rescuer is requested to stop, the workqueue
2295 * shouldn't have any work pending, but @wq->maydays may still have
2296 * pwq(s) queued. This can happen by non-rescuer workers consuming
2297 * all the work items before the rescuer got to them. Go through
2298 * @wq->maydays processing before acting on should_stop so that the
2299 * list is always empty on exit.
2300 */
2301 should_stop = kthread_should_stop();
e22bee78 2302
493a1724 2303 /* see whether any pwq is asking for help */
2e109a28 2304 spin_lock_irq(&wq_mayday_lock);
493a1724
TH
2305
2306 while (!list_empty(&wq->maydays)) {
2307 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2308 struct pool_workqueue, mayday_node);
112202d9 2309 struct worker_pool *pool = pwq->pool;
e22bee78 2310 struct work_struct *work, *n;
82607adc 2311 bool first = true;
e22bee78
TH
2312
2313 __set_current_state(TASK_RUNNING);
493a1724
TH
2314 list_del_init(&pwq->mayday_node);
2315
2e109a28 2316 spin_unlock_irq(&wq_mayday_lock);
e22bee78 2317
51697d39
LJ
2318 worker_attach_to_pool(rescuer, pool);
2319
2320 spin_lock_irq(&pool->lock);
b3104104 2321 rescuer->pool = pool;
e22bee78
TH
2322
2323 /*
2324 * Slurp in all works issued via this workqueue and
2325 * process'em.
2326 */
0479c8c5 2327 WARN_ON_ONCE(!list_empty(scheduled));
82607adc
TH
2328 list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2329 if (get_work_pwq(work) == pwq) {
2330 if (first)
2331 pool->watchdog_ts = jiffies;
e22bee78 2332 move_linked_works(work, scheduled, &n);
82607adc
TH
2333 }
2334 first = false;
2335 }
e22bee78 2336
008847f6
N
2337 if (!list_empty(scheduled)) {
2338 process_scheduled_works(rescuer);
2339
2340 /*
2341 * The above execution of rescued work items could
2342 * have created more to rescue through
2343 * pwq_activate_first_delayed() or chained
2344 * queueing. Let's put @pwq back on mayday list so
2345 * that such back-to-back work items, which may be
2346 * being used to relieve memory pressure, don't
2347 * incur MAYDAY_INTERVAL delay inbetween.
2348 */
2349 if (need_to_create_worker(pool)) {
2350 spin_lock(&wq_mayday_lock);
2351 get_pwq(pwq);
2352 list_move_tail(&pwq->mayday_node, &wq->maydays);
2353 spin_unlock(&wq_mayday_lock);
2354 }
2355 }
7576958a 2356
77668c8b
LJ
2357 /*
2358 * Put the reference grabbed by send_mayday(). @pool won't
13b1d625 2359 * go away while we're still attached to it.
77668c8b
LJ
2360 */
2361 put_pwq(pwq);
2362
7576958a 2363 /*
d8ca83e6 2364 * Leave this pool. If need_more_worker() is %true, notify a
7576958a
TH
2365 * regular worker; otherwise, we end up with 0 concurrency
2366 * and stalling the execution.
2367 */
d8ca83e6 2368 if (need_more_worker(pool))
63d95a91 2369 wake_up_worker(pool);
7576958a 2370
b3104104 2371 rescuer->pool = NULL;
13b1d625
LJ
2372 spin_unlock_irq(&pool->lock);
2373
2374 worker_detach_from_pool(rescuer, pool);
2375
2376 spin_lock_irq(&wq_mayday_lock);
e22bee78
TH
2377 }
2378
2e109a28 2379 spin_unlock_irq(&wq_mayday_lock);
493a1724 2380
4d595b86
LJ
2381 if (should_stop) {
2382 __set_current_state(TASK_RUNNING);
2383 rescuer->task->flags &= ~PF_WQ_WORKER;
2384 return 0;
2385 }
2386
111c225a
TH
2387 /* rescuers should never participate in concurrency management */
2388 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
e22bee78
TH
2389 schedule();
2390 goto repeat;
1da177e4
LT
2391}
2392
fca839c0
TH
2393/**
2394 * check_flush_dependency - check for flush dependency sanity
2395 * @target_wq: workqueue being flushed
2396 * @target_work: work item being flushed (NULL for workqueue flushes)
2397 *
2398 * %current is trying to flush the whole @target_wq or @target_work on it.
2399 * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2400 * reclaiming memory or running on a workqueue which doesn't have
2401 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2402 * a deadlock.
2403 */
2404static void check_flush_dependency(struct workqueue_struct *target_wq,
2405 struct work_struct *target_work)
2406{
2407 work_func_t target_func = target_work ? target_work->func : NULL;
2408 struct worker *worker;
2409
2410 if (target_wq->flags & WQ_MEM_RECLAIM)
2411 return;
2412
2413 worker = current_wq_worker();
2414
2415 WARN_ONCE(current->flags & PF_MEMALLOC,
2416 "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf",
2417 current->pid, current->comm, target_wq->name, target_func);
23d11a58
TH
2418 WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2419 (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
fca839c0
TH
2420 "workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf",
2421 worker->current_pwq->wq->name, worker->current_func,
2422 target_wq->name, target_func);
2423}
2424
fc2e4d70
ON
2425struct wq_barrier {
2426 struct work_struct work;
2427 struct completion done;
2607d7a6 2428 struct task_struct *task; /* purely informational */
fc2e4d70
ON
2429};
2430
2431static void wq_barrier_func(struct work_struct *work)
2432{
2433 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2434 complete(&barr->done);
2435}
2436
4690c4ab
TH
2437/**
2438 * insert_wq_barrier - insert a barrier work
112202d9 2439 * @pwq: pwq to insert barrier into
4690c4ab 2440 * @barr: wq_barrier to insert
affee4b2
TH
2441 * @target: target work to attach @barr to
2442 * @worker: worker currently executing @target, NULL if @target is not executing
4690c4ab 2443 *
affee4b2
TH
2444 * @barr is linked to @target such that @barr is completed only after
2445 * @target finishes execution. Please note that the ordering
2446 * guarantee is observed only with respect to @target and on the local
2447 * cpu.
2448 *
2449 * Currently, a queued barrier can't be canceled. This is because
2450 * try_to_grab_pending() can't determine whether the work to be
2451 * grabbed is at the head of the queue and thus can't clear LINKED
2452 * flag of the previous work while there must be a valid next work
2453 * after a work with LINKED flag set.
2454 *
2455 * Note that when @worker is non-NULL, @target may be modified
112202d9 2456 * underneath us, so we can't reliably determine pwq from @target.
4690c4ab
TH
2457 *
2458 * CONTEXT:
d565ed63 2459 * spin_lock_irq(pool->lock).
4690c4ab 2460 */
112202d9 2461static void insert_wq_barrier(struct pool_workqueue *pwq,
affee4b2
TH
2462 struct wq_barrier *barr,
2463 struct work_struct *target, struct worker *worker)
fc2e4d70 2464{
affee4b2
TH
2465 struct list_head *head;
2466 unsigned int linked = 0;
2467
dc186ad7 2468 /*
d565ed63 2469 * debugobject calls are safe here even with pool->lock locked
dc186ad7
TG
2470 * as we know for sure that this will not trigger any of the
2471 * checks and call back into the fixup functions where we
2472 * might deadlock.
2473 */
ca1cab37 2474 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
22df02bb 2475 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
fc2e4d70 2476 init_completion(&barr->done);
2607d7a6 2477 barr->task = current;
83c22520 2478
affee4b2
TH
2479 /*
2480 * If @target is currently being executed, schedule the
2481 * barrier to the worker; otherwise, put it after @target.
2482 */
2483 if (worker)
2484 head = worker->scheduled.next;
2485 else {
2486 unsigned long *bits = work_data_bits(target);
2487
2488 head = target->entry.next;
2489 /* there can already be other linked works, inherit and set */
2490 linked = *bits & WORK_STRUCT_LINKED;
2491 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2492 }
2493
dc186ad7 2494 debug_work_activate(&barr->work);
112202d9 2495 insert_work(pwq, &barr->work, head,
affee4b2 2496 work_color_to_flags(WORK_NO_COLOR) | linked);
fc2e4d70
ON
2497}
2498
73f53c4a 2499/**
112202d9 2500 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
73f53c4a
TH
2501 * @wq: workqueue being flushed
2502 * @flush_color: new flush color, < 0 for no-op
2503 * @work_color: new work color, < 0 for no-op
2504 *
112202d9 2505 * Prepare pwqs for workqueue flushing.
73f53c4a 2506 *
112202d9
TH
2507 * If @flush_color is non-negative, flush_color on all pwqs should be
2508 * -1. If no pwq has in-flight commands at the specified color, all
2509 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2510 * has in flight commands, its pwq->flush_color is set to
2511 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
73f53c4a
TH
2512 * wakeup logic is armed and %true is returned.
2513 *
2514 * The caller should have initialized @wq->first_flusher prior to
2515 * calling this function with non-negative @flush_color. If
2516 * @flush_color is negative, no flush color update is done and %false
2517 * is returned.
2518 *
112202d9 2519 * If @work_color is non-negative, all pwqs should have the same
73f53c4a
TH
2520 * work_color which is previous to @work_color and all will be
2521 * advanced to @work_color.
2522 *
2523 * CONTEXT:
3c25a55d 2524 * mutex_lock(wq->mutex).
73f53c4a 2525 *
d185af30 2526 * Return:
73f53c4a
TH
2527 * %true if @flush_color >= 0 and there's something to flush. %false
2528 * otherwise.
2529 */
112202d9 2530static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
73f53c4a 2531 int flush_color, int work_color)
1da177e4 2532{
73f53c4a 2533 bool wait = false;
49e3cf44 2534 struct pool_workqueue *pwq;
1da177e4 2535
73f53c4a 2536 if (flush_color >= 0) {
6183c009 2537 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
112202d9 2538 atomic_set(&wq->nr_pwqs_to_flush, 1);
1da177e4 2539 }
2355b70f 2540
49e3cf44 2541 for_each_pwq(pwq, wq) {
112202d9 2542 struct worker_pool *pool = pwq->pool;
fc2e4d70 2543
b09f4fd3 2544 spin_lock_irq(&pool->lock);
83c22520 2545
73f53c4a 2546 if (flush_color >= 0) {
6183c009 2547 WARN_ON_ONCE(pwq->flush_color != -1);
fc2e4d70 2548
112202d9
TH
2549 if (pwq->nr_in_flight[flush_color]) {
2550 pwq->flush_color = flush_color;
2551 atomic_inc(&wq->nr_pwqs_to_flush);
73f53c4a
TH
2552 wait = true;
2553 }
2554 }
1da177e4 2555
73f53c4a 2556 if (work_color >= 0) {
6183c009 2557 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
112202d9 2558 pwq->work_color = work_color;
73f53c4a 2559 }
1da177e4 2560
b09f4fd3 2561 spin_unlock_irq(&pool->lock);
1da177e4 2562 }
2355b70f 2563
112202d9 2564 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
73f53c4a 2565 complete(&wq->first_flusher->done);
14441960 2566
73f53c4a 2567 return wait;
1da177e4
LT
2568}
2569
0fcb78c2 2570/**
1da177e4 2571 * flush_workqueue - ensure that any scheduled work has run to completion.
0fcb78c2 2572 * @wq: workqueue to flush
1da177e4 2573 *
c5aa87bb
TH
2574 * This function sleeps until all work items which were queued on entry
2575 * have finished execution, but it is not livelocked by new incoming ones.
1da177e4 2576 */
7ad5b3a5 2577void flush_workqueue(struct workqueue_struct *wq)
1da177e4 2578{
73f53c4a
TH
2579 struct wq_flusher this_flusher = {
2580 .list = LIST_HEAD_INIT(this_flusher.list),
2581 .flush_color = -1,
2582 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2583 };
2584 int next_color;
1da177e4 2585
3295f0ef
IM
2586 lock_map_acquire(&wq->lockdep_map);
2587 lock_map_release(&wq->lockdep_map);
73f53c4a 2588
3c25a55d 2589 mutex_lock(&wq->mutex);
73f53c4a
TH
2590
2591 /*
2592 * Start-to-wait phase
2593 */
2594 next_color = work_next_color(wq->work_color);
2595
2596 if (next_color != wq->flush_color) {
2597 /*
2598 * Color space is not full. The current work_color
2599 * becomes our flush_color and work_color is advanced
2600 * by one.
2601 */
6183c009 2602 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
73f53c4a
TH
2603 this_flusher.flush_color = wq->work_color;
2604 wq->work_color = next_color;
2605
2606 if (!wq->first_flusher) {
2607 /* no flush in progress, become the first flusher */
6183c009 2608 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
73f53c4a
TH
2609
2610 wq->first_flusher = &this_flusher;
2611
112202d9 2612 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
73f53c4a
TH
2613 wq->work_color)) {
2614 /* nothing to flush, done */
2615 wq->flush_color = next_color;
2616 wq->first_flusher = NULL;
2617 goto out_unlock;
2618 }
2619 } else {
2620 /* wait in queue */
6183c009 2621 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
73f53c4a 2622 list_add_tail(&this_flusher.list, &wq->flusher_queue);
112202d9 2623 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
73f53c4a
TH
2624 }
2625 } else {
2626 /*
2627 * Oops, color space is full, wait on overflow queue.
2628 * The next flush completion will assign us
2629 * flush_color and transfer to flusher_queue.
2630 */
2631 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2632 }
2633
fca839c0
TH
2634 check_flush_dependency(wq, NULL);
2635
3c25a55d 2636 mutex_unlock(&wq->mutex);
73f53c4a
TH
2637
2638 wait_for_completion(&this_flusher.done);
2639
2640 /*
2641 * Wake-up-and-cascade phase
2642 *
2643 * First flushers are responsible for cascading flushes and
2644 * handling overflow. Non-first flushers can simply return.
2645 */
2646 if (wq->first_flusher != &this_flusher)
2647 return;
2648
3c25a55d 2649 mutex_lock(&wq->mutex);
73f53c4a 2650
4ce48b37
TH
2651 /* we might have raced, check again with mutex held */
2652 if (wq->first_flusher != &this_flusher)
2653 goto out_unlock;
2654
73f53c4a
TH
2655 wq->first_flusher = NULL;
2656
6183c009
TH
2657 WARN_ON_ONCE(!list_empty(&this_flusher.list));
2658 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
73f53c4a
TH
2659
2660 while (true) {
2661 struct wq_flusher *next, *tmp;
2662
2663 /* complete all the flushers sharing the current flush color */
2664 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2665 if (next->flush_color != wq->flush_color)
2666 break;
2667 list_del_init(&next->list);
2668 complete(&next->done);
2669 }
2670
6183c009
TH
2671 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2672 wq->flush_color != work_next_color(wq->work_color));
73f53c4a
TH
2673
2674 /* this flush_color is finished, advance by one */
2675 wq->flush_color = work_next_color(wq->flush_color);
2676
2677 /* one color has been freed, handle overflow queue */
2678 if (!list_empty(&wq->flusher_overflow)) {
2679 /*
2680 * Assign the same color to all overflowed
2681 * flushers, advance work_color and append to
2682 * flusher_queue. This is the start-to-wait
2683 * phase for these overflowed flushers.
2684 */
2685 list_for_each_entry(tmp, &wq->flusher_overflow, list)
2686 tmp->flush_color = wq->work_color;
2687
2688 wq->work_color = work_next_color(wq->work_color);
2689
2690 list_splice_tail_init(&wq->flusher_overflow,
2691 &wq->flusher_queue);
112202d9 2692 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
73f53c4a
TH
2693 }
2694
2695 if (list_empty(&wq->flusher_queue)) {
6183c009 2696 WARN_ON_ONCE(wq->flush_color != wq->work_color);
73f53c4a
TH
2697 break;
2698 }
2699
2700 /*
2701 * Need to flush more colors. Make the next flusher
112202d9 2702 * the new first flusher and arm pwqs.
73f53c4a 2703 */
6183c009
TH
2704 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2705 WARN_ON_ONCE(wq->flush_color != next->flush_color);
73f53c4a
TH
2706
2707 list_del_init(&next->list);
2708 wq->first_flusher = next;
2709
112202d9 2710 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
73f53c4a
TH
2711 break;
2712
2713 /*
2714 * Meh... this color is already done, clear first
2715 * flusher and repeat cascading.
2716 */
2717 wq->first_flusher = NULL;
2718 }
2719
2720out_unlock:
3c25a55d 2721 mutex_unlock(&wq->mutex);
1da177e4 2722}
1dadafa8 2723EXPORT_SYMBOL(flush_workqueue);
1da177e4 2724
9c5a2ba7
TH
2725/**
2726 * drain_workqueue - drain a workqueue
2727 * @wq: workqueue to drain
2728 *
2729 * Wait until the workqueue becomes empty. While draining is in progress,
2730 * only chain queueing is allowed. IOW, only currently pending or running
2731 * work items on @wq can queue further work items on it. @wq is flushed
b749b1b6 2732 * repeatedly until it becomes empty. The number of flushing is determined
9c5a2ba7
TH
2733 * by the depth of chaining and should be relatively short. Whine if it
2734 * takes too long.
2735 */
2736void drain_workqueue(struct workqueue_struct *wq)
2737{
2738 unsigned int flush_cnt = 0;
49e3cf44 2739 struct pool_workqueue *pwq;
9c5a2ba7
TH
2740
2741 /*
2742 * __queue_work() needs to test whether there are drainers, is much
2743 * hotter than drain_workqueue() and already looks at @wq->flags.
618b01eb 2744 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
9c5a2ba7 2745 */
87fc741e 2746 mutex_lock(&wq->mutex);
9c5a2ba7 2747 if (!wq->nr_drainers++)
618b01eb 2748 wq->flags |= __WQ_DRAINING;
87fc741e 2749 mutex_unlock(&wq->mutex);
9c5a2ba7
TH
2750reflush:
2751 flush_workqueue(wq);
2752
b09f4fd3 2753 mutex_lock(&wq->mutex);
76af4d93 2754
49e3cf44 2755 for_each_pwq(pwq, wq) {
fa2563e4 2756 bool drained;
9c5a2ba7 2757
b09f4fd3 2758 spin_lock_irq(&pwq->pool->lock);
112202d9 2759 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
b09f4fd3 2760 spin_unlock_irq(&pwq->pool->lock);
fa2563e4
TT
2761
2762 if (drained)
9c5a2ba7
TH
2763 continue;
2764
2765 if (++flush_cnt == 10 ||
2766 (flush_cnt % 100 == 0 && flush_cnt <= 1000))
c5aa87bb 2767 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
044c782c 2768 wq->name, flush_cnt);
76af4d93 2769
b09f4fd3 2770 mutex_unlock(&wq->mutex);
9c5a2ba7
TH
2771 goto reflush;
2772 }
2773
9c5a2ba7 2774 if (!--wq->nr_drainers)
618b01eb 2775 wq->flags &= ~__WQ_DRAINING;
87fc741e 2776 mutex_unlock(&wq->mutex);
9c5a2ba7
TH
2777}
2778EXPORT_SYMBOL_GPL(drain_workqueue);
2779
606a5020 2780static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
db700897 2781{
affee4b2 2782 struct worker *worker = NULL;
c9e7cf27 2783 struct worker_pool *pool;
112202d9 2784 struct pool_workqueue *pwq;
db700897
ON
2785
2786 might_sleep();
fa1b54e6
TH
2787
2788 local_irq_disable();
c9e7cf27 2789 pool = get_work_pool(work);
fa1b54e6
TH
2790 if (!pool) {
2791 local_irq_enable();
baf59022 2792 return false;
fa1b54e6 2793 }
db700897 2794
fa1b54e6 2795 spin_lock(&pool->lock);
0b3dae68 2796 /* see the comment in try_to_grab_pending() with the same code */
112202d9
TH
2797 pwq = get_work_pwq(work);
2798 if (pwq) {
2799 if (unlikely(pwq->pool != pool))
4690c4ab 2800 goto already_gone;
606a5020 2801 } else {
c9e7cf27 2802 worker = find_worker_executing_work(pool, work);
affee4b2 2803 if (!worker)
4690c4ab 2804 goto already_gone;
112202d9 2805 pwq = worker->current_pwq;
606a5020 2806 }
db700897 2807
fca839c0
TH
2808 check_flush_dependency(pwq->wq, work);
2809
112202d9 2810 insert_wq_barrier(pwq, barr, work, worker);
d565ed63 2811 spin_unlock_irq(&pool->lock);
7a22ad75 2812
e159489b
TH
2813 /*
2814 * If @max_active is 1 or rescuer is in use, flushing another work
2815 * item on the same workqueue may lead to deadlock. Make sure the
2816 * flusher is not running on the same workqueue by verifying write
2817 * access.
2818 */
493008a8 2819 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
112202d9 2820 lock_map_acquire(&pwq->wq->lockdep_map);
e159489b 2821 else
112202d9
TH
2822 lock_map_acquire_read(&pwq->wq->lockdep_map);
2823 lock_map_release(&pwq->wq->lockdep_map);
e159489b 2824
401a8d04 2825 return true;
4690c4ab 2826already_gone:
d565ed63 2827 spin_unlock_irq(&pool->lock);
401a8d04 2828 return false;
db700897 2829}
baf59022
TH
2830
2831/**
2832 * flush_work - wait for a work to finish executing the last queueing instance
2833 * @work: the work to flush
2834 *
606a5020
TH
2835 * Wait until @work has finished execution. @work is guaranteed to be idle
2836 * on return if it hasn't been requeued since flush started.
baf59022 2837 *
d185af30 2838 * Return:
baf59022
TH
2839 * %true if flush_work() waited for the work to finish execution,
2840 * %false if it was already idle.
2841 */
2842bool flush_work(struct work_struct *work)
2843{
12997d1a
BH
2844 struct wq_barrier barr;
2845
0976dfc1
SB
2846 lock_map_acquire(&work->lockdep_map);
2847 lock_map_release(&work->lockdep_map);
2848
12997d1a
BH
2849 if (start_flush_work(work, &barr)) {
2850 wait_for_completion(&barr.done);
2851 destroy_work_on_stack(&barr.work);
2852 return true;
2853 } else {
2854 return false;
2855 }
6e84d644 2856}
606a5020 2857EXPORT_SYMBOL_GPL(flush_work);
6e84d644 2858
8603e1b3
TH
2859struct cwt_wait {
2860 wait_queue_t wait;
2861 struct work_struct *work;
2862};
2863
2864static int cwt_wakefn(wait_queue_t *wait, unsigned mode, int sync, void *key)
2865{
2866 struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
2867
2868 if (cwait->work != key)
2869 return 0;
2870 return autoremove_wake_function(wait, mode, sync, key);
2871}
2872
36e227d2 2873static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
1f1f642e 2874{
8603e1b3 2875 static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
bbb68dfa 2876 unsigned long flags;
1f1f642e
ON
2877 int ret;
2878
2879 do {
bbb68dfa
TH
2880 ret = try_to_grab_pending(work, is_dwork, &flags);
2881 /*
8603e1b3
TH
2882 * If someone else is already canceling, wait for it to
2883 * finish. flush_work() doesn't work for PREEMPT_NONE
2884 * because we may get scheduled between @work's completion
2885 * and the other canceling task resuming and clearing
2886 * CANCELING - flush_work() will return false immediately
2887 * as @work is no longer busy, try_to_grab_pending() will
2888 * return -ENOENT as @work is still being canceled and the
2889 * other canceling task won't be able to clear CANCELING as
2890 * we're hogging the CPU.
2891 *
2892 * Let's wait for completion using a waitqueue. As this
2893 * may lead to the thundering herd problem, use a custom
2894 * wake function which matches @work along with exclusive
2895 * wait and wakeup.
bbb68dfa 2896 */
8603e1b3
TH
2897 if (unlikely(ret == -ENOENT)) {
2898 struct cwt_wait cwait;
2899
2900 init_wait(&cwait.wait);
2901 cwait.wait.func = cwt_wakefn;
2902 cwait.work = work;
2903
2904 prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
2905 TASK_UNINTERRUPTIBLE);
2906 if (work_is_canceling(work))
2907 schedule();
2908 finish_wait(&cancel_waitq, &cwait.wait);
2909 }
1f1f642e
ON
2910 } while (unlikely(ret < 0));
2911
bbb68dfa
TH
2912 /* tell other tasks trying to grab @work to back off */
2913 mark_work_canceling(work);
2914 local_irq_restore(flags);
2915
606a5020 2916 flush_work(work);
7a22ad75 2917 clear_work_data(work);
8603e1b3
TH
2918
2919 /*
2920 * Paired with prepare_to_wait() above so that either
2921 * waitqueue_active() is visible here or !work_is_canceling() is
2922 * visible there.
2923 */
2924 smp_mb();
2925 if (waitqueue_active(&cancel_waitq))
2926 __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
2927
1f1f642e
ON
2928 return ret;
2929}
2930
6e84d644 2931/**
401a8d04
TH
2932 * cancel_work_sync - cancel a work and wait for it to finish
2933 * @work: the work to cancel
6e84d644 2934 *
401a8d04
TH
2935 * Cancel @work and wait for its execution to finish. This function
2936 * can be used even if the work re-queues itself or migrates to
2937 * another workqueue. On return from this function, @work is
2938 * guaranteed to be not pending or executing on any CPU.
1f1f642e 2939 *
401a8d04
TH
2940 * cancel_work_sync(&delayed_work->work) must not be used for
2941 * delayed_work's. Use cancel_delayed_work_sync() instead.
6e84d644 2942 *
401a8d04 2943 * The caller must ensure that the workqueue on which @work was last
6e84d644 2944 * queued can't be destroyed before this function returns.
401a8d04 2945 *
d185af30 2946 * Return:
401a8d04 2947 * %true if @work was pending, %false otherwise.
6e84d644 2948 */
401a8d04 2949bool cancel_work_sync(struct work_struct *work)
6e84d644 2950{
36e227d2 2951 return __cancel_work_timer(work, false);
b89deed3 2952}
28e53bdd 2953EXPORT_SYMBOL_GPL(cancel_work_sync);
b89deed3 2954
6e84d644 2955/**
401a8d04
TH
2956 * flush_delayed_work - wait for a dwork to finish executing the last queueing
2957 * @dwork: the delayed work to flush
6e84d644 2958 *
401a8d04
TH
2959 * Delayed timer is cancelled and the pending work is queued for
2960 * immediate execution. Like flush_work(), this function only
2961 * considers the last queueing instance of @dwork.
1f1f642e 2962 *
d185af30 2963 * Return:
401a8d04
TH
2964 * %true if flush_work() waited for the work to finish execution,
2965 * %false if it was already idle.
6e84d644 2966 */
401a8d04
TH
2967bool flush_delayed_work(struct delayed_work *dwork)
2968{
8930caba 2969 local_irq_disable();
401a8d04 2970 if (del_timer_sync(&dwork->timer))
60c057bc 2971 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
8930caba 2972 local_irq_enable();
401a8d04
TH
2973 return flush_work(&dwork->work);
2974}
2975EXPORT_SYMBOL(flush_delayed_work);
2976
f72b8792
JA
2977static bool __cancel_work(struct work_struct *work, bool is_dwork)
2978{
2979 unsigned long flags;
2980 int ret;
2981
2982 do {
2983 ret = try_to_grab_pending(work, is_dwork, &flags);
2984 } while (unlikely(ret == -EAGAIN));
2985
2986 if (unlikely(ret < 0))
2987 return false;
2988
2989 set_work_pool_and_clear_pending(work, get_work_pool_id(work));
2990 local_irq_restore(flags);
2991 return ret;
2992}
2993
2994/*
2995 * See cancel_delayed_work()
2996 */
2997bool cancel_work(struct work_struct *work)
2998{
2999 return __cancel_work(work, false);
3000}
3001
09383498 3002/**
57b30ae7
TH
3003 * cancel_delayed_work - cancel a delayed work
3004 * @dwork: delayed_work to cancel
09383498 3005 *
d185af30
YB
3006 * Kill off a pending delayed_work.
3007 *
3008 * Return: %true if @dwork was pending and canceled; %false if it wasn't
3009 * pending.
3010 *
3011 * Note:
3012 * The work callback function may still be running on return, unless
3013 * it returns %true and the work doesn't re-arm itself. Explicitly flush or
3014 * use cancel_delayed_work_sync() to wait on it.
09383498 3015 *
57b30ae7 3016 * This function is safe to call from any context including IRQ handler.
09383498 3017 */
57b30ae7 3018bool cancel_delayed_work(struct delayed_work *dwork)
09383498 3019{
f72b8792 3020 return __cancel_work(&dwork->work, true);
09383498 3021}
57b30ae7 3022EXPORT_SYMBOL(cancel_delayed_work);
09383498 3023
401a8d04
TH
3024/**
3025 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3026 * @dwork: the delayed work cancel
3027 *
3028 * This is cancel_work_sync() for delayed works.
3029 *
d185af30 3030 * Return:
401a8d04
TH
3031 * %true if @dwork was pending, %false otherwise.
3032 */
3033bool cancel_delayed_work_sync(struct delayed_work *dwork)
6e84d644 3034{
36e227d2 3035 return __cancel_work_timer(&dwork->work, true);
6e84d644 3036}
f5a421a4 3037EXPORT_SYMBOL(cancel_delayed_work_sync);
1da177e4 3038
b6136773 3039/**
31ddd871 3040 * schedule_on_each_cpu - execute a function synchronously on each online CPU
b6136773 3041 * @func: the function to call
b6136773 3042 *
31ddd871
TH
3043 * schedule_on_each_cpu() executes @func on each online CPU using the
3044 * system workqueue and blocks until all CPUs have completed.
b6136773 3045 * schedule_on_each_cpu() is very slow.
31ddd871 3046 *
d185af30 3047 * Return:
31ddd871 3048 * 0 on success, -errno on failure.
b6136773 3049 */
65f27f38 3050int schedule_on_each_cpu(work_func_t func)
15316ba8
CL
3051{
3052 int cpu;
38f51568 3053 struct work_struct __percpu *works;
15316ba8 3054
b6136773
AM
3055 works = alloc_percpu(struct work_struct);
3056 if (!works)
15316ba8 3057 return -ENOMEM;
b6136773 3058
93981800
TH
3059 get_online_cpus();
3060
15316ba8 3061 for_each_online_cpu(cpu) {
9bfb1839
IM
3062 struct work_struct *work = per_cpu_ptr(works, cpu);
3063
3064 INIT_WORK(work, func);
b71ab8c2 3065 schedule_work_on(cpu, work);
65a64464 3066 }
93981800
TH
3067
3068 for_each_online_cpu(cpu)
3069 flush_work(per_cpu_ptr(works, cpu));
3070
95402b38 3071 put_online_cpus();
b6136773 3072 free_percpu(works);
15316ba8
CL
3073 return 0;
3074}
3075
1fa44eca
JB
3076/**
3077 * execute_in_process_context - reliably execute the routine with user context
3078 * @fn: the function to execute
1fa44eca
JB
3079 * @ew: guaranteed storage for the execute work structure (must
3080 * be available when the work executes)
3081 *
3082 * Executes the function immediately if process context is available,
3083 * otherwise schedules the function for delayed execution.
3084 *
d185af30 3085 * Return: 0 - function was executed
1fa44eca
JB
3086 * 1 - function was scheduled for execution
3087 */
65f27f38 3088int execute_in_process_context(work_func_t fn, struct execute_work *ew)
1fa44eca
JB
3089{
3090 if (!in_interrupt()) {
65f27f38 3091 fn(&ew->work);
1fa44eca
JB
3092 return 0;
3093 }
3094
65f27f38 3095 INIT_WORK(&ew->work, fn);
1fa44eca
JB
3096 schedule_work(&ew->work);
3097
3098 return 1;
3099}
3100EXPORT_SYMBOL_GPL(execute_in_process_context);
3101
6ba94429
FW
3102/**
3103 * free_workqueue_attrs - free a workqueue_attrs
3104 * @attrs: workqueue_attrs to free
226223ab 3105 *
6ba94429 3106 * Undo alloc_workqueue_attrs().
226223ab 3107 */
6ba94429 3108void free_workqueue_attrs(struct workqueue_attrs *attrs)
226223ab 3109{
6ba94429
FW
3110 if (attrs) {
3111 free_cpumask_var(attrs->cpumask);
3112 kfree(attrs);
3113 }
226223ab
TH
3114}
3115
6ba94429
FW
3116/**
3117 * alloc_workqueue_attrs - allocate a workqueue_attrs
3118 * @gfp_mask: allocation mask to use
3119 *
3120 * Allocate a new workqueue_attrs, initialize with default settings and
3121 * return it.
3122 *
3123 * Return: The allocated new workqueue_attr on success. %NULL on failure.
3124 */
3125struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
226223ab 3126{
6ba94429 3127 struct workqueue_attrs *attrs;
226223ab 3128
6ba94429
FW
3129 attrs = kzalloc(sizeof(*attrs), gfp_mask);
3130 if (!attrs)
3131 goto fail;
3132 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3133 goto fail;
3134
3135 cpumask_copy(attrs->cpumask, cpu_possible_mask);
3136 return attrs;
3137fail:
3138 free_workqueue_attrs(attrs);
3139 return NULL;
226223ab
TH
3140}
3141
6ba94429
FW
3142static void copy_workqueue_attrs(struct workqueue_attrs *to,
3143 const struct workqueue_attrs *from)
226223ab 3144{
6ba94429
FW
3145 to->nice = from->nice;
3146 cpumask_copy(to->cpumask, from->cpumask);
3147 /*
3148 * Unlike hash and equality test, this function doesn't ignore
3149 * ->no_numa as it is used for both pool and wq attrs. Instead,
3150 * get_unbound_pool() explicitly clears ->no_numa after copying.
3151 */
3152 to->no_numa = from->no_numa;
226223ab
TH
3153}
3154
6ba94429
FW
3155/* hash value of the content of @attr */
3156static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
226223ab 3157{
6ba94429 3158 u32 hash = 0;
226223ab 3159
6ba94429
FW
3160 hash = jhash_1word(attrs->nice, hash);
3161 hash = jhash(cpumask_bits(attrs->cpumask),
3162 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3163 return hash;
226223ab 3164}
226223ab 3165
6ba94429
FW
3166/* content equality test */
3167static bool wqattrs_equal(const struct workqueue_attrs *a,
3168 const struct workqueue_attrs *b)
226223ab 3169{
6ba94429
FW
3170 if (a->nice != b->nice)
3171 return false;
3172 if (!cpumask_equal(a->cpumask, b->cpumask))
3173 return false;
3174 return true;
226223ab
TH
3175}
3176
6ba94429
FW
3177/**
3178 * init_worker_pool - initialize a newly zalloc'd worker_pool
3179 * @pool: worker_pool to initialize
3180 *
402dd89d 3181 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
6ba94429
FW
3182 *
3183 * Return: 0 on success, -errno on failure. Even on failure, all fields
3184 * inside @pool proper are initialized and put_unbound_pool() can be called
3185 * on @pool safely to release it.
3186 */
3187static int init_worker_pool(struct worker_pool *pool)
226223ab 3188{
6ba94429
FW
3189 spin_lock_init(&pool->lock);
3190 pool->id = -1;
3191 pool->cpu = -1;
3192 pool->node = NUMA_NO_NODE;
3193 pool->flags |= POOL_DISASSOCIATED;
82607adc 3194 pool->watchdog_ts = jiffies;
6ba94429
FW
3195 INIT_LIST_HEAD(&pool->worklist);
3196 INIT_LIST_HEAD(&pool->idle_list);
3197 hash_init(pool->busy_hash);
226223ab 3198
6ba94429
FW
3199 init_timer_deferrable(&pool->idle_timer);
3200 pool->idle_timer.function = idle_worker_timeout;
3201 pool->idle_timer.data = (unsigned long)pool;
226223ab 3202
6ba94429
FW
3203 setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3204 (unsigned long)pool);
226223ab 3205
6ba94429
FW
3206 mutex_init(&pool->manager_arb);
3207 mutex_init(&pool->attach_mutex);
3208 INIT_LIST_HEAD(&pool->workers);
226223ab 3209
6ba94429
FW
3210 ida_init(&pool->worker_ida);
3211 INIT_HLIST_NODE(&pool->hash_node);
3212 pool->refcnt = 1;
226223ab 3213
6ba94429
FW
3214 /* shouldn't fail above this point */
3215 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3216 if (!pool->attrs)
3217 return -ENOMEM;
3218 return 0;
226223ab
TH
3219}
3220
6ba94429 3221static void rcu_free_wq(struct rcu_head *rcu)
226223ab 3222{
6ba94429
FW
3223 struct workqueue_struct *wq =
3224 container_of(rcu, struct workqueue_struct, rcu);
226223ab 3225
6ba94429
FW
3226 if (!(wq->flags & WQ_UNBOUND))
3227 free_percpu(wq->cpu_pwqs);
226223ab 3228 else
6ba94429 3229 free_workqueue_attrs(wq->unbound_attrs);
226223ab 3230
6ba94429
FW
3231 kfree(wq->rescuer);
3232 kfree(wq);
226223ab
TH
3233}
3234
6ba94429 3235static void rcu_free_pool(struct rcu_head *rcu)
226223ab 3236{
6ba94429 3237 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
226223ab 3238
6ba94429
FW
3239 ida_destroy(&pool->worker_ida);
3240 free_workqueue_attrs(pool->attrs);
3241 kfree(pool);
226223ab
TH
3242}
3243
6ba94429
FW
3244/**
3245 * put_unbound_pool - put a worker_pool
3246 * @pool: worker_pool to put
3247 *
3248 * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
3249 * safe manner. get_unbound_pool() calls this function on its failure path
3250 * and this function should be able to release pools which went through,
3251 * successfully or not, init_worker_pool().
3252 *
3253 * Should be called with wq_pool_mutex held.
3254 */
3255static void put_unbound_pool(struct worker_pool *pool)
226223ab 3256{
6ba94429
FW
3257 DECLARE_COMPLETION_ONSTACK(detach_completion);
3258 struct worker *worker;
226223ab 3259
6ba94429 3260 lockdep_assert_held(&wq_pool_mutex);
226223ab 3261
6ba94429
FW
3262 if (--pool->refcnt)
3263 return;
226223ab 3264
6ba94429
FW
3265 /* sanity checks */
3266 if (WARN_ON(!(pool->cpu < 0)) ||
3267 WARN_ON(!list_empty(&pool->worklist)))
3268 return;
226223ab 3269
6ba94429
FW
3270 /* release id and unhash */
3271 if (pool->id >= 0)
3272 idr_remove(&worker_pool_idr, pool->id);
3273 hash_del(&pool->hash_node);
d55262c4 3274
6ba94429
FW
3275 /*
3276 * Become the manager and destroy all workers. Grabbing
3277 * manager_arb prevents @pool's workers from blocking on
3278 * attach_mutex.
3279 */
3280 mutex_lock(&pool->manager_arb);
d55262c4 3281
6ba94429
FW
3282 spin_lock_irq(&pool->lock);
3283 while ((worker = first_idle_worker(pool)))
3284 destroy_worker(worker);
3285 WARN_ON(pool->nr_workers || pool->nr_idle);
3286 spin_unlock_irq(&pool->lock);
d55262c4 3287
6ba94429
FW
3288 mutex_lock(&pool->attach_mutex);
3289 if (!list_empty(&pool->workers))
3290 pool->detach_completion = &detach_completion;
3291 mutex_unlock(&pool->attach_mutex);
226223ab 3292
6ba94429
FW
3293 if (pool->detach_completion)
3294 wait_for_completion(pool->detach_completion);
226223ab 3295
6ba94429 3296 mutex_unlock(&pool->manager_arb);
226223ab 3297
6ba94429
FW
3298 /* shut down the timers */
3299 del_timer_sync(&pool->idle_timer);
3300 del_timer_sync(&pool->mayday_timer);
226223ab 3301
6ba94429
FW
3302 /* sched-RCU protected to allow dereferences from get_work_pool() */
3303 call_rcu_sched(&pool->rcu, rcu_free_pool);
226223ab
TH
3304}
3305
3306/**
6ba94429
FW
3307 * get_unbound_pool - get a worker_pool with the specified attributes
3308 * @attrs: the attributes of the worker_pool to get
226223ab 3309 *
6ba94429
FW
3310 * Obtain a worker_pool which has the same attributes as @attrs, bump the
3311 * reference count and return it. If there already is a matching
3312 * worker_pool, it will be used; otherwise, this function attempts to
3313 * create a new one.
226223ab 3314 *
6ba94429 3315 * Should be called with wq_pool_mutex held.
226223ab 3316 *
6ba94429
FW
3317 * Return: On success, a worker_pool with the same attributes as @attrs.
3318 * On failure, %NULL.
226223ab 3319 */
6ba94429 3320static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
226223ab 3321{
6ba94429
FW
3322 u32 hash = wqattrs_hash(attrs);
3323 struct worker_pool *pool;
3324 int node;
e2273584 3325 int target_node = NUMA_NO_NODE;
226223ab 3326
6ba94429 3327 lockdep_assert_held(&wq_pool_mutex);
226223ab 3328
6ba94429
FW
3329 /* do we already have a matching pool? */
3330 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3331 if (wqattrs_equal(pool->attrs, attrs)) {
3332 pool->refcnt++;
3333 return pool;
3334 }
3335 }
226223ab 3336
e2273584
XP
3337 /* if cpumask is contained inside a NUMA node, we belong to that node */
3338 if (wq_numa_enabled) {
3339 for_each_node(node) {
3340 if (cpumask_subset(attrs->cpumask,
3341 wq_numa_possible_cpumask[node])) {
3342 target_node = node;
3343 break;
3344 }
3345 }
3346 }
3347
6ba94429 3348 /* nope, create a new one */
e2273584 3349 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
6ba94429
FW
3350 if (!pool || init_worker_pool(pool) < 0)
3351 goto fail;
3352
3353 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3354 copy_workqueue_attrs(pool->attrs, attrs);
e2273584 3355 pool->node = target_node;
226223ab
TH
3356
3357 /*
6ba94429
FW
3358 * no_numa isn't a worker_pool attribute, always clear it. See
3359 * 'struct workqueue_attrs' comments for detail.
226223ab 3360 */
6ba94429 3361 pool->attrs->no_numa = false;
226223ab 3362
6ba94429
FW
3363 if (worker_pool_assign_id(pool) < 0)
3364 goto fail;
226223ab 3365
6ba94429
FW
3366 /* create and start the initial worker */
3367 if (!create_worker(pool))
3368 goto fail;
226223ab 3369
6ba94429
FW
3370 /* install */
3371 hash_add(unbound_pool_hash, &pool->hash_node, hash);
226223ab 3372
6ba94429
FW
3373 return pool;
3374fail:
3375 if (pool)
3376 put_unbound_pool(pool);
3377 return NULL;
226223ab 3378}
226223ab 3379
6ba94429 3380static void rcu_free_pwq(struct rcu_head *rcu)
7a4e344c 3381{
6ba94429
FW
3382 kmem_cache_free(pwq_cache,
3383 container_of(rcu, struct pool_workqueue, rcu));
7a4e344c
TH
3384}
3385
6ba94429
FW
3386/*
3387 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3388 * and needs to be destroyed.
7a4e344c 3389 */
6ba94429 3390static void pwq_unbound_release_workfn(struct work_struct *work)
7a4e344c 3391{
6ba94429
FW
3392 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3393 unbound_release_work);
3394 struct workqueue_struct *wq = pwq->wq;
3395 struct worker_pool *pool = pwq->pool;
3396 bool is_last;
7a4e344c 3397
6ba94429
FW
3398 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3399 return;
7a4e344c 3400
6ba94429
FW
3401 mutex_lock(&wq->mutex);
3402 list_del_rcu(&pwq->pwqs_node);
3403 is_last = list_empty(&wq->pwqs);
3404 mutex_unlock(&wq->mutex);
3405
3406 mutex_lock(&wq_pool_mutex);
3407 put_unbound_pool(pool);
3408 mutex_unlock(&wq_pool_mutex);
3409
3410 call_rcu_sched(&pwq->rcu, rcu_free_pwq);
7a4e344c 3411
2865a8fb 3412 /*
6ba94429
FW
3413 * If we're the last pwq going away, @wq is already dead and no one
3414 * is gonna access it anymore. Schedule RCU free.
2865a8fb 3415 */
6ba94429
FW
3416 if (is_last)
3417 call_rcu_sched(&wq->rcu, rcu_free_wq);
29c91e99
TH
3418}
3419
7a4e344c 3420/**
6ba94429
FW
3421 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3422 * @pwq: target pool_workqueue
d185af30 3423 *
6ba94429
FW
3424 * If @pwq isn't freezing, set @pwq->max_active to the associated
3425 * workqueue's saved_max_active and activate delayed work items
3426 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
7a4e344c 3427 */
6ba94429 3428static void pwq_adjust_max_active(struct pool_workqueue *pwq)
4e1a1f9a 3429{
6ba94429
FW
3430 struct workqueue_struct *wq = pwq->wq;
3431 bool freezable = wq->flags & WQ_FREEZABLE;
4e1a1f9a 3432
6ba94429
FW
3433 /* for @wq->saved_max_active */
3434 lockdep_assert_held(&wq->mutex);
4e1a1f9a 3435
6ba94429
FW
3436 /* fast exit for non-freezable wqs */
3437 if (!freezable && pwq->max_active == wq->saved_max_active)
3438 return;
7a4e344c 3439
6ba94429 3440 spin_lock_irq(&pwq->pool->lock);
29c91e99 3441
6ba94429
FW
3442 /*
3443 * During [un]freezing, the caller is responsible for ensuring that
3444 * this function is called at least once after @workqueue_freezing
3445 * is updated and visible.
3446 */
3447 if (!freezable || !workqueue_freezing) {
3448 pwq->max_active = wq->saved_max_active;
4e1a1f9a 3449
6ba94429
FW
3450 while (!list_empty(&pwq->delayed_works) &&
3451 pwq->nr_active < pwq->max_active)
3452 pwq_activate_first_delayed(pwq);
e2dca7ad 3453
6ba94429
FW
3454 /*
3455 * Need to kick a worker after thawed or an unbound wq's
3456 * max_active is bumped. It's a slow path. Do it always.
3457 */
3458 wake_up_worker(pwq->pool);
3459 } else {
3460 pwq->max_active = 0;
3461 }
e2dca7ad 3462
6ba94429 3463 spin_unlock_irq(&pwq->pool->lock);
e2dca7ad
TH
3464}
3465
6ba94429
FW
3466/* initialize newly alloced @pwq which is associated with @wq and @pool */
3467static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3468 struct worker_pool *pool)
29c91e99 3469{
6ba94429 3470 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
29c91e99 3471
6ba94429
FW
3472 memset(pwq, 0, sizeof(*pwq));
3473
3474 pwq->pool = pool;
3475 pwq->wq = wq;
3476 pwq->flush_color = -1;
3477 pwq->refcnt = 1;
3478 INIT_LIST_HEAD(&pwq->delayed_works);
3479 INIT_LIST_HEAD(&pwq->pwqs_node);
3480 INIT_LIST_HEAD(&pwq->mayday_node);
3481 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
29c91e99
TH
3482}
3483
6ba94429
FW
3484/* sync @pwq with the current state of its associated wq and link it */
3485static void link_pwq(struct pool_workqueue *pwq)
29c91e99 3486{
6ba94429 3487 struct workqueue_struct *wq = pwq->wq;
29c91e99 3488
6ba94429 3489 lockdep_assert_held(&wq->mutex);
a892cacc 3490
6ba94429
FW
3491 /* may be called multiple times, ignore if already linked */
3492 if (!list_empty(&pwq->pwqs_node))
29c91e99 3493 return;
29c91e99 3494
6ba94429
FW
3495 /* set the matching work_color */
3496 pwq->work_color = wq->work_color;
29c91e99 3497
6ba94429
FW
3498 /* sync max_active to the current setting */
3499 pwq_adjust_max_active(pwq);
29c91e99 3500
6ba94429
FW
3501 /* link in @pwq */
3502 list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3503}
29c91e99 3504
6ba94429
FW
3505/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3506static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3507 const struct workqueue_attrs *attrs)
3508{
3509 struct worker_pool *pool;
3510 struct pool_workqueue *pwq;
60f5a4bc 3511
6ba94429 3512 lockdep_assert_held(&wq_pool_mutex);
60f5a4bc 3513
6ba94429
FW
3514 pool = get_unbound_pool(attrs);
3515 if (!pool)
3516 return NULL;
60f5a4bc 3517
6ba94429
FW
3518 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3519 if (!pwq) {
3520 put_unbound_pool(pool);
3521 return NULL;
3522 }
29c91e99 3523
6ba94429
FW
3524 init_pwq(pwq, wq, pool);
3525 return pwq;
3526}
29c91e99 3527
29c91e99 3528/**
30186c6f 3529 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
042f7df1 3530 * @attrs: the wq_attrs of the default pwq of the target workqueue
6ba94429
FW
3531 * @node: the target NUMA node
3532 * @cpu_going_down: if >= 0, the CPU to consider as offline
3533 * @cpumask: outarg, the resulting cpumask
29c91e99 3534 *
6ba94429
FW
3535 * Calculate the cpumask a workqueue with @attrs should use on @node. If
3536 * @cpu_going_down is >= 0, that cpu is considered offline during
3537 * calculation. The result is stored in @cpumask.
a892cacc 3538 *
6ba94429
FW
3539 * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3540 * enabled and @node has online CPUs requested by @attrs, the returned
3541 * cpumask is the intersection of the possible CPUs of @node and
3542 * @attrs->cpumask.
d185af30 3543 *
6ba94429
FW
3544 * The caller is responsible for ensuring that the cpumask of @node stays
3545 * stable.
3546 *
3547 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3548 * %false if equal.
29c91e99 3549 */
6ba94429
FW
3550static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3551 int cpu_going_down, cpumask_t *cpumask)
29c91e99 3552{
6ba94429
FW
3553 if (!wq_numa_enabled || attrs->no_numa)
3554 goto use_dfl;
29c91e99 3555
6ba94429
FW
3556 /* does @node have any online CPUs @attrs wants? */
3557 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3558 if (cpu_going_down >= 0)
3559 cpumask_clear_cpu(cpu_going_down, cpumask);
29c91e99 3560
6ba94429
FW
3561 if (cpumask_empty(cpumask))
3562 goto use_dfl;
4c16bd32
TH
3563
3564 /* yeap, return possible CPUs in @node that @attrs wants */
3565 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3566 return !cpumask_equal(cpumask, attrs->cpumask);
3567
3568use_dfl:
3569 cpumask_copy(cpumask, attrs->cpumask);
3570 return false;
3571}
3572
1befcf30
TH
3573/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3574static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3575 int node,
3576 struct pool_workqueue *pwq)
3577{
3578 struct pool_workqueue *old_pwq;
3579
5b95e1af 3580 lockdep_assert_held(&wq_pool_mutex);
1befcf30
TH
3581 lockdep_assert_held(&wq->mutex);
3582
3583 /* link_pwq() can handle duplicate calls */
3584 link_pwq(pwq);
3585
3586 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3587 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3588 return old_pwq;
3589}
3590
2d5f0764
LJ
3591/* context to store the prepared attrs & pwqs before applying */
3592struct apply_wqattrs_ctx {
3593 struct workqueue_struct *wq; /* target workqueue */
3594 struct workqueue_attrs *attrs; /* attrs to apply */
042f7df1 3595 struct list_head list; /* queued for batching commit */
2d5f0764
LJ
3596 struct pool_workqueue *dfl_pwq;
3597 struct pool_workqueue *pwq_tbl[];
3598};
3599
3600/* free the resources after success or abort */
3601static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
3602{
3603 if (ctx) {
3604 int node;
3605
3606 for_each_node(node)
3607 put_pwq_unlocked(ctx->pwq_tbl[node]);
3608 put_pwq_unlocked(ctx->dfl_pwq);
3609
3610 free_workqueue_attrs(ctx->attrs);
3611
3612 kfree(ctx);
3613 }
3614}
3615
3616/* allocate the attrs and pwqs for later installation */
3617static struct apply_wqattrs_ctx *
3618apply_wqattrs_prepare(struct workqueue_struct *wq,
3619 const struct workqueue_attrs *attrs)
9e8cd2f5 3620{
2d5f0764 3621 struct apply_wqattrs_ctx *ctx;
4c16bd32 3622 struct workqueue_attrs *new_attrs, *tmp_attrs;
2d5f0764 3623 int node;
9e8cd2f5 3624
2d5f0764 3625 lockdep_assert_held(&wq_pool_mutex);
9e8cd2f5 3626
2d5f0764
LJ
3627 ctx = kzalloc(sizeof(*ctx) + nr_node_ids * sizeof(ctx->pwq_tbl[0]),
3628 GFP_KERNEL);
8719dcea 3629
13e2e556 3630 new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4c16bd32 3631 tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
2d5f0764
LJ
3632 if (!ctx || !new_attrs || !tmp_attrs)
3633 goto out_free;
13e2e556 3634
042f7df1
LJ
3635 /*
3636 * Calculate the attrs of the default pwq.
3637 * If the user configured cpumask doesn't overlap with the
3638 * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
3639 */
13e2e556 3640 copy_workqueue_attrs(new_attrs, attrs);
b05a7928 3641 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);
042f7df1
LJ
3642 if (unlikely(cpumask_empty(new_attrs->cpumask)))
3643 cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask);
13e2e556 3644
4c16bd32
TH
3645 /*
3646 * We may create multiple pwqs with differing cpumasks. Make a
3647 * copy of @new_attrs which will be modified and used to obtain
3648 * pools.
3649 */
3650 copy_workqueue_attrs(tmp_attrs, new_attrs);
3651
4c16bd32
TH
3652 /*
3653 * If something goes wrong during CPU up/down, we'll fall back to
3654 * the default pwq covering whole @attrs->cpumask. Always create
3655 * it even if we don't use it immediately.
3656 */
2d5f0764
LJ
3657 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3658 if (!ctx->dfl_pwq)
3659 goto out_free;
4c16bd32
TH
3660
3661 for_each_node(node) {
042f7df1 3662 if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
2d5f0764
LJ
3663 ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3664 if (!ctx->pwq_tbl[node])
3665 goto out_free;
4c16bd32 3666 } else {
2d5f0764
LJ
3667 ctx->dfl_pwq->refcnt++;
3668 ctx->pwq_tbl[node] = ctx->dfl_pwq;
4c16bd32
TH
3669 }
3670 }
3671
042f7df1
LJ
3672 /* save the user configured attrs and sanitize it. */
3673 copy_workqueue_attrs(new_attrs, attrs);
3674 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
2d5f0764 3675 ctx->attrs = new_attrs;
042f7df1 3676
2d5f0764
LJ
3677 ctx->wq = wq;
3678 free_workqueue_attrs(tmp_attrs);
3679 return ctx;
3680
3681out_free:
3682 free_workqueue_attrs(tmp_attrs);
3683 free_workqueue_attrs(new_attrs);
3684 apply_wqattrs_cleanup(ctx);
3685 return NULL;
3686}
3687
3688/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
3689static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
3690{
3691 int node;
9e8cd2f5 3692
4c16bd32 3693 /* all pwqs have been created successfully, let's install'em */
2d5f0764 3694 mutex_lock(&ctx->wq->mutex);
a892cacc 3695
2d5f0764 3696 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4c16bd32
TH
3697
3698 /* save the previous pwq and install the new one */
f147f29e 3699 for_each_node(node)
2d5f0764
LJ
3700 ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
3701 ctx->pwq_tbl[node]);
4c16bd32
TH
3702
3703 /* @dfl_pwq might not have been used, ensure it's linked */
2d5f0764
LJ
3704 link_pwq(ctx->dfl_pwq);
3705 swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
f147f29e 3706
2d5f0764
LJ
3707 mutex_unlock(&ctx->wq->mutex);
3708}
9e8cd2f5 3709
a0111cf6
LJ
3710static void apply_wqattrs_lock(void)
3711{
3712 /* CPUs should stay stable across pwq creations and installations */
3713 get_online_cpus();
3714 mutex_lock(&wq_pool_mutex);
3715}
3716
3717static void apply_wqattrs_unlock(void)
3718{
3719 mutex_unlock(&wq_pool_mutex);
3720 put_online_cpus();
3721}
3722
3723static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
3724 const struct workqueue_attrs *attrs)
2d5f0764
LJ
3725{
3726 struct apply_wqattrs_ctx *ctx;
4c16bd32 3727
2d5f0764
LJ
3728 /* only unbound workqueues can change attributes */
3729 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3730 return -EINVAL;
13e2e556 3731
2d5f0764
LJ
3732 /* creating multiple pwqs breaks ordering guarantee */
3733 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3734 return -EINVAL;
3735
2d5f0764 3736 ctx = apply_wqattrs_prepare(wq, attrs);
6201171e 3737 if (!ctx)
3738 return -ENOMEM;
2d5f0764
LJ
3739
3740 /* the ctx has been prepared successfully, let's commit it */
6201171e 3741 apply_wqattrs_commit(ctx);
2d5f0764
LJ
3742 apply_wqattrs_cleanup(ctx);
3743
6201171e 3744 return 0;
9e8cd2f5
TH
3745}
3746
a0111cf6
LJ
3747/**
3748 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3749 * @wq: the target workqueue
3750 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3751 *
3752 * Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
3753 * machines, this function maps a separate pwq to each NUMA node with
3754 * possibles CPUs in @attrs->cpumask so that work items are affine to the
3755 * NUMA node it was issued on. Older pwqs are released as in-flight work
3756 * items finish. Note that a work item which repeatedly requeues itself
3757 * back-to-back will stay on its current pwq.
3758 *
3759 * Performs GFP_KERNEL allocations.
3760 *
3761 * Return: 0 on success and -errno on failure.
3762 */
3763int apply_workqueue_attrs(struct workqueue_struct *wq,
3764 const struct workqueue_attrs *attrs)
3765{
3766 int ret;
3767
3768 apply_wqattrs_lock();
3769 ret = apply_workqueue_attrs_locked(wq, attrs);
3770 apply_wqattrs_unlock();
3771
3772 return ret;
3773}
3774
4c16bd32
TH
3775/**
3776 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
3777 * @wq: the target workqueue
3778 * @cpu: the CPU coming up or going down
3779 * @online: whether @cpu is coming up or going down
3780 *
3781 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
3782 * %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
3783 * @wq accordingly.
3784 *
3785 * If NUMA affinity can't be adjusted due to memory allocation failure, it
3786 * falls back to @wq->dfl_pwq which may not be optimal but is always
3787 * correct.
3788 *
3789 * Note that when the last allowed CPU of a NUMA node goes offline for a
3790 * workqueue with a cpumask spanning multiple nodes, the workers which were
3791 * already executing the work items for the workqueue will lose their CPU
3792 * affinity and may execute on any CPU. This is similar to how per-cpu
3793 * workqueues behave on CPU_DOWN. If a workqueue user wants strict
3794 * affinity, it's the user's responsibility to flush the work item from
3795 * CPU_DOWN_PREPARE.
3796 */
3797static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
3798 bool online)
3799{
3800 int node = cpu_to_node(cpu);
3801 int cpu_off = online ? -1 : cpu;
3802 struct pool_workqueue *old_pwq = NULL, *pwq;
3803 struct workqueue_attrs *target_attrs;
3804 cpumask_t *cpumask;
3805
3806 lockdep_assert_held(&wq_pool_mutex);
3807
f7142ed4
LJ
3808 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||
3809 wq->unbound_attrs->no_numa)
4c16bd32
TH
3810 return;
3811
3812 /*
3813 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
3814 * Let's use a preallocated one. The following buf is protected by
3815 * CPU hotplug exclusion.
3816 */
3817 target_attrs = wq_update_unbound_numa_attrs_buf;
3818 cpumask = target_attrs->cpumask;
3819
4c16bd32
TH
3820 copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
3821 pwq = unbound_pwq_by_node(wq, node);
3822
3823 /*
3824 * Let's determine what needs to be done. If the target cpumask is
042f7df1
LJ
3825 * different from the default pwq's, we need to compare it to @pwq's
3826 * and create a new one if they don't match. If the target cpumask
3827 * equals the default pwq's, the default pwq should be used.
4c16bd32 3828 */
042f7df1 3829 if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
4c16bd32 3830 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
f7142ed4 3831 return;
4c16bd32 3832 } else {
534a3fbb 3833 goto use_dfl_pwq;
4c16bd32
TH
3834 }
3835
4c16bd32
TH
3836 /* create a new pwq */
3837 pwq = alloc_unbound_pwq(wq, target_attrs);
3838 if (!pwq) {
2d916033
FF
3839 pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
3840 wq->name);
77f300b1 3841 goto use_dfl_pwq;
4c16bd32
TH
3842 }
3843
f7142ed4 3844 /* Install the new pwq. */
4c16bd32
TH
3845 mutex_lock(&wq->mutex);
3846 old_pwq = numa_pwq_tbl_install(wq, node, pwq);
3847 goto out_unlock;
3848
3849use_dfl_pwq:
f7142ed4 3850 mutex_lock(&wq->mutex);
4c16bd32
TH
3851 spin_lock_irq(&wq->dfl_pwq->pool->lock);
3852 get_pwq(wq->dfl_pwq);
3853 spin_unlock_irq(&wq->dfl_pwq->pool->lock);
3854 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
3855out_unlock:
3856 mutex_unlock(&wq->mutex);
3857 put_pwq_unlocked(old_pwq);
3858}
3859
30cdf249 3860static int alloc_and_link_pwqs(struct workqueue_struct *wq)
0f900049 3861{
49e3cf44 3862 bool highpri = wq->flags & WQ_HIGHPRI;
8a2b7538 3863 int cpu, ret;
30cdf249
TH
3864
3865 if (!(wq->flags & WQ_UNBOUND)) {
420c0ddb
TH
3866 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
3867 if (!wq->cpu_pwqs)
30cdf249
TH
3868 return -ENOMEM;
3869
3870 for_each_possible_cpu(cpu) {
7fb98ea7
TH
3871 struct pool_workqueue *pwq =
3872 per_cpu_ptr(wq->cpu_pwqs, cpu);
7a62c2c8 3873 struct worker_pool *cpu_pools =
f02ae73a 3874 per_cpu(cpu_worker_pools, cpu);
f3421797 3875
f147f29e
TH
3876 init_pwq(pwq, wq, &cpu_pools[highpri]);
3877
3878 mutex_lock(&wq->mutex);
1befcf30 3879 link_pwq(pwq);
f147f29e 3880 mutex_unlock(&wq->mutex);
30cdf249 3881 }
9e8cd2f5 3882 return 0;
8a2b7538
TH
3883 } else if (wq->flags & __WQ_ORDERED) {
3884 ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
3885 /* there should only be single pwq for ordering guarantee */
3886 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
3887 wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
3888 "ordering guarantee broken for workqueue %s\n", wq->name);
3889 return ret;
30cdf249 3890 } else {
9e8cd2f5 3891 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
30cdf249 3892 }
0f900049
TH
3893}
3894
f3421797
TH
3895static int wq_clamp_max_active(int max_active, unsigned int flags,
3896 const char *name)
b71ab8c2 3897{
f3421797
TH
3898 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
3899
3900 if (max_active < 1 || max_active > lim)
044c782c
VI
3901 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
3902 max_active, name, 1, lim);
b71ab8c2 3903
f3421797 3904 return clamp_val(max_active, 1, lim);
b71ab8c2
TH
3905}
3906
b196be89 3907struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
d320c038
TH
3908 unsigned int flags,
3909 int max_active,
3910 struct lock_class_key *key,
b196be89 3911 const char *lock_name, ...)
1da177e4 3912{
df2d5ae4 3913 size_t tbl_size = 0;
ecf6881f 3914 va_list args;
1da177e4 3915 struct workqueue_struct *wq;
49e3cf44 3916 struct pool_workqueue *pwq;
b196be89 3917
cee22a15
VK
3918 /* see the comment above the definition of WQ_POWER_EFFICIENT */
3919 if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
3920 flags |= WQ_UNBOUND;
3921
ecf6881f 3922 /* allocate wq and format name */
df2d5ae4 3923 if (flags & WQ_UNBOUND)
ddcb57e2 3924 tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
df2d5ae4
TH
3925
3926 wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
b196be89 3927 if (!wq)
d2c1d404 3928 return NULL;
b196be89 3929
6029a918
TH
3930 if (flags & WQ_UNBOUND) {
3931 wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3932 if (!wq->unbound_attrs)
3933 goto err_free_wq;
3934 }
3935
ecf6881f
TH
3936 va_start(args, lock_name);
3937 vsnprintf(wq->name, sizeof(wq->name), fmt, args);
b196be89 3938 va_end(args);
1da177e4 3939
d320c038 3940 max_active = max_active ?: WQ_DFL_ACTIVE;
b196be89 3941 max_active = wq_clamp_max_active(max_active, flags, wq->name);
3af24433 3942
b196be89 3943 /* init wq */
97e37d7b 3944 wq->flags = flags;
a0a1a5fd 3945 wq->saved_max_active = max_active;
3c25a55d 3946 mutex_init(&wq->mutex);
112202d9 3947 atomic_set(&wq->nr_pwqs_to_flush, 0);
30cdf249 3948 INIT_LIST_HEAD(&wq->pwqs);
73f53c4a
TH
3949 INIT_LIST_HEAD(&wq->flusher_queue);
3950 INIT_LIST_HEAD(&wq->flusher_overflow);
493a1724 3951 INIT_LIST_HEAD(&wq->maydays);
502ca9d8 3952
eb13ba87 3953 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
cce1a165 3954 INIT_LIST_HEAD(&wq->list);
3af24433 3955
30cdf249 3956 if (alloc_and_link_pwqs(wq) < 0)
d2c1d404 3957 goto err_free_wq;
1537663f 3958
493008a8
TH
3959 /*
3960 * Workqueues which may be used during memory reclaim should
3961 * have a rescuer to guarantee forward progress.
3962 */
3963 if (flags & WQ_MEM_RECLAIM) {
e22bee78
TH
3964 struct worker *rescuer;
3965
f7537df5 3966 rescuer = alloc_worker(NUMA_NO_NODE);
e22bee78 3967 if (!rescuer)
d2c1d404 3968 goto err_destroy;
e22bee78 3969
111c225a
TH
3970 rescuer->rescue_wq = wq;
3971 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
b196be89 3972 wq->name);
d2c1d404
TH
3973 if (IS_ERR(rescuer->task)) {
3974 kfree(rescuer);
3975 goto err_destroy;
3976 }
e22bee78 3977
d2c1d404 3978 wq->rescuer = rescuer;
25834c73 3979 kthread_bind_mask(rescuer->task, cpu_possible_mask);
e22bee78 3980 wake_up_process(rescuer->task);
3af24433
ON
3981 }
3982
226223ab
TH
3983 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
3984 goto err_destroy;
3985
a0a1a5fd 3986 /*
68e13a67
LJ
3987 * wq_pool_mutex protects global freeze state and workqueues list.
3988 * Grab it, adjust max_active and add the new @wq to workqueues
3989 * list.
a0a1a5fd 3990 */
68e13a67 3991 mutex_lock(&wq_pool_mutex);
a0a1a5fd 3992
a357fc03 3993 mutex_lock(&wq->mutex);
699ce097
TH
3994 for_each_pwq(pwq, wq)
3995 pwq_adjust_max_active(pwq);
a357fc03 3996 mutex_unlock(&wq->mutex);
a0a1a5fd 3997
e2dca7ad 3998 list_add_tail_rcu(&wq->list, &workqueues);
a0a1a5fd 3999
68e13a67 4000 mutex_unlock(&wq_pool_mutex);
1537663f 4001
3af24433 4002 return wq;
d2c1d404
TH
4003
4004err_free_wq:
6029a918 4005 free_workqueue_attrs(wq->unbound_attrs);
d2c1d404
TH
4006 kfree(wq);
4007 return NULL;
4008err_destroy:
4009 destroy_workqueue(wq);
4690c4ab 4010 return NULL;
3af24433 4011}
d320c038 4012EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
1da177e4 4013
3af24433
ON
4014/**
4015 * destroy_workqueue - safely terminate a workqueue
4016 * @wq: target workqueue
4017 *
4018 * Safely destroy a workqueue. All work currently pending will be done first.
4019 */
4020void destroy_workqueue(struct workqueue_struct *wq)
4021{
49e3cf44 4022 struct pool_workqueue *pwq;
4c16bd32 4023 int node;
3af24433 4024
9c5a2ba7
TH
4025 /* drain it before proceeding with destruction */
4026 drain_workqueue(wq);
c8efcc25 4027
6183c009 4028 /* sanity checks */
b09f4fd3 4029 mutex_lock(&wq->mutex);
49e3cf44 4030 for_each_pwq(pwq, wq) {
6183c009
TH
4031 int i;
4032
76af4d93
TH
4033 for (i = 0; i < WORK_NR_COLORS; i++) {
4034 if (WARN_ON(pwq->nr_in_flight[i])) {
b09f4fd3 4035 mutex_unlock(&wq->mutex);
6183c009 4036 return;
76af4d93
TH
4037 }
4038 }
4039
5c529597 4040 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
8864b4e5 4041 WARN_ON(pwq->nr_active) ||
76af4d93 4042 WARN_ON(!list_empty(&pwq->delayed_works))) {
b09f4fd3 4043 mutex_unlock(&wq->mutex);
6183c009 4044 return;
76af4d93 4045 }
6183c009 4046 }
b09f4fd3 4047 mutex_unlock(&wq->mutex);
6183c009 4048
a0a1a5fd
TH
4049 /*
4050 * wq list is used to freeze wq, remove from list after
4051 * flushing is complete in case freeze races us.
4052 */
68e13a67 4053 mutex_lock(&wq_pool_mutex);
e2dca7ad 4054 list_del_rcu(&wq->list);
68e13a67 4055 mutex_unlock(&wq_pool_mutex);
3af24433 4056
226223ab
TH
4057 workqueue_sysfs_unregister(wq);
4058
e2dca7ad 4059 if (wq->rescuer)
e22bee78 4060 kthread_stop(wq->rescuer->task);
e22bee78 4061
8864b4e5
TH
4062 if (!(wq->flags & WQ_UNBOUND)) {
4063 /*
4064 * The base ref is never dropped on per-cpu pwqs. Directly
e2dca7ad 4065 * schedule RCU free.
8864b4e5 4066 */
e2dca7ad 4067 call_rcu_sched(&wq->rcu, rcu_free_wq);
8864b4e5
TH
4068 } else {
4069 /*
4070 * We're the sole accessor of @wq at this point. Directly
4c16bd32
TH
4071 * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4072 * @wq will be freed when the last pwq is released.
8864b4e5 4073 */
4c16bd32
TH
4074 for_each_node(node) {
4075 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4076 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4077 put_pwq_unlocked(pwq);
4078 }
4079
4080 /*
4081 * Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4082 * put. Don't access it afterwards.
4083 */
4084 pwq = wq->dfl_pwq;
4085 wq->dfl_pwq = NULL;
dce90d47 4086 put_pwq_unlocked(pwq);
29c91e99 4087 }
3af24433
ON
4088}
4089EXPORT_SYMBOL_GPL(destroy_workqueue);
4090
dcd989cb
TH
4091/**
4092 * workqueue_set_max_active - adjust max_active of a workqueue
4093 * @wq: target workqueue
4094 * @max_active: new max_active value.
4095 *
4096 * Set max_active of @wq to @max_active.
4097 *
4098 * CONTEXT:
4099 * Don't call from IRQ context.
4100 */
4101void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4102{
49e3cf44 4103 struct pool_workqueue *pwq;
dcd989cb 4104
8719dcea
TH
4105 /* disallow meddling with max_active for ordered workqueues */
4106 if (WARN_ON(wq->flags & __WQ_ORDERED))
4107 return;
4108
f3421797 4109 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
dcd989cb 4110
a357fc03 4111 mutex_lock(&wq->mutex);
dcd989cb
TH
4112
4113 wq->saved_max_active = max_active;
4114
699ce097
TH
4115 for_each_pwq(pwq, wq)
4116 pwq_adjust_max_active(pwq);
93981800 4117
a357fc03 4118 mutex_unlock(&wq->mutex);
15316ba8 4119}
dcd989cb 4120EXPORT_SYMBOL_GPL(workqueue_set_max_active);
15316ba8 4121
e6267616
TH
4122/**
4123 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4124 *
4125 * Determine whether %current is a workqueue rescuer. Can be used from
4126 * work functions to determine whether it's being run off the rescuer task.
d185af30
YB
4127 *
4128 * Return: %true if %current is a workqueue rescuer. %false otherwise.
e6267616
TH
4129 */
4130bool current_is_workqueue_rescuer(void)
4131{
4132 struct worker *worker = current_wq_worker();
4133
6a092dfd 4134 return worker && worker->rescue_wq;
e6267616
TH
4135}
4136
eef6a7d5 4137/**
dcd989cb
TH
4138 * workqueue_congested - test whether a workqueue is congested
4139 * @cpu: CPU in question
4140 * @wq: target workqueue
eef6a7d5 4141 *
dcd989cb
TH
4142 * Test whether @wq's cpu workqueue for @cpu is congested. There is
4143 * no synchronization around this function and the test result is
4144 * unreliable and only useful as advisory hints or for debugging.
eef6a7d5 4145 *
d3251859
TH
4146 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4147 * Note that both per-cpu and unbound workqueues may be associated with
4148 * multiple pool_workqueues which have separate congested states. A
4149 * workqueue being congested on one CPU doesn't mean the workqueue is also
4150 * contested on other CPUs / NUMA nodes.
4151 *
d185af30 4152 * Return:
dcd989cb 4153 * %true if congested, %false otherwise.
eef6a7d5 4154 */
d84ff051 4155bool workqueue_congested(int cpu, struct workqueue_struct *wq)
1da177e4 4156{
7fb98ea7 4157 struct pool_workqueue *pwq;
76af4d93
TH
4158 bool ret;
4159
88109453 4160 rcu_read_lock_sched();
7fb98ea7 4161
d3251859
TH
4162 if (cpu == WORK_CPU_UNBOUND)
4163 cpu = smp_processor_id();
4164
7fb98ea7
TH
4165 if (!(wq->flags & WQ_UNBOUND))
4166 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4167 else
df2d5ae4 4168 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
dcd989cb 4169
76af4d93 4170 ret = !list_empty(&pwq->delayed_works);
88109453 4171 rcu_read_unlock_sched();
76af4d93
TH
4172
4173 return ret;
1da177e4 4174}
dcd989cb 4175EXPORT_SYMBOL_GPL(workqueue_congested);
1da177e4 4176
dcd989cb
TH
4177/**
4178 * work_busy - test whether a work is currently pending or running
4179 * @work: the work to be tested
4180 *
4181 * Test whether @work is currently pending or running. There is no
4182 * synchronization around this function and the test result is
4183 * unreliable and only useful as advisory hints or for debugging.
dcd989cb 4184 *
d185af30 4185 * Return:
dcd989cb
TH
4186 * OR'd bitmask of WORK_BUSY_* bits.
4187 */
4188unsigned int work_busy(struct work_struct *work)
1da177e4 4189{
fa1b54e6 4190 struct worker_pool *pool;
dcd989cb
TH
4191 unsigned long flags;
4192 unsigned int ret = 0;
1da177e4 4193
dcd989cb
TH
4194 if (work_pending(work))
4195 ret |= WORK_BUSY_PENDING;
1da177e4 4196
fa1b54e6
TH
4197 local_irq_save(flags);
4198 pool = get_work_pool(work);
038366c5 4199 if (pool) {
fa1b54e6 4200 spin_lock(&pool->lock);
038366c5
LJ
4201 if (find_worker_executing_work(pool, work))
4202 ret |= WORK_BUSY_RUNNING;
fa1b54e6 4203 spin_unlock(&pool->lock);
038366c5 4204 }
fa1b54e6 4205 local_irq_restore(flags);
1da177e4 4206
dcd989cb 4207 return ret;
1da177e4 4208}
dcd989cb 4209EXPORT_SYMBOL_GPL(work_busy);
1da177e4 4210
3d1cb205
TH
4211/**
4212 * set_worker_desc - set description for the current work item
4213 * @fmt: printf-style format string
4214 * @...: arguments for the format string
4215 *
4216 * This function can be called by a running work function to describe what
4217 * the work item is about. If the worker task gets dumped, this
4218 * information will be printed out together to help debugging. The
4219 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
4220 */
4221void set_worker_desc(const char *fmt, ...)
4222{
4223 struct worker *worker = current_wq_worker();
4224 va_list args;
4225
4226 if (worker) {
4227 va_start(args, fmt);
4228 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4229 va_end(args);
4230 worker->desc_valid = true;
4231 }
4232}
4233
4234/**
4235 * print_worker_info - print out worker information and description
4236 * @log_lvl: the log level to use when printing
4237 * @task: target task
4238 *
4239 * If @task is a worker and currently executing a work item, print out the
4240 * name of the workqueue being serviced and worker description set with
4241 * set_worker_desc() by the currently executing work item.
4242 *
4243 * This function can be safely called on any task as long as the
4244 * task_struct itself is accessible. While safe, this function isn't
4245 * synchronized and may print out mixups or garbages of limited length.
4246 */
4247void print_worker_info(const char *log_lvl, struct task_struct *task)
4248{
4249 work_func_t *fn = NULL;
4250 char name[WQ_NAME_LEN] = { };
4251 char desc[WORKER_DESC_LEN] = { };
4252 struct pool_workqueue *pwq = NULL;
4253 struct workqueue_struct *wq = NULL;
4254 bool desc_valid = false;
4255 struct worker *worker;
4256
4257 if (!(task->flags & PF_WQ_WORKER))
4258 return;
4259
4260 /*
4261 * This function is called without any synchronization and @task
4262 * could be in any state. Be careful with dereferences.
4263 */
e700591a 4264 worker = kthread_probe_data(task);
3d1cb205
TH
4265
4266 /*
4267 * Carefully copy the associated workqueue's workfn and name. Keep
4268 * the original last '\0' in case the original contains garbage.
4269 */
4270 probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4271 probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4272 probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4273 probe_kernel_read(name, wq->name, sizeof(name) - 1);
4274
4275 /* copy worker description */
4276 probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4277 if (desc_valid)
4278 probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4279
4280 if (fn || name[0] || desc[0]) {
4281 printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4282 if (desc[0])
4283 pr_cont(" (%s)", desc);
4284 pr_cont("\n");
4285 }
4286}
4287
3494fc30
TH
4288static void pr_cont_pool_info(struct worker_pool *pool)
4289{
4290 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4291 if (pool->node != NUMA_NO_NODE)
4292 pr_cont(" node=%d", pool->node);
4293 pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4294}
4295
4296static void pr_cont_work(bool comma, struct work_struct *work)
4297{
4298 if (work->func == wq_barrier_func) {
4299 struct wq_barrier *barr;
4300
4301 barr = container_of(work, struct wq_barrier, work);
4302
4303 pr_cont("%s BAR(%d)", comma ? "," : "",
4304 task_pid_nr(barr->task));
4305 } else {
4306 pr_cont("%s %pf", comma ? "," : "", work->func);
4307 }
4308}
4309
4310static void show_pwq(struct pool_workqueue *pwq)
4311{
4312 struct worker_pool *pool = pwq->pool;
4313 struct work_struct *work;
4314 struct worker *worker;
4315 bool has_in_flight = false, has_pending = false;
4316 int bkt;
4317
4318 pr_info(" pwq %d:", pool->id);
4319 pr_cont_pool_info(pool);
4320
4321 pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active,
4322 !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4323
4324 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4325 if (worker->current_pwq == pwq) {
4326 has_in_flight = true;
4327 break;
4328 }
4329 }
4330 if (has_in_flight) {
4331 bool comma = false;
4332
4333 pr_info(" in-flight:");
4334 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4335 if (worker->current_pwq != pwq)
4336 continue;
4337
4338 pr_cont("%s %d%s:%pf", comma ? "," : "",
4339 task_pid_nr(worker->task),
4340 worker == pwq->wq->rescuer ? "(RESCUER)" : "",
4341 worker->current_func);
4342 list_for_each_entry(work, &worker->scheduled, entry)
4343 pr_cont_work(false, work);
4344 comma = true;
4345 }
4346 pr_cont("\n");
4347 }
4348
4349 list_for_each_entry(work, &pool->worklist, entry) {
4350 if (get_work_pwq(work) == pwq) {
4351 has_pending = true;
4352 break;
4353 }
4354 }
4355 if (has_pending) {
4356 bool comma = false;
4357
4358 pr_info(" pending:");
4359 list_for_each_entry(work, &pool->worklist, entry) {
4360 if (get_work_pwq(work) != pwq)
4361 continue;
4362
4363 pr_cont_work(comma, work);
4364 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4365 }
4366 pr_cont("\n");
4367 }
4368
4369 if (!list_empty(&pwq->delayed_works)) {
4370 bool comma = false;
4371
4372 pr_info(" delayed:");
4373 list_for_each_entry(work, &pwq->delayed_works, entry) {
4374 pr_cont_work(comma, work);
4375 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4376 }
4377 pr_cont("\n");
4378 }
4379}
4380
4381/**
4382 * show_workqueue_state - dump workqueue state
4383 *
7b776af6
RL
4384 * Called from a sysrq handler or try_to_freeze_tasks() and prints out
4385 * all busy workqueues and pools.
3494fc30
TH
4386 */
4387void show_workqueue_state(void)
4388{
4389 struct workqueue_struct *wq;
4390 struct worker_pool *pool;
4391 unsigned long flags;
4392 int pi;
4393
4394 rcu_read_lock_sched();
4395
4396 pr_info("Showing busy workqueues and worker pools:\n");
4397
4398 list_for_each_entry_rcu(wq, &workqueues, list) {
4399 struct pool_workqueue *pwq;
4400 bool idle = true;
4401
4402 for_each_pwq(pwq, wq) {
4403 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) {
4404 idle = false;
4405 break;
4406 }
4407 }
4408 if (idle)
4409 continue;
4410
4411 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4412
4413 for_each_pwq(pwq, wq) {
4414 spin_lock_irqsave(&pwq->pool->lock, flags);
4415 if (pwq->nr_active || !list_empty(&pwq->delayed_works))
4416 show_pwq(pwq);
4417 spin_unlock_irqrestore(&pwq->pool->lock, flags);
4418 }
4419 }
4420
4421 for_each_pool(pool, pi) {
4422 struct worker *worker;
4423 bool first = true;
4424
4425 spin_lock_irqsave(&pool->lock, flags);
4426 if (pool->nr_workers == pool->nr_idle)
4427 goto next_pool;
4428
4429 pr_info("pool %d:", pool->id);
4430 pr_cont_pool_info(pool);
82607adc
TH
4431 pr_cont(" hung=%us workers=%d",
4432 jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,
4433 pool->nr_workers);
3494fc30
TH
4434 if (pool->manager)
4435 pr_cont(" manager: %d",
4436 task_pid_nr(pool->manager->task));
4437 list_for_each_entry(worker, &pool->idle_list, entry) {
4438 pr_cont(" %s%d", first ? "idle: " : "",
4439 task_pid_nr(worker->task));
4440 first = false;
4441 }
4442 pr_cont("\n");
4443 next_pool:
4444 spin_unlock_irqrestore(&pool->lock, flags);
4445 }
4446
4447 rcu_read_unlock_sched();
4448}
4449
db7bccf4
TH
4450/*
4451 * CPU hotplug.
4452 *
e22bee78 4453 * There are two challenges in supporting CPU hotplug. Firstly, there
112202d9 4454 * are a lot of assumptions on strong associations among work, pwq and
706026c2 4455 * pool which make migrating pending and scheduled works very
e22bee78 4456 * difficult to implement without impacting hot paths. Secondly,
94cf58bb 4457 * worker pools serve mix of short, long and very long running works making
e22bee78
TH
4458 * blocked draining impractical.
4459 *
24647570 4460 * This is solved by allowing the pools to be disassociated from the CPU
628c78e7
TH
4461 * running as an unbound one and allowing it to be reattached later if the
4462 * cpu comes back online.
db7bccf4 4463 */
1da177e4 4464
706026c2 4465static void wq_unbind_fn(struct work_struct *work)
3af24433 4466{
38db41d9 4467 int cpu = smp_processor_id();
4ce62e9e 4468 struct worker_pool *pool;
db7bccf4 4469 struct worker *worker;
3af24433 4470
f02ae73a 4471 for_each_cpu_worker_pool(pool, cpu) {
92f9c5c4 4472 mutex_lock(&pool->attach_mutex);
94cf58bb 4473 spin_lock_irq(&pool->lock);
3af24433 4474
94cf58bb 4475 /*
92f9c5c4 4476 * We've blocked all attach/detach operations. Make all workers
94cf58bb
TH
4477 * unbound and set DISASSOCIATED. Before this, all workers
4478 * except for the ones which are still executing works from
4479 * before the last CPU down must be on the cpu. After
4480 * this, they may become diasporas.
4481 */
da028469 4482 for_each_pool_worker(worker, pool)
c9e7cf27 4483 worker->flags |= WORKER_UNBOUND;
06ba38a9 4484
24647570 4485 pool->flags |= POOL_DISASSOCIATED;
f2d5a0ee 4486
94cf58bb 4487 spin_unlock_irq(&pool->lock);
92f9c5c4 4488 mutex_unlock(&pool->attach_mutex);
628c78e7 4489
eb283428
LJ
4490 /*
4491 * Call schedule() so that we cross rq->lock and thus can
4492 * guarantee sched callbacks see the %WORKER_UNBOUND flag.
4493 * This is necessary as scheduler callbacks may be invoked
4494 * from other cpus.
4495 */
4496 schedule();
06ba38a9 4497
eb283428
LJ
4498 /*
4499 * Sched callbacks are disabled now. Zap nr_running.
4500 * After this, nr_running stays zero and need_more_worker()
4501 * and keep_working() are always true as long as the
4502 * worklist is not empty. This pool now behaves as an
4503 * unbound (in terms of concurrency management) pool which
4504 * are served by workers tied to the pool.
4505 */
e19e397a 4506 atomic_set(&pool->nr_running, 0);
eb283428
LJ
4507
4508 /*
4509 * With concurrency management just turned off, a busy
4510 * worker blocking could lead to lengthy stalls. Kick off
4511 * unbound chain execution of currently pending work items.
4512 */
4513 spin_lock_irq(&pool->lock);
4514 wake_up_worker(pool);
4515 spin_unlock_irq(&pool->lock);
4516 }
3af24433 4517}
3af24433 4518
bd7c089e
TH
4519/**
4520 * rebind_workers - rebind all workers of a pool to the associated CPU
4521 * @pool: pool of interest
4522 *
a9ab775b 4523 * @pool->cpu is coming online. Rebind all workers to the CPU.
bd7c089e
TH
4524 */
4525static void rebind_workers(struct worker_pool *pool)
4526{
a9ab775b 4527 struct worker *worker;
bd7c089e 4528
92f9c5c4 4529 lockdep_assert_held(&pool->attach_mutex);
bd7c089e 4530
a9ab775b
TH
4531 /*
4532 * Restore CPU affinity of all workers. As all idle workers should
4533 * be on the run-queue of the associated CPU before any local
402dd89d 4534 * wake-ups for concurrency management happen, restore CPU affinity
a9ab775b
TH
4535 * of all workers first and then clear UNBOUND. As we're called
4536 * from CPU_ONLINE, the following shouldn't fail.
4537 */
da028469 4538 for_each_pool_worker(worker, pool)
a9ab775b
TH
4539 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4540 pool->attrs->cpumask) < 0);
bd7c089e 4541
a9ab775b 4542 spin_lock_irq(&pool->lock);
f7c17d26
WL
4543
4544 /*
4545 * XXX: CPU hotplug notifiers are weird and can call DOWN_FAILED
4546 * w/o preceding DOWN_PREPARE. Work around it. CPU hotplug is
4547 * being reworked and this can go away in time.
4548 */
4549 if (!(pool->flags & POOL_DISASSOCIATED)) {
4550 spin_unlock_irq(&pool->lock);
4551 return;
4552 }
4553
3de5e884 4554 pool->flags &= ~POOL_DISASSOCIATED;
bd7c089e 4555
da028469 4556 for_each_pool_worker(worker, pool) {
a9ab775b 4557 unsigned int worker_flags = worker->flags;
bd7c089e
TH
4558
4559 /*
a9ab775b
TH
4560 * A bound idle worker should actually be on the runqueue
4561 * of the associated CPU for local wake-ups targeting it to
4562 * work. Kick all idle workers so that they migrate to the
4563 * associated CPU. Doing this in the same loop as
4564 * replacing UNBOUND with REBOUND is safe as no worker will
4565 * be bound before @pool->lock is released.
bd7c089e 4566 */
a9ab775b
TH
4567 if (worker_flags & WORKER_IDLE)
4568 wake_up_process(worker->task);
bd7c089e 4569
a9ab775b
TH
4570 /*
4571 * We want to clear UNBOUND but can't directly call
4572 * worker_clr_flags() or adjust nr_running. Atomically
4573 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
4574 * @worker will clear REBOUND using worker_clr_flags() when
4575 * it initiates the next execution cycle thus restoring
4576 * concurrency management. Note that when or whether
4577 * @worker clears REBOUND doesn't affect correctness.
4578 *
4579 * ACCESS_ONCE() is necessary because @worker->flags may be
4580 * tested without holding any lock in
4581 * wq_worker_waking_up(). Without it, NOT_RUNNING test may
4582 * fail incorrectly leading to premature concurrency
4583 * management operations.
4584 */
4585 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4586 worker_flags |= WORKER_REBOUND;
4587 worker_flags &= ~WORKER_UNBOUND;
4588 ACCESS_ONCE(worker->flags) = worker_flags;
bd7c089e 4589 }
a9ab775b
TH
4590
4591 spin_unlock_irq(&pool->lock);
bd7c089e
TH
4592}
4593
7dbc725e
TH
4594/**
4595 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
4596 * @pool: unbound pool of interest
4597 * @cpu: the CPU which is coming up
4598 *
4599 * An unbound pool may end up with a cpumask which doesn't have any online
4600 * CPUs. When a worker of such pool get scheduled, the scheduler resets
4601 * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
4602 * online CPU before, cpus_allowed of all its workers should be restored.
4603 */
4604static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4605{
4606 static cpumask_t cpumask;
4607 struct worker *worker;
7dbc725e 4608
92f9c5c4 4609 lockdep_assert_held(&pool->attach_mutex);
7dbc725e
TH
4610
4611 /* is @cpu allowed for @pool? */
4612 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4613 return;
4614
7dbc725e 4615 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
7dbc725e
TH
4616
4617 /* as we're called from CPU_ONLINE, the following shouldn't fail */
da028469 4618 for_each_pool_worker(worker, pool)
d945b5e9 4619 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
7dbc725e
TH
4620}
4621
7ee681b2
TG
4622int workqueue_prepare_cpu(unsigned int cpu)
4623{
4624 struct worker_pool *pool;
4625
4626 for_each_cpu_worker_pool(pool, cpu) {
4627 if (pool->nr_workers)
4628 continue;
4629 if (!create_worker(pool))
4630 return -ENOMEM;
4631 }
4632 return 0;
4633}
4634
4635int workqueue_online_cpu(unsigned int cpu)
3af24433 4636{
4ce62e9e 4637 struct worker_pool *pool;
4c16bd32 4638 struct workqueue_struct *wq;
7dbc725e 4639 int pi;
3ce63377 4640
7ee681b2 4641 mutex_lock(&wq_pool_mutex);
7dbc725e 4642
7ee681b2
TG
4643 for_each_pool(pool, pi) {
4644 mutex_lock(&pool->attach_mutex);
94cf58bb 4645
7ee681b2
TG
4646 if (pool->cpu == cpu)
4647 rebind_workers(pool);
4648 else if (pool->cpu < 0)
4649 restore_unbound_workers_cpumask(pool, cpu);
94cf58bb 4650
7ee681b2
TG
4651 mutex_unlock(&pool->attach_mutex);
4652 }
6ba94429 4653
7ee681b2
TG
4654 /* update NUMA affinity of unbound workqueues */
4655 list_for_each_entry(wq, &workqueues, list)
4656 wq_update_unbound_numa(wq, cpu, true);
6ba94429 4657
7ee681b2
TG
4658 mutex_unlock(&wq_pool_mutex);
4659 return 0;
6ba94429
FW
4660}
4661
7ee681b2 4662int workqueue_offline_cpu(unsigned int cpu)
6ba94429 4663{
6ba94429
FW
4664 struct work_struct unbind_work;
4665 struct workqueue_struct *wq;
4666
7ee681b2
TG
4667 /* unbinding per-cpu workers should happen on the local CPU */
4668 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4669 queue_work_on(cpu, system_highpri_wq, &unbind_work);
4670
4671 /* update NUMA affinity of unbound workqueues */
4672 mutex_lock(&wq_pool_mutex);
4673 list_for_each_entry(wq, &workqueues, list)
4674 wq_update_unbound_numa(wq, cpu, false);
4675 mutex_unlock(&wq_pool_mutex);
4676
4677 /* wait for per-cpu unbinding to finish */
4678 flush_work(&unbind_work);
4679 destroy_work_on_stack(&unbind_work);
4680 return 0;
6ba94429
FW
4681}
4682
4683#ifdef CONFIG_SMP
4684
4685struct work_for_cpu {
4686 struct work_struct work;
4687 long (*fn)(void *);
4688 void *arg;
4689 long ret;
4690};
4691
4692static void work_for_cpu_fn(struct work_struct *work)
4693{
4694 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4695
4696 wfc->ret = wfc->fn(wfc->arg);
4697}
4698
4699/**
22aceb31 4700 * work_on_cpu - run a function in thread context on a particular cpu
6ba94429
FW
4701 * @cpu: the cpu to run on
4702 * @fn: the function to run
4703 * @arg: the function arg
4704 *
4705 * It is up to the caller to ensure that the cpu doesn't go offline.
4706 * The caller must not hold any locks which would prevent @fn from completing.
4707 *
4708 * Return: The value @fn returns.
4709 */
4710long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
4711{
4712 struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4713
4714 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4715 schedule_work_on(cpu, &wfc.work);
4716 flush_work(&wfc.work);
4717 destroy_work_on_stack(&wfc.work);
4718 return wfc.ret;
4719}
4720EXPORT_SYMBOL_GPL(work_on_cpu);
4721#endif /* CONFIG_SMP */
4722
4723#ifdef CONFIG_FREEZER
4724
4725/**
4726 * freeze_workqueues_begin - begin freezing workqueues
4727 *
4728 * Start freezing workqueues. After this function returns, all freezable
4729 * workqueues will queue new works to their delayed_works list instead of
4730 * pool->worklist.
4731 *
4732 * CONTEXT:
4733 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4734 */
4735void freeze_workqueues_begin(void)
4736{
4737 struct workqueue_struct *wq;
4738 struct pool_workqueue *pwq;
4739
4740 mutex_lock(&wq_pool_mutex);
4741
4742 WARN_ON_ONCE(workqueue_freezing);
4743 workqueue_freezing = true;
4744
4745 list_for_each_entry(wq, &workqueues, list) {
4746 mutex_lock(&wq->mutex);
4747 for_each_pwq(pwq, wq)
4748 pwq_adjust_max_active(pwq);
4749 mutex_unlock(&wq->mutex);
4750 }
4751
4752 mutex_unlock(&wq_pool_mutex);
4753}
4754
4755/**
4756 * freeze_workqueues_busy - are freezable workqueues still busy?
4757 *
4758 * Check whether freezing is complete. This function must be called
4759 * between freeze_workqueues_begin() and thaw_workqueues().
4760 *
4761 * CONTEXT:
4762 * Grabs and releases wq_pool_mutex.
4763 *
4764 * Return:
4765 * %true if some freezable workqueues are still busy. %false if freezing
4766 * is complete.
4767 */
4768bool freeze_workqueues_busy(void)
4769{
4770 bool busy = false;
4771 struct workqueue_struct *wq;
4772 struct pool_workqueue *pwq;
4773
4774 mutex_lock(&wq_pool_mutex);
4775
4776 WARN_ON_ONCE(!workqueue_freezing);
4777
4778 list_for_each_entry(wq, &workqueues, list) {
4779 if (!(wq->flags & WQ_FREEZABLE))
4780 continue;
4781 /*
4782 * nr_active is monotonically decreasing. It's safe
4783 * to peek without lock.
4784 */
4785 rcu_read_lock_sched();
4786 for_each_pwq(pwq, wq) {
4787 WARN_ON_ONCE(pwq->nr_active < 0);
4788 if (pwq->nr_active) {
4789 busy = true;
4790 rcu_read_unlock_sched();
4791 goto out_unlock;
4792 }
4793 }
4794 rcu_read_unlock_sched();
4795 }
4796out_unlock:
4797 mutex_unlock(&wq_pool_mutex);
4798 return busy;
4799}
4800
4801/**
4802 * thaw_workqueues - thaw workqueues
4803 *
4804 * Thaw workqueues. Normal queueing is restored and all collected
4805 * frozen works are transferred to their respective pool worklists.
4806 *
4807 * CONTEXT:
4808 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4809 */
4810void thaw_workqueues(void)
4811{
4812 struct workqueue_struct *wq;
4813 struct pool_workqueue *pwq;
4814
4815 mutex_lock(&wq_pool_mutex);
4816
4817 if (!workqueue_freezing)
4818 goto out_unlock;
4819
4820 workqueue_freezing = false;
4821
4822 /* restore max_active and repopulate worklist */
4823 list_for_each_entry(wq, &workqueues, list) {
4824 mutex_lock(&wq->mutex);
4825 for_each_pwq(pwq, wq)
4826 pwq_adjust_max_active(pwq);
4827 mutex_unlock(&wq->mutex);
4828 }
4829
4830out_unlock:
4831 mutex_unlock(&wq_pool_mutex);
4832}
4833#endif /* CONFIG_FREEZER */
4834
042f7df1
LJ
4835static int workqueue_apply_unbound_cpumask(void)
4836{
4837 LIST_HEAD(ctxs);
4838 int ret = 0;
4839 struct workqueue_struct *wq;
4840 struct apply_wqattrs_ctx *ctx, *n;
4841
4842 lockdep_assert_held(&wq_pool_mutex);
4843
4844 list_for_each_entry(wq, &workqueues, list) {
4845 if (!(wq->flags & WQ_UNBOUND))
4846 continue;
4847 /* creating multiple pwqs breaks ordering guarantee */
4848 if (wq->flags & __WQ_ORDERED)
4849 continue;
4850
4851 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
4852 if (!ctx) {
4853 ret = -ENOMEM;
4854 break;
4855 }
4856
4857 list_add_tail(&ctx->list, &ctxs);
4858 }
4859
4860 list_for_each_entry_safe(ctx, n, &ctxs, list) {
4861 if (!ret)
4862 apply_wqattrs_commit(ctx);
4863 apply_wqattrs_cleanup(ctx);
4864 }
4865
4866 return ret;
4867}
4868
4869/**
4870 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
4871 * @cpumask: the cpumask to set
4872 *
4873 * The low-level workqueues cpumask is a global cpumask that limits
4874 * the affinity of all unbound workqueues. This function check the @cpumask
4875 * and apply it to all unbound workqueues and updates all pwqs of them.
4876 *
4877 * Retun: 0 - Success
4878 * -EINVAL - Invalid @cpumask
4879 * -ENOMEM - Failed to allocate memory for attrs or pwqs.
4880 */
4881int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
4882{
4883 int ret = -EINVAL;
4884 cpumask_var_t saved_cpumask;
4885
4886 if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL))
4887 return -ENOMEM;
4888
042f7df1
LJ
4889 cpumask_and(cpumask, cpumask, cpu_possible_mask);
4890 if (!cpumask_empty(cpumask)) {
a0111cf6 4891 apply_wqattrs_lock();
042f7df1
LJ
4892
4893 /* save the old wq_unbound_cpumask. */
4894 cpumask_copy(saved_cpumask, wq_unbound_cpumask);
4895
4896 /* update wq_unbound_cpumask at first and apply it to wqs. */
4897 cpumask_copy(wq_unbound_cpumask, cpumask);
4898 ret = workqueue_apply_unbound_cpumask();
4899
4900 /* restore the wq_unbound_cpumask when failed. */
4901 if (ret < 0)
4902 cpumask_copy(wq_unbound_cpumask, saved_cpumask);
4903
a0111cf6 4904 apply_wqattrs_unlock();
042f7df1 4905 }
042f7df1
LJ
4906
4907 free_cpumask_var(saved_cpumask);
4908 return ret;
4909}
4910
6ba94429
FW
4911#ifdef CONFIG_SYSFS
4912/*
4913 * Workqueues with WQ_SYSFS flag set is visible to userland via
4914 * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
4915 * following attributes.
4916 *
4917 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
4918 * max_active RW int : maximum number of in-flight work items
4919 *
4920 * Unbound workqueues have the following extra attributes.
4921 *
4922 * id RO int : the associated pool ID
4923 * nice RW int : nice value of the workers
4924 * cpumask RW mask : bitmask of allowed CPUs for the workers
4925 */
4926struct wq_device {
4927 struct workqueue_struct *wq;
4928 struct device dev;
4929};
4930
4931static struct workqueue_struct *dev_to_wq(struct device *dev)
4932{
4933 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
4934
4935 return wq_dev->wq;
4936}
4937
4938static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
4939 char *buf)
4940{
4941 struct workqueue_struct *wq = dev_to_wq(dev);
4942
4943 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
4944}
4945static DEVICE_ATTR_RO(per_cpu);
4946
4947static ssize_t max_active_show(struct device *dev,
4948 struct device_attribute *attr, char *buf)
4949{
4950 struct workqueue_struct *wq = dev_to_wq(dev);
4951
4952 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
4953}
4954
4955static ssize_t max_active_store(struct device *dev,
4956 struct device_attribute *attr, const char *buf,
4957 size_t count)
4958{
4959 struct workqueue_struct *wq = dev_to_wq(dev);
4960 int val;
4961
4962 if (sscanf(buf, "%d", &val) != 1 || val <= 0)
4963 return -EINVAL;
4964
4965 workqueue_set_max_active(wq, val);
4966 return count;
4967}
4968static DEVICE_ATTR_RW(max_active);
4969
4970static struct attribute *wq_sysfs_attrs[] = {
4971 &dev_attr_per_cpu.attr,
4972 &dev_attr_max_active.attr,
4973 NULL,
4974};
4975ATTRIBUTE_GROUPS(wq_sysfs);
4976
4977static ssize_t wq_pool_ids_show(struct device *dev,
4978 struct device_attribute *attr, char *buf)
4979{
4980 struct workqueue_struct *wq = dev_to_wq(dev);
4981 const char *delim = "";
4982 int node, written = 0;
4983
4984 rcu_read_lock_sched();
4985 for_each_node(node) {
4986 written += scnprintf(buf + written, PAGE_SIZE - written,
4987 "%s%d:%d", delim, node,
4988 unbound_pwq_by_node(wq, node)->pool->id);
4989 delim = " ";
4990 }
4991 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
4992 rcu_read_unlock_sched();
4993
4994 return written;
4995}
4996
4997static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
4998 char *buf)
4999{
5000 struct workqueue_struct *wq = dev_to_wq(dev);
5001 int written;
5002
5003 mutex_lock(&wq->mutex);
5004 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
5005 mutex_unlock(&wq->mutex);
5006
5007 return written;
5008}
5009
5010/* prepare workqueue_attrs for sysfs store operations */
5011static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
5012{
5013 struct workqueue_attrs *attrs;
5014
899a94fe
LJ
5015 lockdep_assert_held(&wq_pool_mutex);
5016
6ba94429
FW
5017 attrs = alloc_workqueue_attrs(GFP_KERNEL);
5018 if (!attrs)
5019 return NULL;
5020
6ba94429 5021 copy_workqueue_attrs(attrs, wq->unbound_attrs);
6ba94429
FW
5022 return attrs;
5023}
5024
5025static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
5026 const char *buf, size_t count)
5027{
5028 struct workqueue_struct *wq = dev_to_wq(dev);
5029 struct workqueue_attrs *attrs;
d4d3e257
LJ
5030 int ret = -ENOMEM;
5031
5032 apply_wqattrs_lock();
6ba94429
FW
5033
5034 attrs = wq_sysfs_prep_attrs(wq);
5035 if (!attrs)
d4d3e257 5036 goto out_unlock;
6ba94429
FW
5037
5038 if (sscanf(buf, "%d", &attrs->nice) == 1 &&
5039 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
d4d3e257 5040 ret = apply_workqueue_attrs_locked(wq, attrs);
6ba94429
FW
5041 else
5042 ret = -EINVAL;
5043
d4d3e257
LJ
5044out_unlock:
5045 apply_wqattrs_unlock();
6ba94429
FW
5046 free_workqueue_attrs(attrs);
5047 return ret ?: count;
5048}
5049
5050static ssize_t wq_cpumask_show(struct device *dev,
5051 struct device_attribute *attr, char *buf)
5052{
5053 struct workqueue_struct *wq = dev_to_wq(dev);
5054 int written;
5055
5056 mutex_lock(&wq->mutex);
5057 written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5058 cpumask_pr_args(wq->unbound_attrs->cpumask));
5059 mutex_unlock(&wq->mutex);
5060 return written;
5061}
5062
5063static ssize_t wq_cpumask_store(struct device *dev,
5064 struct device_attribute *attr,
5065 const char *buf, size_t count)
5066{
5067 struct workqueue_struct *wq = dev_to_wq(dev);
5068 struct workqueue_attrs *attrs;
d4d3e257
LJ
5069 int ret = -ENOMEM;
5070
5071 apply_wqattrs_lock();
6ba94429
FW
5072
5073 attrs = wq_sysfs_prep_attrs(wq);
5074 if (!attrs)
d4d3e257 5075 goto out_unlock;
6ba94429
FW
5076
5077 ret = cpumask_parse(buf, attrs->cpumask);
5078 if (!ret)
d4d3e257 5079 ret = apply_workqueue_attrs_locked(wq, attrs);
6ba94429 5080
d4d3e257
LJ
5081out_unlock:
5082 apply_wqattrs_unlock();
6ba94429
FW
5083 free_workqueue_attrs(attrs);
5084 return ret ?: count;
5085}
5086
5087static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
5088 char *buf)
5089{
5090 struct workqueue_struct *wq = dev_to_wq(dev);
5091 int written;
7dbc725e 5092
6ba94429
FW
5093 mutex_lock(&wq->mutex);
5094 written = scnprintf(buf, PAGE_SIZE, "%d\n",
5095 !wq->unbound_attrs->no_numa);
5096 mutex_unlock(&wq->mutex);
4c16bd32 5097
6ba94429 5098 return written;
65758202
TH
5099}
5100
6ba94429
FW
5101static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
5102 const char *buf, size_t count)
65758202 5103{
6ba94429
FW
5104 struct workqueue_struct *wq = dev_to_wq(dev);
5105 struct workqueue_attrs *attrs;
d4d3e257
LJ
5106 int v, ret = -ENOMEM;
5107
5108 apply_wqattrs_lock();
4c16bd32 5109
6ba94429
FW
5110 attrs = wq_sysfs_prep_attrs(wq);
5111 if (!attrs)
d4d3e257 5112 goto out_unlock;
4c16bd32 5113
6ba94429
FW
5114 ret = -EINVAL;
5115 if (sscanf(buf, "%d", &v) == 1) {
5116 attrs->no_numa = !v;
d4d3e257 5117 ret = apply_workqueue_attrs_locked(wq, attrs);
65758202 5118 }
6ba94429 5119
d4d3e257
LJ
5120out_unlock:
5121 apply_wqattrs_unlock();
6ba94429
FW
5122 free_workqueue_attrs(attrs);
5123 return ret ?: count;
65758202
TH
5124}
5125
6ba94429
FW
5126static struct device_attribute wq_sysfs_unbound_attrs[] = {
5127 __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5128 __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5129 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5130 __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5131 __ATTR_NULL,
5132};
8ccad40d 5133
6ba94429
FW
5134static struct bus_type wq_subsys = {
5135 .name = "workqueue",
5136 .dev_groups = wq_sysfs_groups,
2d3854a3
RR
5137};
5138
b05a7928
FW
5139static ssize_t wq_unbound_cpumask_show(struct device *dev,
5140 struct device_attribute *attr, char *buf)
5141{
5142 int written;
5143
042f7df1 5144 mutex_lock(&wq_pool_mutex);
b05a7928
FW
5145 written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5146 cpumask_pr_args(wq_unbound_cpumask));
042f7df1 5147 mutex_unlock(&wq_pool_mutex);
b05a7928
FW
5148
5149 return written;
5150}
5151
042f7df1
LJ
5152static ssize_t wq_unbound_cpumask_store(struct device *dev,
5153 struct device_attribute *attr, const char *buf, size_t count)
5154{
5155 cpumask_var_t cpumask;
5156 int ret;
5157
5158 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5159 return -ENOMEM;
5160
5161 ret = cpumask_parse(buf, cpumask);
5162 if (!ret)
5163 ret = workqueue_set_unbound_cpumask(cpumask);
5164
5165 free_cpumask_var(cpumask);
5166 return ret ? ret : count;
5167}
5168
b05a7928 5169static struct device_attribute wq_sysfs_cpumask_attr =
042f7df1
LJ
5170 __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5171 wq_unbound_cpumask_store);
b05a7928 5172
6ba94429 5173static int __init wq_sysfs_init(void)
2d3854a3 5174{
b05a7928
FW
5175 int err;
5176
5177 err = subsys_virtual_register(&wq_subsys, NULL);
5178 if (err)
5179 return err;
5180
5181 return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
2d3854a3 5182}
6ba94429 5183core_initcall(wq_sysfs_init);
2d3854a3 5184
6ba94429 5185static void wq_device_release(struct device *dev)
2d3854a3 5186{
6ba94429 5187 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
6b44003e 5188
6ba94429 5189 kfree(wq_dev);
2d3854a3 5190}
a0a1a5fd
TH
5191
5192/**
6ba94429
FW
5193 * workqueue_sysfs_register - make a workqueue visible in sysfs
5194 * @wq: the workqueue to register
a0a1a5fd 5195 *
6ba94429
FW
5196 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
5197 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5198 * which is the preferred method.
a0a1a5fd 5199 *
6ba94429
FW
5200 * Workqueue user should use this function directly iff it wants to apply
5201 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5202 * apply_workqueue_attrs() may race against userland updating the
5203 * attributes.
5204 *
5205 * Return: 0 on success, -errno on failure.
a0a1a5fd 5206 */
6ba94429 5207int workqueue_sysfs_register(struct workqueue_struct *wq)
a0a1a5fd 5208{
6ba94429
FW
5209 struct wq_device *wq_dev;
5210 int ret;
a0a1a5fd 5211
6ba94429 5212 /*
402dd89d 5213 * Adjusting max_active or creating new pwqs by applying
6ba94429
FW
5214 * attributes breaks ordering guarantee. Disallow exposing ordered
5215 * workqueues.
5216 */
5217 if (WARN_ON(wq->flags & __WQ_ORDERED))
5218 return -EINVAL;
a0a1a5fd 5219
6ba94429
FW
5220 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5221 if (!wq_dev)
5222 return -ENOMEM;
5bcab335 5223
6ba94429
FW
5224 wq_dev->wq = wq;
5225 wq_dev->dev.bus = &wq_subsys;
6ba94429 5226 wq_dev->dev.release = wq_device_release;
23217b44 5227 dev_set_name(&wq_dev->dev, "%s", wq->name);
a0a1a5fd 5228
6ba94429
FW
5229 /*
5230 * unbound_attrs are created separately. Suppress uevent until
5231 * everything is ready.
5232 */
5233 dev_set_uevent_suppress(&wq_dev->dev, true);
a0a1a5fd 5234
6ba94429
FW
5235 ret = device_register(&wq_dev->dev);
5236 if (ret) {
5237 kfree(wq_dev);
5238 wq->wq_dev = NULL;
5239 return ret;
5240 }
a0a1a5fd 5241
6ba94429
FW
5242 if (wq->flags & WQ_UNBOUND) {
5243 struct device_attribute *attr;
a0a1a5fd 5244
6ba94429
FW
5245 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5246 ret = device_create_file(&wq_dev->dev, attr);
5247 if (ret) {
5248 device_unregister(&wq_dev->dev);
5249 wq->wq_dev = NULL;
5250 return ret;
a0a1a5fd
TH
5251 }
5252 }
5253 }
6ba94429
FW
5254
5255 dev_set_uevent_suppress(&wq_dev->dev, false);
5256 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5257 return 0;
a0a1a5fd
TH
5258}
5259
5260/**
6ba94429
FW
5261 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5262 * @wq: the workqueue to unregister
a0a1a5fd 5263 *
6ba94429 5264 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
a0a1a5fd 5265 */
6ba94429 5266static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
a0a1a5fd 5267{
6ba94429 5268 struct wq_device *wq_dev = wq->wq_dev;
8b03ae3c 5269
6ba94429
FW
5270 if (!wq->wq_dev)
5271 return;
a0a1a5fd 5272
6ba94429
FW
5273 wq->wq_dev = NULL;
5274 device_unregister(&wq_dev->dev);
a0a1a5fd 5275}
6ba94429
FW
5276#else /* CONFIG_SYSFS */
5277static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
5278#endif /* CONFIG_SYSFS */
a0a1a5fd 5279
82607adc
TH
5280/*
5281 * Workqueue watchdog.
5282 *
5283 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5284 * flush dependency, a concurrency managed work item which stays RUNNING
5285 * indefinitely. Workqueue stalls can be very difficult to debug as the
5286 * usual warning mechanisms don't trigger and internal workqueue state is
5287 * largely opaque.
5288 *
5289 * Workqueue watchdog monitors all worker pools periodically and dumps
5290 * state if some pools failed to make forward progress for a while where
5291 * forward progress is defined as the first item on ->worklist changing.
5292 *
5293 * This mechanism is controlled through the kernel parameter
5294 * "workqueue.watchdog_thresh" which can be updated at runtime through the
5295 * corresponding sysfs parameter file.
5296 */
5297#ifdef CONFIG_WQ_WATCHDOG
5298
5299static void wq_watchdog_timer_fn(unsigned long data);
5300
5301static unsigned long wq_watchdog_thresh = 30;
5302static struct timer_list wq_watchdog_timer =
5303 TIMER_DEFERRED_INITIALIZER(wq_watchdog_timer_fn, 0, 0);
5304
5305static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
5306static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
5307
5308static void wq_watchdog_reset_touched(void)
5309{
5310 int cpu;
5311
5312 wq_watchdog_touched = jiffies;
5313 for_each_possible_cpu(cpu)
5314 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5315}
5316
5317static void wq_watchdog_timer_fn(unsigned long data)
5318{
5319 unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
5320 bool lockup_detected = false;
5321 struct worker_pool *pool;
5322 int pi;
5323
5324 if (!thresh)
5325 return;
5326
5327 rcu_read_lock();
5328
5329 for_each_pool(pool, pi) {
5330 unsigned long pool_ts, touched, ts;
5331
5332 if (list_empty(&pool->worklist))
5333 continue;
5334
5335 /* get the latest of pool and touched timestamps */
5336 pool_ts = READ_ONCE(pool->watchdog_ts);
5337 touched = READ_ONCE(wq_watchdog_touched);
5338
5339 if (time_after(pool_ts, touched))
5340 ts = pool_ts;
5341 else
5342 ts = touched;
5343
5344 if (pool->cpu >= 0) {
5345 unsigned long cpu_touched =
5346 READ_ONCE(per_cpu(wq_watchdog_touched_cpu,
5347 pool->cpu));
5348 if (time_after(cpu_touched, ts))
5349 ts = cpu_touched;
5350 }
5351
5352 /* did we stall? */
5353 if (time_after(jiffies, ts + thresh)) {
5354 lockup_detected = true;
5355 pr_emerg("BUG: workqueue lockup - pool");
5356 pr_cont_pool_info(pool);
5357 pr_cont(" stuck for %us!\n",
5358 jiffies_to_msecs(jiffies - pool_ts) / 1000);
5359 }
5360 }
5361
5362 rcu_read_unlock();
5363
5364 if (lockup_detected)
5365 show_workqueue_state();
5366
5367 wq_watchdog_reset_touched();
5368 mod_timer(&wq_watchdog_timer, jiffies + thresh);
5369}
5370
5371void wq_watchdog_touch(int cpu)
5372{
5373 if (cpu >= 0)
5374 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5375 else
5376 wq_watchdog_touched = jiffies;
5377}
5378
5379static void wq_watchdog_set_thresh(unsigned long thresh)
5380{
5381 wq_watchdog_thresh = 0;
5382 del_timer_sync(&wq_watchdog_timer);
5383
5384 if (thresh) {
5385 wq_watchdog_thresh = thresh;
5386 wq_watchdog_reset_touched();
5387 mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
5388 }
5389}
5390
5391static int wq_watchdog_param_set_thresh(const char *val,
5392 const struct kernel_param *kp)
5393{
5394 unsigned long thresh;
5395 int ret;
5396
5397 ret = kstrtoul(val, 0, &thresh);
5398 if (ret)
5399 return ret;
5400
5401 if (system_wq)
5402 wq_watchdog_set_thresh(thresh);
5403 else
5404 wq_watchdog_thresh = thresh;
5405
5406 return 0;
5407}
5408
5409static const struct kernel_param_ops wq_watchdog_thresh_ops = {
5410 .set = wq_watchdog_param_set_thresh,
5411 .get = param_get_ulong,
5412};
5413
5414module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
5415 0644);
5416
5417static void wq_watchdog_init(void)
5418{
5419 wq_watchdog_set_thresh(wq_watchdog_thresh);
5420}
5421
5422#else /* CONFIG_WQ_WATCHDOG */
5423
5424static inline void wq_watchdog_init(void) { }
5425
5426#endif /* CONFIG_WQ_WATCHDOG */
5427
bce90380
TH
5428static void __init wq_numa_init(void)
5429{
5430 cpumask_var_t *tbl;
5431 int node, cpu;
5432
bce90380
TH
5433 if (num_possible_nodes() <= 1)
5434 return;
5435
d55262c4
TH
5436 if (wq_disable_numa) {
5437 pr_info("workqueue: NUMA affinity support disabled\n");
5438 return;
5439 }
5440
4c16bd32
TH
5441 wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
5442 BUG_ON(!wq_update_unbound_numa_attrs_buf);
5443
bce90380
TH
5444 /*
5445 * We want masks of possible CPUs of each node which isn't readily
5446 * available. Build one from cpu_to_node() which should have been
5447 * fully initialized by now.
5448 */
ddcb57e2 5449 tbl = kzalloc(nr_node_ids * sizeof(tbl[0]), GFP_KERNEL);
bce90380
TH
5450 BUG_ON(!tbl);
5451
5452 for_each_node(node)
5a6024f1 5453 BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
1be0c25d 5454 node_online(node) ? node : NUMA_NO_NODE));
bce90380
TH
5455
5456 for_each_possible_cpu(cpu) {
5457 node = cpu_to_node(cpu);
5458 if (WARN_ON(node == NUMA_NO_NODE)) {
5459 pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5460 /* happens iff arch is bonkers, let's just proceed */
5461 return;
5462 }
5463 cpumask_set_cpu(cpu, tbl[node]);
5464 }
5465
5466 wq_numa_possible_cpumask = tbl;
5467 wq_numa_enabled = true;
5468}
5469
6ee0578b 5470static int __init init_workqueues(void)
1da177e4 5471{
7a4e344c
TH
5472 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5473 int i, cpu;
c34056a3 5474
e904e6c2
TH
5475 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5476
b05a7928
FW
5477 BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5478 cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
5479
e904e6c2
TH
5480 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5481
bce90380
TH
5482 wq_numa_init();
5483
706026c2 5484 /* initialize CPU pools */
29c91e99 5485 for_each_possible_cpu(cpu) {
4ce62e9e 5486 struct worker_pool *pool;
8b03ae3c 5487
7a4e344c 5488 i = 0;
f02ae73a 5489 for_each_cpu_worker_pool(pool, cpu) {
7a4e344c 5490 BUG_ON(init_worker_pool(pool));
ec22ca5e 5491 pool->cpu = cpu;
29c91e99 5492 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
7a4e344c 5493 pool->attrs->nice = std_nice[i++];
f3f90ad4 5494 pool->node = cpu_to_node(cpu);
7a4e344c 5495
9daf9e67 5496 /* alloc pool ID */
68e13a67 5497 mutex_lock(&wq_pool_mutex);
9daf9e67 5498 BUG_ON(worker_pool_assign_id(pool));
68e13a67 5499 mutex_unlock(&wq_pool_mutex);
4ce62e9e 5500 }
8b03ae3c
TH
5501 }
5502
e22bee78 5503 /* create the initial worker */
29c91e99 5504 for_each_online_cpu(cpu) {
4ce62e9e 5505 struct worker_pool *pool;
e22bee78 5506
f02ae73a 5507 for_each_cpu_worker_pool(pool, cpu) {
29c91e99 5508 pool->flags &= ~POOL_DISASSOCIATED;
051e1850 5509 BUG_ON(!create_worker(pool));
4ce62e9e 5510 }
e22bee78
TH
5511 }
5512
8a2b7538 5513 /* create default unbound and ordered wq attrs */
29c91e99
TH
5514 for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5515 struct workqueue_attrs *attrs;
5516
5517 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
29c91e99 5518 attrs->nice = std_nice[i];
29c91e99 5519 unbound_std_wq_attrs[i] = attrs;
8a2b7538
TH
5520
5521 /*
5522 * An ordered wq should have only one pwq as ordering is
5523 * guaranteed by max_active which is enforced by pwqs.
5524 * Turn off NUMA so that dfl_pwq is used for all nodes.
5525 */
5526 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5527 attrs->nice = std_nice[i];
5528 attrs->no_numa = true;
5529 ordered_wq_attrs[i] = attrs;
29c91e99
TH
5530 }
5531
d320c038 5532 system_wq = alloc_workqueue("events", 0, 0);
1aabe902 5533 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
d320c038 5534 system_long_wq = alloc_workqueue("events_long", 0, 0);
f3421797
TH
5535 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5536 WQ_UNBOUND_MAX_ACTIVE);
24d51add
TH
5537 system_freezable_wq = alloc_workqueue("events_freezable",
5538 WQ_FREEZABLE, 0);
0668106c
VK
5539 system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5540 WQ_POWER_EFFICIENT, 0);
5541 system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5542 WQ_FREEZABLE | WQ_POWER_EFFICIENT,
5543 0);
1aabe902 5544 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
0668106c
VK
5545 !system_unbound_wq || !system_freezable_wq ||
5546 !system_power_efficient_wq ||
5547 !system_freezable_power_efficient_wq);
82607adc
TH
5548
5549 wq_watchdog_init();
5550
6ee0578b 5551 return 0;
1da177e4 5552}
6ee0578b 5553early_initcall(init_workqueues);