cgroup: explicitly track whether a cgroup_subsys_state is visible to userland
[linux-2.6-block.git] / kernel / cgroup.c
CommitLineData
ddbcc7e8 1/*
ddbcc7e8
PM
2 * Generic process-grouping system.
3 *
4 * Based originally on the cpuset system, extracted by Paul Menage
5 * Copyright (C) 2006 Google, Inc
6 *
0dea1168
KS
7 * Notifications support
8 * Copyright (C) 2009 Nokia Corporation
9 * Author: Kirill A. Shutemov
10 *
ddbcc7e8
PM
11 * Copyright notices from the original cpuset code:
12 * --------------------------------------------------
13 * Copyright (C) 2003 BULL SA.
14 * Copyright (C) 2004-2006 Silicon Graphics, Inc.
15 *
16 * Portions derived from Patrick Mochel's sysfs code.
17 * sysfs is Copyright (c) 2001-3 Patrick Mochel
18 *
19 * 2003-10-10 Written by Simon Derr.
20 * 2003-10-22 Updates by Stephen Hemminger.
21 * 2004 May-July Rework by Paul Jackson.
22 * ---------------------------------------------------
23 *
24 * This file is subject to the terms and conditions of the GNU General Public
25 * License. See the file COPYING in the main directory of the Linux
26 * distribution for more details.
27 */
28
ed3d261b
JP
29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
ddbcc7e8 31#include <linux/cgroup.h>
2ce9738b 32#include <linux/cred.h>
c6d57f33 33#include <linux/ctype.h>
ddbcc7e8 34#include <linux/errno.h>
2ce9738b 35#include <linux/init_task.h>
ddbcc7e8
PM
36#include <linux/kernel.h>
37#include <linux/list.h>
c9482a5b 38#include <linux/magic.h>
ddbcc7e8
PM
39#include <linux/mm.h>
40#include <linux/mutex.h>
41#include <linux/mount.h>
42#include <linux/pagemap.h>
a424316c 43#include <linux/proc_fs.h>
ddbcc7e8
PM
44#include <linux/rcupdate.h>
45#include <linux/sched.h>
ddbcc7e8 46#include <linux/slab.h>
ddbcc7e8 47#include <linux/spinlock.h>
1ed13287 48#include <linux/percpu-rwsem.h>
ddbcc7e8 49#include <linux/string.h>
bbcb81d0 50#include <linux/sort.h>
81a6a5cd 51#include <linux/kmod.h>
846c7bb0
BS
52#include <linux/delayacct.h>
53#include <linux/cgroupstats.h>
0ac801fe 54#include <linux/hashtable.h>
096b7fe0 55#include <linux/pid_namespace.h>
2c6ab6d2 56#include <linux/idr.h>
d1d9fd33 57#include <linux/vmalloc.h> /* TODO: replace with more sophisticated array */
c4c27fbd 58#include <linux/kthread.h>
776f02fa 59#include <linux/delay.h>
60063497 60#include <linux/atomic.h>
e93ad19d 61#include <linux/cpuset.h>
bd1060a1 62#include <net/sock.h>
ddbcc7e8 63
b1a21367
TH
64/*
65 * pidlists linger the following amount before being destroyed. The goal
66 * is avoiding frequent destruction in the middle of consecutive read calls
67 * Expiring in the middle is a performance problem not a correctness one.
68 * 1 sec should be enough.
69 */
70#define CGROUP_PIDLIST_DESTROY_DELAY HZ
71
8d7e6fb0
TH
72#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \
73 MAX_CFTYPE_NAME + 2)
74
e25e2cbb
TH
75/*
76 * cgroup_mutex is the master lock. Any modification to cgroup or its
77 * hierarchy must be performed while holding it.
78 *
f0d9a5f1 79 * css_set_lock protects task->cgroups pointer, the list of css_set
0e1d768f 80 * objects, and the chain of tasks off each css_set.
e25e2cbb 81 *
0e1d768f
TH
82 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
83 * cgroup.h can use them for lockdep annotations.
e25e2cbb 84 */
2219449a
TH
85#ifdef CONFIG_PROVE_RCU
86DEFINE_MUTEX(cgroup_mutex);
f0d9a5f1 87DEFINE_SPINLOCK(css_set_lock);
0e1d768f 88EXPORT_SYMBOL_GPL(cgroup_mutex);
f0d9a5f1 89EXPORT_SYMBOL_GPL(css_set_lock);
2219449a 90#else
81a6a5cd 91static DEFINE_MUTEX(cgroup_mutex);
f0d9a5f1 92static DEFINE_SPINLOCK(css_set_lock);
2219449a
TH
93#endif
94
6fa4918d 95/*
15a4c835
TH
96 * Protects cgroup_idr and css_idr so that IDs can be released without
97 * grabbing cgroup_mutex.
6fa4918d
TH
98 */
99static DEFINE_SPINLOCK(cgroup_idr_lock);
100
34c06254
TH
101/*
102 * Protects cgroup_file->kn for !self csses. It synchronizes notifications
103 * against file removal/re-creation across css hiding.
104 */
105static DEFINE_SPINLOCK(cgroup_file_kn_lock);
106
69e943b7
TH
107/*
108 * Protects cgroup_subsys->release_agent_path. Modifying it also requires
109 * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock.
110 */
111static DEFINE_SPINLOCK(release_agent_path_lock);
81a6a5cd 112
1ed13287
TH
113struct percpu_rw_semaphore cgroup_threadgroup_rwsem;
114
8353da1f 115#define cgroup_assert_mutex_or_rcu_locked() \
f78f5b90
PM
116 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
117 !lockdep_is_held(&cgroup_mutex), \
8353da1f 118 "cgroup_mutex or RCU read lock required");
780cd8b3 119
e5fca243
TH
120/*
121 * cgroup destruction makes heavy use of work items and there can be a lot
122 * of concurrent destructions. Use a separate workqueue so that cgroup
123 * destruction work items don't end up filling up max_active of system_wq
124 * which may lead to deadlock.
125 */
126static struct workqueue_struct *cgroup_destroy_wq;
127
b1a21367
TH
128/*
129 * pidlist destructions need to be flushed on cgroup destruction. Use a
130 * separate workqueue as flush domain.
131 */
132static struct workqueue_struct *cgroup_pidlist_destroy_wq;
133
3ed80a62 134/* generate an array of cgroup subsystem pointers */
073219e9 135#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
3ed80a62 136static struct cgroup_subsys *cgroup_subsys[] = {
ddbcc7e8
PM
137#include <linux/cgroup_subsys.h>
138};
073219e9
TH
139#undef SUBSYS
140
141/* array of cgroup subsystem names */
142#define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
143static const char *cgroup_subsys_name[] = {
ddbcc7e8
PM
144#include <linux/cgroup_subsys.h>
145};
073219e9 146#undef SUBSYS
ddbcc7e8 147
49d1dc4b
TH
148/* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */
149#define SUBSYS(_x) \
150 DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key); \
151 DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key); \
152 EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key); \
153 EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key);
154#include <linux/cgroup_subsys.h>
155#undef SUBSYS
156
157#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key,
158static struct static_key_true *cgroup_subsys_enabled_key[] = {
159#include <linux/cgroup_subsys.h>
160};
161#undef SUBSYS
162
163#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key,
164static struct static_key_true *cgroup_subsys_on_dfl_key[] = {
165#include <linux/cgroup_subsys.h>
166};
167#undef SUBSYS
168
ddbcc7e8 169/*
3dd06ffa 170 * The default hierarchy, reserved for the subsystems that are otherwise
9871bf95
TH
171 * unattached - it never has more than a single cgroup, and all tasks are
172 * part of that cgroup.
ddbcc7e8 173 */
a2dd4247 174struct cgroup_root cgrp_dfl_root;
d0ec4230 175EXPORT_SYMBOL_GPL(cgrp_dfl_root);
9871bf95 176
a2dd4247
TH
177/*
178 * The default hierarchy always exists but is hidden until mounted for the
179 * first time. This is for backward compatibility.
180 */
a7165264 181static bool cgrp_dfl_visible;
ddbcc7e8 182
223ffb29 183/* Controllers blocked by the commandline in v1 */
6e5c8307 184static u16 cgroup_no_v1_mask;
223ffb29 185
5533e011 186/* some controllers are not supported in the default hierarchy */
a7165264 187static u16 cgrp_dfl_inhibit_ss_mask;
5533e011 188
ddbcc7e8
PM
189/* The list of hierarchy roots */
190
9871bf95
TH
191static LIST_HEAD(cgroup_roots);
192static int cgroup_root_count;
ddbcc7e8 193
3417ae1f 194/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
1a574231 195static DEFINE_IDR(cgroup_hierarchy_idr);
2c6ab6d2 196
794611a1 197/*
0cb51d71
TH
198 * Assign a monotonically increasing serial number to csses. It guarantees
199 * cgroups with bigger numbers are newer than those with smaller numbers.
200 * Also, as csses are always appended to the parent's ->children list, it
201 * guarantees that sibling csses are always sorted in the ascending serial
202 * number order on the list. Protected by cgroup_mutex.
794611a1 203 */
0cb51d71 204static u64 css_serial_nr_next = 1;
794611a1 205
cb4a3167
AS
206/*
207 * These bitmask flags indicate whether tasks in the fork and exit paths have
208 * fork/exit handlers to call. This avoids us having to do extra work in the
209 * fork/exit path to check which subsystems have fork/exit callbacks.
ddbcc7e8 210 */
6e5c8307
TH
211static u16 have_fork_callback __read_mostly;
212static u16 have_exit_callback __read_mostly;
213static u16 have_free_callback __read_mostly;
ddbcc7e8 214
7e47682e 215/* Ditto for the can_fork callback. */
6e5c8307 216static u16 have_canfork_callback __read_mostly;
7e47682e 217
67e9c74b 218static struct file_system_type cgroup2_fs_type;
a14c6874
TH
219static struct cftype cgroup_dfl_base_files[];
220static struct cftype cgroup_legacy_base_files[];
628f7cd4 221
6e5c8307 222static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask);
ed27b9f7 223static void css_task_iter_advance(struct css_task_iter *it);
42809dd4 224static int cgroup_destroy_locked(struct cgroup *cgrp);
6cd0f5bb
TH
225static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
226 struct cgroup_subsys *ss);
9d755d33 227static void css_release(struct percpu_ref *ref);
f8f22e53 228static void kill_css(struct cgroup_subsys_state *css);
4df8dc90
TH
229static int cgroup_addrm_files(struct cgroup_subsys_state *css,
230 struct cgroup *cgrp, struct cftype cfts[],
2bb566cb 231 bool is_add);
42809dd4 232
fc5ed1e9
TH
233/**
234 * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
235 * @ssid: subsys ID of interest
236 *
237 * cgroup_subsys_enabled() can only be used with literal subsys names which
238 * is fine for individual subsystems but unsuitable for cgroup core. This
239 * is slower static_key_enabled() based test indexed by @ssid.
240 */
241static bool cgroup_ssid_enabled(int ssid)
242{
243 return static_key_enabled(cgroup_subsys_enabled_key[ssid]);
244}
245
223ffb29
JW
246static bool cgroup_ssid_no_v1(int ssid)
247{
248 return cgroup_no_v1_mask & (1 << ssid);
249}
250
9e10a130
TH
251/**
252 * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
253 * @cgrp: the cgroup of interest
254 *
255 * The default hierarchy is the v2 interface of cgroup and this function
256 * can be used to test whether a cgroup is on the default hierarchy for
257 * cases where a subsystem should behave differnetly depending on the
258 * interface version.
259 *
260 * The set of behaviors which change on the default hierarchy are still
261 * being determined and the mount option is prefixed with __DEVEL__.
262 *
263 * List of changed behaviors:
264 *
265 * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
266 * and "name" are disallowed.
267 *
268 * - When mounting an existing superblock, mount options should match.
269 *
270 * - Remount is disallowed.
271 *
272 * - rename(2) is disallowed.
273 *
274 * - "tasks" is removed. Everything should be at process granularity. Use
275 * "cgroup.procs" instead.
276 *
277 * - "cgroup.procs" is not sorted. pids will be unique unless they got
278 * recycled inbetween reads.
279 *
280 * - "release_agent" and "notify_on_release" are removed. Replacement
281 * notification mechanism will be implemented.
282 *
283 * - "cgroup.clone_children" is removed.
284 *
285 * - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup
286 * and its descendants contain no task; otherwise, 1. The file also
287 * generates kernfs notification which can be monitored through poll and
288 * [di]notify when the value of the file changes.
289 *
290 * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
291 * take masks of ancestors with non-empty cpus/mems, instead of being
292 * moved to an ancestor.
293 *
294 * - cpuset: a task can be moved into an empty cpuset, and again it takes
295 * masks of ancestors.
296 *
297 * - memcg: use_hierarchy is on by default and the cgroup file for the flag
298 * is not created.
299 *
300 * - blkcg: blk-throttle becomes properly hierarchical.
301 *
302 * - debug: disallowed on the default hierarchy.
303 */
304static bool cgroup_on_dfl(const struct cgroup *cgrp)
305{
306 return cgrp->root == &cgrp_dfl_root;
307}
308
6fa4918d
TH
309/* IDR wrappers which synchronize using cgroup_idr_lock */
310static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end,
311 gfp_t gfp_mask)
312{
313 int ret;
314
315 idr_preload(gfp_mask);
54504e97 316 spin_lock_bh(&cgroup_idr_lock);
d0164adc 317 ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM);
54504e97 318 spin_unlock_bh(&cgroup_idr_lock);
6fa4918d
TH
319 idr_preload_end();
320 return ret;
321}
322
323static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id)
324{
325 void *ret;
326
54504e97 327 spin_lock_bh(&cgroup_idr_lock);
6fa4918d 328 ret = idr_replace(idr, ptr, id);
54504e97 329 spin_unlock_bh(&cgroup_idr_lock);
6fa4918d
TH
330 return ret;
331}
332
333static void cgroup_idr_remove(struct idr *idr, int id)
334{
54504e97 335 spin_lock_bh(&cgroup_idr_lock);
6fa4918d 336 idr_remove(idr, id);
54504e97 337 spin_unlock_bh(&cgroup_idr_lock);
6fa4918d
TH
338}
339
d51f39b0
TH
340static struct cgroup *cgroup_parent(struct cgroup *cgrp)
341{
342 struct cgroup_subsys_state *parent_css = cgrp->self.parent;
343
344 if (parent_css)
345 return container_of(parent_css, struct cgroup, self);
346 return NULL;
347}
348
95109b62
TH
349/**
350 * cgroup_css - obtain a cgroup's css for the specified subsystem
351 * @cgrp: the cgroup of interest
9d800df1 352 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
95109b62 353 *
ca8bdcaf
TH
354 * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This
355 * function must be called either under cgroup_mutex or rcu_read_lock() and
356 * the caller is responsible for pinning the returned css if it wants to
357 * keep accessing it outside the said locks. This function may return
358 * %NULL if @cgrp doesn't have @subsys_id enabled.
95109b62
TH
359 */
360static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
ca8bdcaf 361 struct cgroup_subsys *ss)
95109b62 362{
ca8bdcaf 363 if (ss)
aec25020 364 return rcu_dereference_check(cgrp->subsys[ss->id],
ace2bee8 365 lockdep_is_held(&cgroup_mutex));
ca8bdcaf 366 else
9d800df1 367 return &cgrp->self;
95109b62 368}
42809dd4 369
aec3dfcb
TH
370/**
371 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
372 * @cgrp: the cgroup of interest
9d800df1 373 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
aec3dfcb 374 *
d0f702e6 375 * Similar to cgroup_css() but returns the effective css, which is defined
aec3dfcb
TH
376 * as the matching css of the nearest ancestor including self which has @ss
377 * enabled. If @ss is associated with the hierarchy @cgrp is on, this
378 * function is guaranteed to return non-NULL css.
379 */
380static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp,
381 struct cgroup_subsys *ss)
382{
383 lockdep_assert_held(&cgroup_mutex);
384
385 if (!ss)
9d800df1 386 return &cgrp->self;
aec3dfcb
TH
387
388 if (!(cgrp->root->subsys_mask & (1 << ss->id)))
389 return NULL;
390
eeecbd19
TH
391 /*
392 * This function is used while updating css associations and thus
8699b776 393 * can't test the csses directly. Use ->subtree_ss_mask.
eeecbd19 394 */
d51f39b0 395 while (cgroup_parent(cgrp) &&
8699b776 396 !(cgroup_parent(cgrp)->subtree_ss_mask & (1 << ss->id)))
d51f39b0 397 cgrp = cgroup_parent(cgrp);
aec3dfcb
TH
398
399 return cgroup_css(cgrp, ss);
95109b62 400}
42809dd4 401
eeecbd19
TH
402/**
403 * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
404 * @cgrp: the cgroup of interest
405 * @ss: the subsystem of interest
406 *
407 * Find and get the effective css of @cgrp for @ss. The effective css is
408 * defined as the matching css of the nearest ancestor including self which
409 * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on,
410 * the root css is returned, so this function always returns a valid css.
411 * The returned css must be put using css_put().
412 */
413struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp,
414 struct cgroup_subsys *ss)
415{
416 struct cgroup_subsys_state *css;
417
418 rcu_read_lock();
419
420 do {
421 css = cgroup_css(cgrp, ss);
422
423 if (css && css_tryget_online(css))
424 goto out_unlock;
425 cgrp = cgroup_parent(cgrp);
426 } while (cgrp);
427
428 css = init_css_set.subsys[ss->id];
429 css_get(css);
430out_unlock:
431 rcu_read_unlock();
432 return css;
433}
434
ddbcc7e8 435/* convenient tests for these bits */
54766d4a 436static inline bool cgroup_is_dead(const struct cgroup *cgrp)
ddbcc7e8 437{
184faf32 438 return !(cgrp->self.flags & CSS_ONLINE);
ddbcc7e8
PM
439}
440
052c3f3a
TH
441static void cgroup_get(struct cgroup *cgrp)
442{
443 WARN_ON_ONCE(cgroup_is_dead(cgrp));
444 css_get(&cgrp->self);
445}
446
447static bool cgroup_tryget(struct cgroup *cgrp)
448{
449 return css_tryget(&cgrp->self);
450}
451
b4168640 452struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
59f5296b 453{
2bd59d48 454 struct cgroup *cgrp = of->kn->parent->priv;
b4168640 455 struct cftype *cft = of_cft(of);
2bd59d48
TH
456
457 /*
458 * This is open and unprotected implementation of cgroup_css().
459 * seq_css() is only called from a kernfs file operation which has
460 * an active reference on the file. Because all the subsystem
461 * files are drained before a css is disassociated with a cgroup,
462 * the matching css from the cgroup's subsys table is guaranteed to
463 * be and stay valid until the enclosing operation is complete.
464 */
465 if (cft->ss)
466 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]);
467 else
9d800df1 468 return &cgrp->self;
59f5296b 469}
b4168640 470EXPORT_SYMBOL_GPL(of_css);
59f5296b 471
e9685a03 472static int notify_on_release(const struct cgroup *cgrp)
81a6a5cd 473{
bd89aabc 474 return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
81a6a5cd
PM
475}
476
1c6727af
TH
477/**
478 * for_each_css - iterate all css's of a cgroup
479 * @css: the iteration cursor
480 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
481 * @cgrp: the target cgroup to iterate css's of
482 *
aec3dfcb 483 * Should be called under cgroup_[tree_]mutex.
1c6727af
TH
484 */
485#define for_each_css(css, ssid, cgrp) \
486 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
487 if (!((css) = rcu_dereference_check( \
488 (cgrp)->subsys[(ssid)], \
489 lockdep_is_held(&cgroup_mutex)))) { } \
490 else
491
aec3dfcb
TH
492/**
493 * for_each_e_css - iterate all effective css's of a cgroup
494 * @css: the iteration cursor
495 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
496 * @cgrp: the target cgroup to iterate css's of
497 *
498 * Should be called under cgroup_[tree_]mutex.
499 */
500#define for_each_e_css(css, ssid, cgrp) \
501 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
502 if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \
503 ; \
504 else
505
30159ec7 506/**
3ed80a62 507 * for_each_subsys - iterate all enabled cgroup subsystems
30159ec7 508 * @ss: the iteration cursor
780cd8b3 509 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
30159ec7 510 */
780cd8b3 511#define for_each_subsys(ss, ssid) \
3ed80a62
TH
512 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \
513 (((ss) = cgroup_subsys[ssid]) || true); (ssid)++)
30159ec7 514
cb4a3167 515/**
b4e0eeaf 516 * do_each_subsys_mask - filter for_each_subsys with a bitmask
cb4a3167
AS
517 * @ss: the iteration cursor
518 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
b4e0eeaf 519 * @ss_mask: the bitmask
cb4a3167
AS
520 *
521 * The block will only run for cases where the ssid-th bit (1 << ssid) of
b4e0eeaf 522 * @ss_mask is set.
cb4a3167 523 */
b4e0eeaf
TH
524#define do_each_subsys_mask(ss, ssid, ss_mask) do { \
525 unsigned long __ss_mask = (ss_mask); \
526 if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */ \
4a705c5c 527 (ssid) = 0; \
b4e0eeaf
TH
528 break; \
529 } \
530 for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) { \
531 (ss) = cgroup_subsys[ssid]; \
532 {
533
534#define while_each_subsys_mask() \
535 } \
536 } \
537} while (false)
cb4a3167 538
985ed670
TH
539/* iterate across the hierarchies */
540#define for_each_root(root) \
5549c497 541 list_for_each_entry((root), &cgroup_roots, root_list)
ddbcc7e8 542
f8f22e53
TH
543/* iterate over child cgrps, lock should be held throughout iteration */
544#define cgroup_for_each_live_child(child, cgrp) \
d5c419b6 545 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
8353da1f 546 if (({ lockdep_assert_held(&cgroup_mutex); \
f8f22e53
TH
547 cgroup_is_dead(child); })) \
548 ; \
549 else
7ae1bad9 550
81a6a5cd 551static void cgroup_release_agent(struct work_struct *work);
bd89aabc 552static void check_for_release(struct cgroup *cgrp);
81a6a5cd 553
69d0206c
TH
554/*
555 * A cgroup can be associated with multiple css_sets as different tasks may
556 * belong to different cgroups on different hierarchies. In the other
557 * direction, a css_set is naturally associated with multiple cgroups.
558 * This M:N relationship is represented by the following link structure
559 * which exists for each association and allows traversing the associations
560 * from both sides.
561 */
562struct cgrp_cset_link {
563 /* the cgroup and css_set this link associates */
564 struct cgroup *cgrp;
565 struct css_set *cset;
566
567 /* list of cgrp_cset_links anchored at cgrp->cset_links */
568 struct list_head cset_link;
569
570 /* list of cgrp_cset_links anchored at css_set->cgrp_links */
571 struct list_head cgrp_link;
817929ec
PM
572};
573
172a2c06
TH
574/*
575 * The default css_set - used by init and its children prior to any
817929ec
PM
576 * hierarchies being mounted. It contains a pointer to the root state
577 * for each subsystem. Also used to anchor the list of css_sets. Not
578 * reference-counted, to improve performance when child cgroups
579 * haven't been created.
580 */
5024ae29 581struct css_set init_css_set = {
172a2c06
TH
582 .refcount = ATOMIC_INIT(1),
583 .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links),
584 .tasks = LIST_HEAD_INIT(init_css_set.tasks),
585 .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks),
586 .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node),
587 .mg_node = LIST_HEAD_INIT(init_css_set.mg_node),
ed27b9f7 588 .task_iters = LIST_HEAD_INIT(init_css_set.task_iters),
172a2c06 589};
817929ec 590
172a2c06 591static int css_set_count = 1; /* 1 for init_css_set */
817929ec 592
0de0942d
TH
593/**
594 * css_set_populated - does a css_set contain any tasks?
595 * @cset: target css_set
596 */
597static bool css_set_populated(struct css_set *cset)
598{
f0d9a5f1 599 lockdep_assert_held(&css_set_lock);
0de0942d
TH
600
601 return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);
602}
603
842b597e
TH
604/**
605 * cgroup_update_populated - updated populated count of a cgroup
606 * @cgrp: the target cgroup
607 * @populated: inc or dec populated count
608 *
0de0942d
TH
609 * One of the css_sets associated with @cgrp is either getting its first
610 * task or losing the last. Update @cgrp->populated_cnt accordingly. The
611 * count is propagated towards root so that a given cgroup's populated_cnt
612 * is zero iff the cgroup and all its descendants don't contain any tasks.
842b597e
TH
613 *
614 * @cgrp's interface file "cgroup.populated" is zero if
615 * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt
616 * changes from or to zero, userland is notified that the content of the
617 * interface file has changed. This can be used to detect when @cgrp and
618 * its descendants become populated or empty.
619 */
620static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
621{
f0d9a5f1 622 lockdep_assert_held(&css_set_lock);
842b597e
TH
623
624 do {
625 bool trigger;
626
627 if (populated)
628 trigger = !cgrp->populated_cnt++;
629 else
630 trigger = !--cgrp->populated_cnt;
631
632 if (!trigger)
633 break;
634
ad2ed2b3 635 check_for_release(cgrp);
6f60eade
TH
636 cgroup_file_notify(&cgrp->events_file);
637
d51f39b0 638 cgrp = cgroup_parent(cgrp);
842b597e
TH
639 } while (cgrp);
640}
641
0de0942d
TH
642/**
643 * css_set_update_populated - update populated state of a css_set
644 * @cset: target css_set
645 * @populated: whether @cset is populated or depopulated
646 *
647 * @cset is either getting the first task or losing the last. Update the
648 * ->populated_cnt of all associated cgroups accordingly.
649 */
650static void css_set_update_populated(struct css_set *cset, bool populated)
651{
652 struct cgrp_cset_link *link;
653
f0d9a5f1 654 lockdep_assert_held(&css_set_lock);
0de0942d
TH
655
656 list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
657 cgroup_update_populated(link->cgrp, populated);
658}
659
f6d7d049
TH
660/**
661 * css_set_move_task - move a task from one css_set to another
662 * @task: task being moved
663 * @from_cset: css_set @task currently belongs to (may be NULL)
664 * @to_cset: new css_set @task is being moved to (may be NULL)
665 * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
666 *
667 * Move @task from @from_cset to @to_cset. If @task didn't belong to any
668 * css_set, @from_cset can be NULL. If @task is being disassociated
669 * instead of moved, @to_cset can be NULL.
670 *
ed27b9f7
TH
671 * This function automatically handles populated_cnt updates and
672 * css_task_iter adjustments but the caller is responsible for managing
673 * @from_cset and @to_cset's reference counts.
f6d7d049
TH
674 */
675static void css_set_move_task(struct task_struct *task,
676 struct css_set *from_cset, struct css_set *to_cset,
677 bool use_mg_tasks)
678{
f0d9a5f1 679 lockdep_assert_held(&css_set_lock);
f6d7d049 680
20b454a6
TH
681 if (to_cset && !css_set_populated(to_cset))
682 css_set_update_populated(to_cset, true);
683
f6d7d049 684 if (from_cset) {
ed27b9f7
TH
685 struct css_task_iter *it, *pos;
686
f6d7d049 687 WARN_ON_ONCE(list_empty(&task->cg_list));
ed27b9f7
TH
688
689 /*
690 * @task is leaving, advance task iterators which are
691 * pointing to it so that they can resume at the next
692 * position. Advancing an iterator might remove it from
693 * the list, use safe walk. See css_task_iter_advance*()
694 * for details.
695 */
696 list_for_each_entry_safe(it, pos, &from_cset->task_iters,
697 iters_node)
698 if (it->task_pos == &task->cg_list)
699 css_task_iter_advance(it);
700
f6d7d049
TH
701 list_del_init(&task->cg_list);
702 if (!css_set_populated(from_cset))
703 css_set_update_populated(from_cset, false);
704 } else {
705 WARN_ON_ONCE(!list_empty(&task->cg_list));
706 }
707
708 if (to_cset) {
709 /*
710 * We are synchronized through cgroup_threadgroup_rwsem
711 * against PF_EXITING setting such that we can't race
712 * against cgroup_exit() changing the css_set to
713 * init_css_set and dropping the old one.
714 */
715 WARN_ON_ONCE(task->flags & PF_EXITING);
716
f6d7d049
TH
717 rcu_assign_pointer(task->cgroups, to_cset);
718 list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks :
719 &to_cset->tasks);
720 }
721}
722
7717f7ba
PM
723/*
724 * hash table for cgroup groups. This improves the performance to find
725 * an existing css_set. This hash doesn't (currently) take into
726 * account cgroups in empty hierarchies.
727 */
472b1053 728#define CSS_SET_HASH_BITS 7
0ac801fe 729static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS);
472b1053 730
0ac801fe 731static unsigned long css_set_hash(struct cgroup_subsys_state *css[])
472b1053 732{
0ac801fe 733 unsigned long key = 0UL;
30159ec7
TH
734 struct cgroup_subsys *ss;
735 int i;
472b1053 736
30159ec7 737 for_each_subsys(ss, i)
0ac801fe
LZ
738 key += (unsigned long)css[i];
739 key = (key >> 16) ^ key;
472b1053 740
0ac801fe 741 return key;
472b1053
LZ
742}
743
a25eb52e 744static void put_css_set_locked(struct css_set *cset)
b4f48b63 745{
69d0206c 746 struct cgrp_cset_link *link, *tmp_link;
2d8f243a
TH
747 struct cgroup_subsys *ss;
748 int ssid;
5abb8855 749
f0d9a5f1 750 lockdep_assert_held(&css_set_lock);
89c5509b
TH
751
752 if (!atomic_dec_and_test(&cset->refcount))
146aa1bd 753 return;
81a6a5cd 754
53254f90
TH
755 /* This css_set is dead. unlink it and release cgroup and css refs */
756 for_each_subsys(ss, ssid) {
2d8f243a 757 list_del(&cset->e_cset_node[ssid]);
53254f90
TH
758 css_put(cset->subsys[ssid]);
759 }
5abb8855 760 hash_del(&cset->hlist);
2c6ab6d2
PM
761 css_set_count--;
762
69d0206c 763 list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {
69d0206c
TH
764 list_del(&link->cset_link);
765 list_del(&link->cgrp_link);
2ceb231b
TH
766 if (cgroup_parent(link->cgrp))
767 cgroup_put(link->cgrp);
2c6ab6d2 768 kfree(link);
81a6a5cd 769 }
2c6ab6d2 770
5abb8855 771 kfree_rcu(cset, rcu_head);
b4f48b63
PM
772}
773
a25eb52e 774static void put_css_set(struct css_set *cset)
89c5509b
TH
775{
776 /*
777 * Ensure that the refcount doesn't hit zero while any readers
778 * can see it. Similar to atomic_dec_and_lock(), but for an
779 * rwlock
780 */
781 if (atomic_add_unless(&cset->refcount, -1, 1))
782 return;
783
f0d9a5f1 784 spin_lock_bh(&css_set_lock);
a25eb52e 785 put_css_set_locked(cset);
f0d9a5f1 786 spin_unlock_bh(&css_set_lock);
89c5509b
TH
787}
788
817929ec
PM
789/*
790 * refcounted get/put for css_set objects
791 */
5abb8855 792static inline void get_css_set(struct css_set *cset)
817929ec 793{
5abb8855 794 atomic_inc(&cset->refcount);
817929ec
PM
795}
796
b326f9d0 797/**
7717f7ba 798 * compare_css_sets - helper function for find_existing_css_set().
5abb8855
TH
799 * @cset: candidate css_set being tested
800 * @old_cset: existing css_set for a task
7717f7ba
PM
801 * @new_cgrp: cgroup that's being entered by the task
802 * @template: desired set of css pointers in css_set (pre-calculated)
803 *
6f4b7e63 804 * Returns true if "cset" matches "old_cset" except for the hierarchy
7717f7ba
PM
805 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
806 */
5abb8855
TH
807static bool compare_css_sets(struct css_set *cset,
808 struct css_set *old_cset,
7717f7ba
PM
809 struct cgroup *new_cgrp,
810 struct cgroup_subsys_state *template[])
811{
812 struct list_head *l1, *l2;
813
aec3dfcb
TH
814 /*
815 * On the default hierarchy, there can be csets which are
816 * associated with the same set of cgroups but different csses.
817 * Let's first ensure that csses match.
818 */
819 if (memcmp(template, cset->subsys, sizeof(cset->subsys)))
7717f7ba 820 return false;
7717f7ba
PM
821
822 /*
823 * Compare cgroup pointers in order to distinguish between
aec3dfcb
TH
824 * different cgroups in hierarchies. As different cgroups may
825 * share the same effective css, this comparison is always
826 * necessary.
7717f7ba 827 */
69d0206c
TH
828 l1 = &cset->cgrp_links;
829 l2 = &old_cset->cgrp_links;
7717f7ba 830 while (1) {
69d0206c 831 struct cgrp_cset_link *link1, *link2;
5abb8855 832 struct cgroup *cgrp1, *cgrp2;
7717f7ba
PM
833
834 l1 = l1->next;
835 l2 = l2->next;
836 /* See if we reached the end - both lists are equal length. */
69d0206c
TH
837 if (l1 == &cset->cgrp_links) {
838 BUG_ON(l2 != &old_cset->cgrp_links);
7717f7ba
PM
839 break;
840 } else {
69d0206c 841 BUG_ON(l2 == &old_cset->cgrp_links);
7717f7ba
PM
842 }
843 /* Locate the cgroups associated with these links. */
69d0206c
TH
844 link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link);
845 link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link);
846 cgrp1 = link1->cgrp;
847 cgrp2 = link2->cgrp;
7717f7ba 848 /* Hierarchies should be linked in the same order. */
5abb8855 849 BUG_ON(cgrp1->root != cgrp2->root);
7717f7ba
PM
850
851 /*
852 * If this hierarchy is the hierarchy of the cgroup
853 * that's changing, then we need to check that this
854 * css_set points to the new cgroup; if it's any other
855 * hierarchy, then this css_set should point to the
856 * same cgroup as the old css_set.
857 */
5abb8855
TH
858 if (cgrp1->root == new_cgrp->root) {
859 if (cgrp1 != new_cgrp)
7717f7ba
PM
860 return false;
861 } else {
5abb8855 862 if (cgrp1 != cgrp2)
7717f7ba
PM
863 return false;
864 }
865 }
866 return true;
867}
868
b326f9d0
TH
869/**
870 * find_existing_css_set - init css array and find the matching css_set
871 * @old_cset: the css_set that we're using before the cgroup transition
872 * @cgrp: the cgroup that we're moving into
873 * @template: out param for the new set of csses, should be clear on entry
817929ec 874 */
5abb8855
TH
875static struct css_set *find_existing_css_set(struct css_set *old_cset,
876 struct cgroup *cgrp,
877 struct cgroup_subsys_state *template[])
b4f48b63 878{
3dd06ffa 879 struct cgroup_root *root = cgrp->root;
30159ec7 880 struct cgroup_subsys *ss;
5abb8855 881 struct css_set *cset;
0ac801fe 882 unsigned long key;
b326f9d0 883 int i;
817929ec 884
aae8aab4
BB
885 /*
886 * Build the set of subsystem state objects that we want to see in the
887 * new css_set. while subsystems can change globally, the entries here
888 * won't change, so no need for locking.
889 */
30159ec7 890 for_each_subsys(ss, i) {
f392e51c 891 if (root->subsys_mask & (1UL << i)) {
aec3dfcb
TH
892 /*
893 * @ss is in this hierarchy, so we want the
894 * effective css from @cgrp.
895 */
896 template[i] = cgroup_e_css(cgrp, ss);
817929ec 897 } else {
aec3dfcb
TH
898 /*
899 * @ss is not in this hierarchy, so we don't want
900 * to change the css.
901 */
5abb8855 902 template[i] = old_cset->subsys[i];
817929ec
PM
903 }
904 }
905
0ac801fe 906 key = css_set_hash(template);
5abb8855
TH
907 hash_for_each_possible(css_set_table, cset, hlist, key) {
908 if (!compare_css_sets(cset, old_cset, cgrp, template))
7717f7ba
PM
909 continue;
910
911 /* This css_set matches what we need */
5abb8855 912 return cset;
472b1053 913 }
817929ec
PM
914
915 /* No existing cgroup group matched */
916 return NULL;
917}
918
69d0206c 919static void free_cgrp_cset_links(struct list_head *links_to_free)
36553434 920{
69d0206c 921 struct cgrp_cset_link *link, *tmp_link;
36553434 922
69d0206c
TH
923 list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) {
924 list_del(&link->cset_link);
36553434
LZ
925 kfree(link);
926 }
927}
928
69d0206c
TH
929/**
930 * allocate_cgrp_cset_links - allocate cgrp_cset_links
931 * @count: the number of links to allocate
932 * @tmp_links: list_head the allocated links are put on
933 *
934 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
935 * through ->cset_link. Returns 0 on success or -errno.
817929ec 936 */
69d0206c 937static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links)
817929ec 938{
69d0206c 939 struct cgrp_cset_link *link;
817929ec 940 int i;
69d0206c
TH
941
942 INIT_LIST_HEAD(tmp_links);
943
817929ec 944 for (i = 0; i < count; i++) {
f4f4be2b 945 link = kzalloc(sizeof(*link), GFP_KERNEL);
817929ec 946 if (!link) {
69d0206c 947 free_cgrp_cset_links(tmp_links);
817929ec
PM
948 return -ENOMEM;
949 }
69d0206c 950 list_add(&link->cset_link, tmp_links);
817929ec
PM
951 }
952 return 0;
953}
954
c12f65d4
LZ
955/**
956 * link_css_set - a helper function to link a css_set to a cgroup
69d0206c 957 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
5abb8855 958 * @cset: the css_set to be linked
c12f65d4
LZ
959 * @cgrp: the destination cgroup
960 */
69d0206c
TH
961static void link_css_set(struct list_head *tmp_links, struct css_set *cset,
962 struct cgroup *cgrp)
c12f65d4 963{
69d0206c 964 struct cgrp_cset_link *link;
c12f65d4 965
69d0206c 966 BUG_ON(list_empty(tmp_links));
6803c006
TH
967
968 if (cgroup_on_dfl(cgrp))
969 cset->dfl_cgrp = cgrp;
970
69d0206c
TH
971 link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link);
972 link->cset = cset;
7717f7ba 973 link->cgrp = cgrp;
842b597e 974
7717f7ba 975 /*
389b9c1b
TH
976 * Always add links to the tail of the lists so that the lists are
977 * in choronological order.
7717f7ba 978 */
389b9c1b 979 list_move_tail(&link->cset_link, &cgrp->cset_links);
69d0206c 980 list_add_tail(&link->cgrp_link, &cset->cgrp_links);
2ceb231b
TH
981
982 if (cgroup_parent(cgrp))
983 cgroup_get(cgrp);
c12f65d4
LZ
984}
985
b326f9d0
TH
986/**
987 * find_css_set - return a new css_set with one cgroup updated
988 * @old_cset: the baseline css_set
989 * @cgrp: the cgroup to be updated
990 *
991 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
992 * substituted into the appropriate hierarchy.
817929ec 993 */
5abb8855
TH
994static struct css_set *find_css_set(struct css_set *old_cset,
995 struct cgroup *cgrp)
817929ec 996{
b326f9d0 997 struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { };
5abb8855 998 struct css_set *cset;
69d0206c
TH
999 struct list_head tmp_links;
1000 struct cgrp_cset_link *link;
2d8f243a 1001 struct cgroup_subsys *ss;
0ac801fe 1002 unsigned long key;
2d8f243a 1003 int ssid;
472b1053 1004
b326f9d0
TH
1005 lockdep_assert_held(&cgroup_mutex);
1006
817929ec
PM
1007 /* First see if we already have a cgroup group that matches
1008 * the desired set */
f0d9a5f1 1009 spin_lock_bh(&css_set_lock);
5abb8855
TH
1010 cset = find_existing_css_set(old_cset, cgrp, template);
1011 if (cset)
1012 get_css_set(cset);
f0d9a5f1 1013 spin_unlock_bh(&css_set_lock);
817929ec 1014
5abb8855
TH
1015 if (cset)
1016 return cset;
817929ec 1017
f4f4be2b 1018 cset = kzalloc(sizeof(*cset), GFP_KERNEL);
5abb8855 1019 if (!cset)
817929ec
PM
1020 return NULL;
1021
69d0206c 1022 /* Allocate all the cgrp_cset_link objects that we'll need */
9871bf95 1023 if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) {
5abb8855 1024 kfree(cset);
817929ec
PM
1025 return NULL;
1026 }
1027
5abb8855 1028 atomic_set(&cset->refcount, 1);
69d0206c 1029 INIT_LIST_HEAD(&cset->cgrp_links);
5abb8855 1030 INIT_LIST_HEAD(&cset->tasks);
c7561128 1031 INIT_LIST_HEAD(&cset->mg_tasks);
1958d2d5 1032 INIT_LIST_HEAD(&cset->mg_preload_node);
b3dc094e 1033 INIT_LIST_HEAD(&cset->mg_node);
ed27b9f7 1034 INIT_LIST_HEAD(&cset->task_iters);
5abb8855 1035 INIT_HLIST_NODE(&cset->hlist);
817929ec
PM
1036
1037 /* Copy the set of subsystem state objects generated in
1038 * find_existing_css_set() */
5abb8855 1039 memcpy(cset->subsys, template, sizeof(cset->subsys));
817929ec 1040
f0d9a5f1 1041 spin_lock_bh(&css_set_lock);
817929ec 1042 /* Add reference counts and links from the new css_set. */
69d0206c 1043 list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) {
7717f7ba 1044 struct cgroup *c = link->cgrp;
69d0206c 1045
7717f7ba
PM
1046 if (c->root == cgrp->root)
1047 c = cgrp;
69d0206c 1048 link_css_set(&tmp_links, cset, c);
7717f7ba 1049 }
817929ec 1050
69d0206c 1051 BUG_ON(!list_empty(&tmp_links));
817929ec 1052
817929ec 1053 css_set_count++;
472b1053 1054
2d8f243a 1055 /* Add @cset to the hash table */
5abb8855
TH
1056 key = css_set_hash(cset->subsys);
1057 hash_add(css_set_table, &cset->hlist, key);
472b1053 1058
53254f90
TH
1059 for_each_subsys(ss, ssid) {
1060 struct cgroup_subsys_state *css = cset->subsys[ssid];
1061
2d8f243a 1062 list_add_tail(&cset->e_cset_node[ssid],
53254f90
TH
1063 &css->cgroup->e_csets[ssid]);
1064 css_get(css);
1065 }
2d8f243a 1066
f0d9a5f1 1067 spin_unlock_bh(&css_set_lock);
817929ec 1068
5abb8855 1069 return cset;
b4f48b63
PM
1070}
1071
3dd06ffa 1072static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root)
7717f7ba 1073{
3dd06ffa 1074 struct cgroup *root_cgrp = kf_root->kn->priv;
2bd59d48 1075
3dd06ffa 1076 return root_cgrp->root;
2bd59d48
TH
1077}
1078
3dd06ffa 1079static int cgroup_init_root_id(struct cgroup_root *root)
f2e85d57
TH
1080{
1081 int id;
1082
1083 lockdep_assert_held(&cgroup_mutex);
1084
985ed670 1085 id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL);
f2e85d57
TH
1086 if (id < 0)
1087 return id;
1088
1089 root->hierarchy_id = id;
1090 return 0;
1091}
1092
3dd06ffa 1093static void cgroup_exit_root_id(struct cgroup_root *root)
f2e85d57
TH
1094{
1095 lockdep_assert_held(&cgroup_mutex);
1096
1097 if (root->hierarchy_id) {
1098 idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id);
1099 root->hierarchy_id = 0;
1100 }
1101}
1102
3dd06ffa 1103static void cgroup_free_root(struct cgroup_root *root)
f2e85d57
TH
1104{
1105 if (root) {
d0f702e6 1106 /* hierarchy ID should already have been released */
f2e85d57
TH
1107 WARN_ON_ONCE(root->hierarchy_id);
1108
1109 idr_destroy(&root->cgroup_idr);
1110 kfree(root);
1111 }
1112}
1113
3dd06ffa 1114static void cgroup_destroy_root(struct cgroup_root *root)
59f5296b 1115{
3dd06ffa 1116 struct cgroup *cgrp = &root->cgrp;
f2e85d57 1117 struct cgrp_cset_link *link, *tmp_link;
f2e85d57 1118
2bd59d48 1119 mutex_lock(&cgroup_mutex);
f2e85d57 1120
776f02fa 1121 BUG_ON(atomic_read(&root->nr_cgrps));
d5c419b6 1122 BUG_ON(!list_empty(&cgrp->self.children));
f2e85d57 1123
f2e85d57 1124 /* Rebind all subsystems back to the default hierarchy */
f392e51c 1125 rebind_subsystems(&cgrp_dfl_root, root->subsys_mask);
7717f7ba 1126
7717f7ba 1127 /*
f2e85d57
TH
1128 * Release all the links from cset_links to this hierarchy's
1129 * root cgroup
7717f7ba 1130 */
f0d9a5f1 1131 spin_lock_bh(&css_set_lock);
f2e85d57
TH
1132
1133 list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) {
1134 list_del(&link->cset_link);
1135 list_del(&link->cgrp_link);
1136 kfree(link);
1137 }
f0d9a5f1
TH
1138
1139 spin_unlock_bh(&css_set_lock);
f2e85d57
TH
1140
1141 if (!list_empty(&root->root_list)) {
1142 list_del(&root->root_list);
1143 cgroup_root_count--;
1144 }
1145
1146 cgroup_exit_root_id(root);
1147
1148 mutex_unlock(&cgroup_mutex);
f2e85d57 1149
2bd59d48 1150 kernfs_destroy_root(root->kf_root);
f2e85d57
TH
1151 cgroup_free_root(root);
1152}
1153
ceb6a081
TH
1154/* look up cgroup associated with given css_set on the specified hierarchy */
1155static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
3dd06ffa 1156 struct cgroup_root *root)
7717f7ba 1157{
7717f7ba
PM
1158 struct cgroup *res = NULL;
1159
96d365e0 1160 lockdep_assert_held(&cgroup_mutex);
f0d9a5f1 1161 lockdep_assert_held(&css_set_lock);
96d365e0 1162
5abb8855 1163 if (cset == &init_css_set) {
3dd06ffa 1164 res = &root->cgrp;
7717f7ba 1165 } else {
69d0206c
TH
1166 struct cgrp_cset_link *link;
1167
1168 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
7717f7ba 1169 struct cgroup *c = link->cgrp;
69d0206c 1170
7717f7ba
PM
1171 if (c->root == root) {
1172 res = c;
1173 break;
1174 }
1175 }
1176 }
96d365e0 1177
7717f7ba
PM
1178 BUG_ON(!res);
1179 return res;
1180}
1181
ddbcc7e8 1182/*
ceb6a081 1183 * Return the cgroup for "task" from the given hierarchy. Must be
f0d9a5f1 1184 * called with cgroup_mutex and css_set_lock held.
ceb6a081
TH
1185 */
1186static struct cgroup *task_cgroup_from_root(struct task_struct *task,
3dd06ffa 1187 struct cgroup_root *root)
ceb6a081
TH
1188{
1189 /*
1190 * No need to lock the task - since we hold cgroup_mutex the
1191 * task can't change groups, so the only thing that can happen
1192 * is that it exits and its css is set back to init_css_set.
1193 */
1194 return cset_cgroup_from_root(task_css_set(task), root);
1195}
1196
ddbcc7e8 1197/*
ddbcc7e8
PM
1198 * A task must hold cgroup_mutex to modify cgroups.
1199 *
1200 * Any task can increment and decrement the count field without lock.
1201 * So in general, code holding cgroup_mutex can't rely on the count
1202 * field not changing. However, if the count goes to zero, then only
956db3ca 1203 * cgroup_attach_task() can increment it again. Because a count of zero
ddbcc7e8
PM
1204 * means that no tasks are currently attached, therefore there is no
1205 * way a task attached to that cgroup can fork (the other way to
1206 * increment the count). So code holding cgroup_mutex can safely
1207 * assume that if the count is zero, it will stay zero. Similarly, if
1208 * a task holds cgroup_mutex on a cgroup with zero count, it
1209 * knows that the cgroup won't be removed, as cgroup_rmdir()
1210 * needs that mutex.
1211 *
ddbcc7e8
PM
1212 * A cgroup can only be deleted if both its 'count' of using tasks
1213 * is zero, and its list of 'children' cgroups is empty. Since all
1214 * tasks in the system use _some_ cgroup, and since there is always at
3dd06ffa 1215 * least one task in the system (init, pid == 1), therefore, root cgroup
ddbcc7e8 1216 * always has either children cgroups and/or using tasks. So we don't
3dd06ffa 1217 * need a special hack to ensure that root cgroup cannot be deleted.
ddbcc7e8
PM
1218 *
1219 * P.S. One more locking exception. RCU is used to guard the
956db3ca 1220 * update of a tasks cgroup pointer by cgroup_attach_task()
ddbcc7e8
PM
1221 */
1222
2bd59d48 1223static struct kernfs_syscall_ops cgroup_kf_syscall_ops;
828c0950 1224static const struct file_operations proc_cgroupstats_operations;
a424316c 1225
8d7e6fb0
TH
1226static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft,
1227 char *buf)
ddbcc7e8 1228{
3e1d2eed
TH
1229 struct cgroup_subsys *ss = cft->ss;
1230
8d7e6fb0
TH
1231 if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) &&
1232 !(cgrp->root->flags & CGRP_ROOT_NOPREFIX))
1233 snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s",
3e1d2eed
TH
1234 cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name,
1235 cft->name);
8d7e6fb0
TH
1236 else
1237 strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX);
1238 return buf;
ddbcc7e8
PM
1239}
1240
f2e85d57
TH
1241/**
1242 * cgroup_file_mode - deduce file mode of a control file
1243 * @cft: the control file in question
1244 *
7dbdb199 1245 * S_IRUGO for read, S_IWUSR for write.
f2e85d57
TH
1246 */
1247static umode_t cgroup_file_mode(const struct cftype *cft)
65dff759 1248{
f2e85d57 1249 umode_t mode = 0;
65dff759 1250
f2e85d57
TH
1251 if (cft->read_u64 || cft->read_s64 || cft->seq_show)
1252 mode |= S_IRUGO;
1253
7dbdb199
TH
1254 if (cft->write_u64 || cft->write_s64 || cft->write) {
1255 if (cft->flags & CFTYPE_WORLD_WRITABLE)
1256 mode |= S_IWUGO;
1257 else
1258 mode |= S_IWUSR;
1259 }
f2e85d57
TH
1260
1261 return mode;
65dff759
LZ
1262}
1263
af0ba678 1264/**
8699b776 1265 * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
af0ba678 1266 * @cgrp: the target cgroup
0f060deb 1267 * @subtree_control: the new subtree_control mask to consider
af0ba678
TH
1268 *
1269 * On the default hierarchy, a subsystem may request other subsystems to be
1270 * enabled together through its ->depends_on mask. In such cases, more
1271 * subsystems than specified in "cgroup.subtree_control" may be enabled.
1272 *
0f060deb
TH
1273 * This function calculates which subsystems need to be enabled if
1274 * @subtree_control is to be applied to @cgrp. The returned mask is always
1275 * a superset of @subtree_control and follows the usual hierarchy rules.
af0ba678 1276 */
6e5c8307 1277static u16 cgroup_calc_subtree_ss_mask(struct cgroup *cgrp, u16 subtree_control)
667c2491 1278{
af0ba678 1279 struct cgroup *parent = cgroup_parent(cgrp);
6e5c8307 1280 u16 cur_ss_mask = subtree_control;
af0ba678
TH
1281 struct cgroup_subsys *ss;
1282 int ssid;
1283
1284 lockdep_assert_held(&cgroup_mutex);
1285
0f060deb
TH
1286 if (!cgroup_on_dfl(cgrp))
1287 return cur_ss_mask;
af0ba678
TH
1288
1289 while (true) {
6e5c8307 1290 u16 new_ss_mask = cur_ss_mask;
af0ba678 1291
b4e0eeaf 1292 do_each_subsys_mask(ss, ssid, cur_ss_mask) {
a966a4ed 1293 new_ss_mask |= ss->depends_on;
b4e0eeaf 1294 } while_each_subsys_mask();
af0ba678
TH
1295
1296 /*
1297 * Mask out subsystems which aren't available. This can
1298 * happen only if some depended-upon subsystems were bound
1299 * to non-default hierarchies.
1300 */
1301 if (parent)
8699b776 1302 new_ss_mask &= parent->subtree_ss_mask;
af0ba678
TH
1303 else
1304 new_ss_mask &= cgrp->root->subsys_mask;
1305
1306 if (new_ss_mask == cur_ss_mask)
1307 break;
1308 cur_ss_mask = new_ss_mask;
1309 }
1310
0f060deb
TH
1311 return cur_ss_mask;
1312}
1313
1314/**
8699b776 1315 * cgroup_refresh_subtree_ss_mask - update subtree_ss_mask
0f060deb
TH
1316 * @cgrp: the target cgroup
1317 *
8699b776
TH
1318 * Update @cgrp->subtree_ss_mask according to the current
1319 * @cgrp->subtree_control using cgroup_calc_subtree_ss_mask().
0f060deb 1320 */
8699b776 1321static void cgroup_refresh_subtree_ss_mask(struct cgroup *cgrp)
0f060deb 1322{
8699b776
TH
1323 cgrp->subtree_ss_mask =
1324 cgroup_calc_subtree_ss_mask(cgrp, cgrp->subtree_control);
667c2491
TH
1325}
1326
a9746d8d
TH
1327/**
1328 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1329 * @kn: the kernfs_node being serviced
1330 *
1331 * This helper undoes cgroup_kn_lock_live() and should be invoked before
1332 * the method finishes if locking succeeded. Note that once this function
1333 * returns the cgroup returned by cgroup_kn_lock_live() may become
1334 * inaccessible any time. If the caller intends to continue to access the
1335 * cgroup, it should pin it before invoking this function.
1336 */
1337static void cgroup_kn_unlock(struct kernfs_node *kn)
ddbcc7e8 1338{
a9746d8d
TH
1339 struct cgroup *cgrp;
1340
1341 if (kernfs_type(kn) == KERNFS_DIR)
1342 cgrp = kn->priv;
1343 else
1344 cgrp = kn->parent->priv;
1345
1346 mutex_unlock(&cgroup_mutex);
a9746d8d
TH
1347
1348 kernfs_unbreak_active_protection(kn);
1349 cgroup_put(cgrp);
ddbcc7e8
PM
1350}
1351
a9746d8d
TH
1352/**
1353 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1354 * @kn: the kernfs_node being serviced
1355 *
1356 * This helper is to be used by a cgroup kernfs method currently servicing
1357 * @kn. It breaks the active protection, performs cgroup locking and
1358 * verifies that the associated cgroup is alive. Returns the cgroup if
1359 * alive; otherwise, %NULL. A successful return should be undone by a
1360 * matching cgroup_kn_unlock() invocation.
1361 *
1362 * Any cgroup kernfs method implementation which requires locking the
1363 * associated cgroup should use this helper. It avoids nesting cgroup
1364 * locking under kernfs active protection and allows all kernfs operations
1365 * including self-removal.
1366 */
1367static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn)
05ef1d7c 1368{
a9746d8d
TH
1369 struct cgroup *cgrp;
1370
1371 if (kernfs_type(kn) == KERNFS_DIR)
1372 cgrp = kn->priv;
1373 else
1374 cgrp = kn->parent->priv;
05ef1d7c 1375
2739d3cc 1376 /*
01f6474c 1377 * We're gonna grab cgroup_mutex which nests outside kernfs
a9746d8d
TH
1378 * active_ref. cgroup liveliness check alone provides enough
1379 * protection against removal. Ensure @cgrp stays accessible and
1380 * break the active_ref protection.
2739d3cc 1381 */
aa32362f
LZ
1382 if (!cgroup_tryget(cgrp))
1383 return NULL;
a9746d8d
TH
1384 kernfs_break_active_protection(kn);
1385
2bd59d48 1386 mutex_lock(&cgroup_mutex);
05ef1d7c 1387
a9746d8d
TH
1388 if (!cgroup_is_dead(cgrp))
1389 return cgrp;
1390
1391 cgroup_kn_unlock(kn);
1392 return NULL;
ddbcc7e8 1393}
05ef1d7c 1394
2739d3cc 1395static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
05ef1d7c 1396{
2bd59d48 1397 char name[CGROUP_FILE_NAME_MAX];
05ef1d7c 1398
01f6474c 1399 lockdep_assert_held(&cgroup_mutex);
34c06254
TH
1400
1401 if (cft->file_offset) {
1402 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
1403 struct cgroup_file *cfile = (void *)css + cft->file_offset;
1404
1405 spin_lock_irq(&cgroup_file_kn_lock);
1406 cfile->kn = NULL;
1407 spin_unlock_irq(&cgroup_file_kn_lock);
1408 }
1409
2bd59d48 1410 kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name));
05ef1d7c
TH
1411}
1412
13af07df 1413/**
4df8dc90
TH
1414 * css_clear_dir - remove subsys files in a cgroup directory
1415 * @css: taget css
1416 * @cgrp_override: specify if target cgroup is different from css->cgroup
13af07df 1417 */
4df8dc90
TH
1418static void css_clear_dir(struct cgroup_subsys_state *css,
1419 struct cgroup *cgrp_override)
05ef1d7c 1420{
4df8dc90
TH
1421 struct cgroup *cgrp = cgrp_override ?: css->cgroup;
1422 struct cftype *cfts;
05ef1d7c 1423
88cb04b9
TH
1424 if (!(css->flags & CSS_VISIBLE))
1425 return;
1426
1427 css->flags &= ~CSS_VISIBLE;
1428
4df8dc90
TH
1429 list_for_each_entry(cfts, &css->ss->cfts, node)
1430 cgroup_addrm_files(css, cgrp, cfts, false);
ddbcc7e8
PM
1431}
1432
ccdca218 1433/**
4df8dc90
TH
1434 * css_populate_dir - create subsys files in a cgroup directory
1435 * @css: target css
1436 * @cgrp_overried: specify if target cgroup is different from css->cgroup
ccdca218
TH
1437 *
1438 * On failure, no file is added.
1439 */
4df8dc90
TH
1440static int css_populate_dir(struct cgroup_subsys_state *css,
1441 struct cgroup *cgrp_override)
ccdca218 1442{
4df8dc90
TH
1443 struct cgroup *cgrp = cgrp_override ?: css->cgroup;
1444 struct cftype *cfts, *failed_cfts;
1445 int ret;
ccdca218 1446
88cb04b9
TH
1447 if (css->flags & CSS_VISIBLE)
1448 return 0;
1449
4df8dc90
TH
1450 if (!css->ss) {
1451 if (cgroup_on_dfl(cgrp))
1452 cfts = cgroup_dfl_base_files;
1453 else
1454 cfts = cgroup_legacy_base_files;
ccdca218 1455
4df8dc90
TH
1456 return cgroup_addrm_files(&cgrp->self, cgrp, cfts, true);
1457 }
ccdca218 1458
4df8dc90
TH
1459 list_for_each_entry(cfts, &css->ss->cfts, node) {
1460 ret = cgroup_addrm_files(css, cgrp, cfts, true);
1461 if (ret < 0) {
1462 failed_cfts = cfts;
1463 goto err;
ccdca218
TH
1464 }
1465 }
88cb04b9
TH
1466
1467 css->flags |= CSS_VISIBLE;
1468
ccdca218
TH
1469 return 0;
1470err:
4df8dc90
TH
1471 list_for_each_entry(cfts, &css->ss->cfts, node) {
1472 if (cfts == failed_cfts)
1473 break;
1474 cgroup_addrm_files(css, cgrp, cfts, false);
1475 }
ccdca218
TH
1476 return ret;
1477}
1478
6e5c8307 1479static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask)
ddbcc7e8 1480{
1ada4838 1481 struct cgroup *dcgrp = &dst_root->cgrp;
30159ec7 1482 struct cgroup_subsys *ss;
6e5c8307 1483 u16 tmp_ss_mask;
2d8f243a 1484 int ssid, i, ret;
ddbcc7e8 1485
ace2bee8 1486 lockdep_assert_held(&cgroup_mutex);
ddbcc7e8 1487
b4e0eeaf 1488 do_each_subsys_mask(ss, ssid, ss_mask) {
7fd8c565
TH
1489 /* if @ss has non-root csses attached to it, can't move */
1490 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)))
3ed80a62 1491 return -EBUSY;
1d5be6b2 1492
5df36032 1493 /* can't move between two non-dummy roots either */
7fd8c565 1494 if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root)
5df36032 1495 return -EBUSY;
b4e0eeaf 1496 } while_each_subsys_mask();
ddbcc7e8 1497
5533e011
TH
1498 /* skip creating root files on dfl_root for inhibited subsystems */
1499 tmp_ss_mask = ss_mask;
1500 if (dst_root == &cgrp_dfl_root)
a7165264 1501 tmp_ss_mask &= ~cgrp_dfl_inhibit_ss_mask;
5533e011 1502
b4e0eeaf 1503 do_each_subsys_mask(ss, ssid, tmp_ss_mask) {
4df8dc90
TH
1504 struct cgroup *scgrp = &ss->root->cgrp;
1505 int tssid;
1506
1507 ret = css_populate_dir(cgroup_css(scgrp, ss), dcgrp);
1508 if (!ret)
1509 continue;
ddbcc7e8 1510
a2dd4247
TH
1511 /*
1512 * Rebinding back to the default root is not allowed to
1513 * fail. Using both default and non-default roots should
1514 * be rare. Moving subsystems back and forth even more so.
1515 * Just warn about it and continue.
1516 */
4df8dc90 1517 if (dst_root == &cgrp_dfl_root) {
a7165264 1518 if (cgrp_dfl_visible) {
6e5c8307 1519 pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n",
4df8dc90
TH
1520 ret, ss_mask);
1521 pr_warn("you may retry by moving them to a different hierarchy and unbinding\n");
1522 }
1523 continue;
a2dd4247 1524 }
4df8dc90 1525
b4e0eeaf 1526 do_each_subsys_mask(ss, tssid, tmp_ss_mask) {
4df8dc90
TH
1527 if (tssid == ssid)
1528 break;
1529 css_clear_dir(cgroup_css(scgrp, ss), dcgrp);
b4e0eeaf 1530 } while_each_subsys_mask();
4df8dc90 1531 return ret;
b4e0eeaf 1532 } while_each_subsys_mask();
3126121f
TH
1533
1534 /*
1535 * Nothing can fail from this point on. Remove files for the
1536 * removed subsystems and rebind each subsystem.
1537 */
b4e0eeaf 1538 do_each_subsys_mask(ss, ssid, ss_mask) {
1ada4838
TH
1539 struct cgroup_root *src_root = ss->root;
1540 struct cgroup *scgrp = &src_root->cgrp;
1541 struct cgroup_subsys_state *css = cgroup_css(scgrp, ss);
2d8f243a 1542 struct css_set *cset;
a8a648c4 1543
1ada4838 1544 WARN_ON(!css || cgroup_css(dcgrp, ss));
a8a648c4 1545
4df8dc90
TH
1546 css_clear_dir(css, NULL);
1547
1ada4838
TH
1548 RCU_INIT_POINTER(scgrp->subsys[ssid], NULL);
1549 rcu_assign_pointer(dcgrp->subsys[ssid], css);
5df36032 1550 ss->root = dst_root;
1ada4838 1551 css->cgroup = dcgrp;
73e80ed8 1552
f0d9a5f1 1553 spin_lock_bh(&css_set_lock);
2d8f243a
TH
1554 hash_for_each(css_set_table, i, cset, hlist)
1555 list_move_tail(&cset->e_cset_node[ss->id],
1ada4838 1556 &dcgrp->e_csets[ss->id]);
f0d9a5f1 1557 spin_unlock_bh(&css_set_lock);
2d8f243a 1558
f392e51c 1559 src_root->subsys_mask &= ~(1 << ssid);
1ada4838 1560 scgrp->subtree_control &= ~(1 << ssid);
8699b776 1561 cgroup_refresh_subtree_ss_mask(scgrp);
f392e51c 1562
bd53d617 1563 /* default hierarchy doesn't enable controllers by default */
f392e51c 1564 dst_root->subsys_mask |= 1 << ssid;
49d1dc4b
TH
1565 if (dst_root == &cgrp_dfl_root) {
1566 static_branch_enable(cgroup_subsys_on_dfl_key[ssid]);
1567 } else {
1ada4838 1568 dcgrp->subtree_control |= 1 << ssid;
8699b776 1569 cgroup_refresh_subtree_ss_mask(dcgrp);
49d1dc4b 1570 static_branch_disable(cgroup_subsys_on_dfl_key[ssid]);
667c2491 1571 }
a8a648c4 1572
5df36032
TH
1573 if (ss->bind)
1574 ss->bind(css);
b4e0eeaf 1575 } while_each_subsys_mask();
ddbcc7e8 1576
1ada4838 1577 kernfs_activate(dcgrp->kn);
ddbcc7e8
PM
1578 return 0;
1579}
1580
2bd59d48
TH
1581static int cgroup_show_options(struct seq_file *seq,
1582 struct kernfs_root *kf_root)
ddbcc7e8 1583{
3dd06ffa 1584 struct cgroup_root *root = cgroup_root_from_kf(kf_root);
ddbcc7e8 1585 struct cgroup_subsys *ss;
b85d2040 1586 int ssid;
ddbcc7e8 1587
d98817d4
TH
1588 if (root != &cgrp_dfl_root)
1589 for_each_subsys(ss, ssid)
1590 if (root->subsys_mask & (1 << ssid))
61e57c0c 1591 seq_show_option(seq, ss->legacy_name, NULL);
93438629 1592 if (root->flags & CGRP_ROOT_NOPREFIX)
ddbcc7e8 1593 seq_puts(seq, ",noprefix");
93438629 1594 if (root->flags & CGRP_ROOT_XATTR)
03b1cde6 1595 seq_puts(seq, ",xattr");
69e943b7
TH
1596
1597 spin_lock(&release_agent_path_lock);
81a6a5cd 1598 if (strlen(root->release_agent_path))
a068acf2
KC
1599 seq_show_option(seq, "release_agent",
1600 root->release_agent_path);
69e943b7
TH
1601 spin_unlock(&release_agent_path_lock);
1602
3dd06ffa 1603 if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags))
97978e6d 1604 seq_puts(seq, ",clone_children");
c6d57f33 1605 if (strlen(root->name))
a068acf2 1606 seq_show_option(seq, "name", root->name);
ddbcc7e8
PM
1607 return 0;
1608}
1609
1610struct cgroup_sb_opts {
6e5c8307 1611 u16 subsys_mask;
69dfa00c 1612 unsigned int flags;
81a6a5cd 1613 char *release_agent;
2260e7fc 1614 bool cpuset_clone_children;
c6d57f33 1615 char *name;
2c6ab6d2
PM
1616 /* User explicitly requested empty subsystem */
1617 bool none;
ddbcc7e8
PM
1618};
1619
cf5d5941 1620static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts)
ddbcc7e8 1621{
32a8cf23
DL
1622 char *token, *o = data;
1623 bool all_ss = false, one_ss = false;
6e5c8307 1624 u16 mask = U16_MAX;
30159ec7 1625 struct cgroup_subsys *ss;
7b9a6ba5 1626 int nr_opts = 0;
30159ec7 1627 int i;
f9ab5b5b
LZ
1628
1629#ifdef CONFIG_CPUSETS
6e5c8307 1630 mask = ~((u16)1 << cpuset_cgrp_id);
f9ab5b5b 1631#endif
ddbcc7e8 1632
c6d57f33 1633 memset(opts, 0, sizeof(*opts));
ddbcc7e8
PM
1634
1635 while ((token = strsep(&o, ",")) != NULL) {
7b9a6ba5
TH
1636 nr_opts++;
1637
ddbcc7e8
PM
1638 if (!*token)
1639 return -EINVAL;
32a8cf23 1640 if (!strcmp(token, "none")) {
2c6ab6d2
PM
1641 /* Explicitly have no subsystems */
1642 opts->none = true;
32a8cf23
DL
1643 continue;
1644 }
1645 if (!strcmp(token, "all")) {
1646 /* Mutually exclusive option 'all' + subsystem name */
1647 if (one_ss)
1648 return -EINVAL;
1649 all_ss = true;
1650 continue;
1651 }
1652 if (!strcmp(token, "noprefix")) {
93438629 1653 opts->flags |= CGRP_ROOT_NOPREFIX;
32a8cf23
DL
1654 continue;
1655 }
1656 if (!strcmp(token, "clone_children")) {
2260e7fc 1657 opts->cpuset_clone_children = true;
32a8cf23
DL
1658 continue;
1659 }
03b1cde6 1660 if (!strcmp(token, "xattr")) {
93438629 1661 opts->flags |= CGRP_ROOT_XATTR;
03b1cde6
AR
1662 continue;
1663 }
32a8cf23 1664 if (!strncmp(token, "release_agent=", 14)) {
81a6a5cd
PM
1665 /* Specifying two release agents is forbidden */
1666 if (opts->release_agent)
1667 return -EINVAL;
c6d57f33 1668 opts->release_agent =
e400c285 1669 kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL);
81a6a5cd
PM
1670 if (!opts->release_agent)
1671 return -ENOMEM;
32a8cf23
DL
1672 continue;
1673 }
1674 if (!strncmp(token, "name=", 5)) {
c6d57f33
PM
1675 const char *name = token + 5;
1676 /* Can't specify an empty name */
1677 if (!strlen(name))
1678 return -EINVAL;
1679 /* Must match [\w.-]+ */
1680 for (i = 0; i < strlen(name); i++) {
1681 char c = name[i];
1682 if (isalnum(c))
1683 continue;
1684 if ((c == '.') || (c == '-') || (c == '_'))
1685 continue;
1686 return -EINVAL;
1687 }
1688 /* Specifying two names is forbidden */
1689 if (opts->name)
1690 return -EINVAL;
1691 opts->name = kstrndup(name,
e400c285 1692 MAX_CGROUP_ROOT_NAMELEN - 1,
c6d57f33
PM
1693 GFP_KERNEL);
1694 if (!opts->name)
1695 return -ENOMEM;
32a8cf23
DL
1696
1697 continue;
1698 }
1699
30159ec7 1700 for_each_subsys(ss, i) {
3e1d2eed 1701 if (strcmp(token, ss->legacy_name))
32a8cf23 1702 continue;
fc5ed1e9 1703 if (!cgroup_ssid_enabled(i))
32a8cf23 1704 continue;
223ffb29
JW
1705 if (cgroup_ssid_no_v1(i))
1706 continue;
32a8cf23
DL
1707
1708 /* Mutually exclusive option 'all' + subsystem name */
1709 if (all_ss)
1710 return -EINVAL;
69dfa00c 1711 opts->subsys_mask |= (1 << i);
32a8cf23
DL
1712 one_ss = true;
1713
1714 break;
1715 }
1716 if (i == CGROUP_SUBSYS_COUNT)
1717 return -ENOENT;
1718 }
1719
7b9a6ba5
TH
1720 /*
1721 * If the 'all' option was specified select all the subsystems,
1722 * otherwise if 'none', 'name=' and a subsystem name options were
1723 * not specified, let's default to 'all'
1724 */
1725 if (all_ss || (!one_ss && !opts->none && !opts->name))
1726 for_each_subsys(ss, i)
223ffb29 1727 if (cgroup_ssid_enabled(i) && !cgroup_ssid_no_v1(i))
7b9a6ba5
TH
1728 opts->subsys_mask |= (1 << i);
1729
1730 /*
1731 * We either have to specify by name or by subsystems. (So all
1732 * empty hierarchies must have a name).
1733 */
1734 if (!opts->subsys_mask && !opts->name)
1735 return -EINVAL;
1736
f9ab5b5b
LZ
1737 /*
1738 * Option noprefix was introduced just for backward compatibility
1739 * with the old cpuset, so we allow noprefix only if mounting just
1740 * the cpuset subsystem.
1741 */
93438629 1742 if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask))
f9ab5b5b
LZ
1743 return -EINVAL;
1744
2c6ab6d2 1745 /* Can't specify "none" and some subsystems */
a1a71b45 1746 if (opts->subsys_mask && opts->none)
2c6ab6d2
PM
1747 return -EINVAL;
1748
ddbcc7e8
PM
1749 return 0;
1750}
1751
2bd59d48 1752static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data)
ddbcc7e8
PM
1753{
1754 int ret = 0;
3dd06ffa 1755 struct cgroup_root *root = cgroup_root_from_kf(kf_root);
ddbcc7e8 1756 struct cgroup_sb_opts opts;
6e5c8307 1757 u16 added_mask, removed_mask;
ddbcc7e8 1758
aa6ec29b
TH
1759 if (root == &cgrp_dfl_root) {
1760 pr_err("remount is not allowed\n");
873fe09e
TH
1761 return -EINVAL;
1762 }
1763
ddbcc7e8
PM
1764 mutex_lock(&cgroup_mutex);
1765
1766 /* See what subsystems are wanted */
1767 ret = parse_cgroupfs_options(data, &opts);
1768 if (ret)
1769 goto out_unlock;
1770
f392e51c 1771 if (opts.subsys_mask != root->subsys_mask || opts.release_agent)
ed3d261b 1772 pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n",
a2a1f9ea 1773 task_tgid_nr(current), current->comm);
8b5a5a9d 1774
f392e51c
TH
1775 added_mask = opts.subsys_mask & ~root->subsys_mask;
1776 removed_mask = root->subsys_mask & ~opts.subsys_mask;
13af07df 1777
cf5d5941 1778 /* Don't allow flags or name to change at remount */
7450e90b 1779 if ((opts.flags ^ root->flags) ||
cf5d5941 1780 (opts.name && strcmp(opts.name, root->name))) {
69dfa00c 1781 pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n",
7450e90b 1782 opts.flags, opts.name ?: "", root->flags, root->name);
c6d57f33
PM
1783 ret = -EINVAL;
1784 goto out_unlock;
1785 }
1786
f172e67c 1787 /* remounting is not allowed for populated hierarchies */
d5c419b6 1788 if (!list_empty(&root->cgrp.self.children)) {
f172e67c 1789 ret = -EBUSY;
0670e08b 1790 goto out_unlock;
cf5d5941 1791 }
ddbcc7e8 1792
5df36032 1793 ret = rebind_subsystems(root, added_mask);
3126121f 1794 if (ret)
0670e08b 1795 goto out_unlock;
ddbcc7e8 1796
3dd06ffa 1797 rebind_subsystems(&cgrp_dfl_root, removed_mask);
5df36032 1798
69e943b7
TH
1799 if (opts.release_agent) {
1800 spin_lock(&release_agent_path_lock);
81a6a5cd 1801 strcpy(root->release_agent_path, opts.release_agent);
69e943b7
TH
1802 spin_unlock(&release_agent_path_lock);
1803 }
ddbcc7e8 1804 out_unlock:
66bdc9cf 1805 kfree(opts.release_agent);
c6d57f33 1806 kfree(opts.name);
ddbcc7e8 1807 mutex_unlock(&cgroup_mutex);
ddbcc7e8
PM
1808 return ret;
1809}
1810
afeb0f9f
TH
1811/*
1812 * To reduce the fork() overhead for systems that are not actually using
1813 * their cgroups capability, we don't maintain the lists running through
1814 * each css_set to its tasks until we see the list actually used - in other
1815 * words after the first mount.
1816 */
1817static bool use_task_css_set_links __read_mostly;
1818
1819static void cgroup_enable_task_cg_lists(void)
1820{
1821 struct task_struct *p, *g;
1822
f0d9a5f1 1823 spin_lock_bh(&css_set_lock);
afeb0f9f
TH
1824
1825 if (use_task_css_set_links)
1826 goto out_unlock;
1827
1828 use_task_css_set_links = true;
1829
1830 /*
1831 * We need tasklist_lock because RCU is not safe against
1832 * while_each_thread(). Besides, a forking task that has passed
1833 * cgroup_post_fork() without seeing use_task_css_set_links = 1
1834 * is not guaranteed to have its child immediately visible in the
1835 * tasklist if we walk through it with RCU.
1836 */
1837 read_lock(&tasklist_lock);
1838 do_each_thread(g, p) {
afeb0f9f
TH
1839 WARN_ON_ONCE(!list_empty(&p->cg_list) ||
1840 task_css_set(p) != &init_css_set);
1841
1842 /*
1843 * We should check if the process is exiting, otherwise
1844 * it will race with cgroup_exit() in that the list
1845 * entry won't be deleted though the process has exited.
f153ad11
TH
1846 * Do it while holding siglock so that we don't end up
1847 * racing against cgroup_exit().
afeb0f9f 1848 */
f153ad11 1849 spin_lock_irq(&p->sighand->siglock);
eaf797ab
TH
1850 if (!(p->flags & PF_EXITING)) {
1851 struct css_set *cset = task_css_set(p);
1852
0de0942d
TH
1853 if (!css_set_populated(cset))
1854 css_set_update_populated(cset, true);
389b9c1b 1855 list_add_tail(&p->cg_list, &cset->tasks);
eaf797ab
TH
1856 get_css_set(cset);
1857 }
f153ad11 1858 spin_unlock_irq(&p->sighand->siglock);
afeb0f9f
TH
1859 } while_each_thread(g, p);
1860 read_unlock(&tasklist_lock);
1861out_unlock:
f0d9a5f1 1862 spin_unlock_bh(&css_set_lock);
afeb0f9f 1863}
ddbcc7e8 1864
cc31edce
PM
1865static void init_cgroup_housekeeping(struct cgroup *cgrp)
1866{
2d8f243a
TH
1867 struct cgroup_subsys *ss;
1868 int ssid;
1869
d5c419b6
TH
1870 INIT_LIST_HEAD(&cgrp->self.sibling);
1871 INIT_LIST_HEAD(&cgrp->self.children);
69d0206c 1872 INIT_LIST_HEAD(&cgrp->cset_links);
72a8cb30
BB
1873 INIT_LIST_HEAD(&cgrp->pidlists);
1874 mutex_init(&cgrp->pidlist_mutex);
9d800df1 1875 cgrp->self.cgroup = cgrp;
184faf32 1876 cgrp->self.flags |= CSS_ONLINE;
2d8f243a
TH
1877
1878 for_each_subsys(ss, ssid)
1879 INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
f8f22e53
TH
1880
1881 init_waitqueue_head(&cgrp->offline_waitq);
971ff493 1882 INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent);
cc31edce 1883}
c6d57f33 1884
3dd06ffa 1885static void init_cgroup_root(struct cgroup_root *root,
172a2c06 1886 struct cgroup_sb_opts *opts)
ddbcc7e8 1887{
3dd06ffa 1888 struct cgroup *cgrp = &root->cgrp;
b0ca5a84 1889
ddbcc7e8 1890 INIT_LIST_HEAD(&root->root_list);
3c9c825b 1891 atomic_set(&root->nr_cgrps, 1);
bd89aabc 1892 cgrp->root = root;
cc31edce 1893 init_cgroup_housekeeping(cgrp);
4e96ee8e 1894 idr_init(&root->cgroup_idr);
c6d57f33 1895
c6d57f33
PM
1896 root->flags = opts->flags;
1897 if (opts->release_agent)
1898 strcpy(root->release_agent_path, opts->release_agent);
1899 if (opts->name)
1900 strcpy(root->name, opts->name);
2260e7fc 1901 if (opts->cpuset_clone_children)
3dd06ffa 1902 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags);
c6d57f33
PM
1903}
1904
6e5c8307 1905static int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask)
2c6ab6d2 1906{
d427dfeb 1907 LIST_HEAD(tmp_links);
3dd06ffa 1908 struct cgroup *root_cgrp = &root->cgrp;
d427dfeb 1909 struct css_set *cset;
d427dfeb 1910 int i, ret;
2c6ab6d2 1911
d427dfeb 1912 lockdep_assert_held(&cgroup_mutex);
c6d57f33 1913
cf780b7d 1914 ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL);
d427dfeb 1915 if (ret < 0)
2bd59d48 1916 goto out;
d427dfeb 1917 root_cgrp->id = ret;
b11cfb58 1918 root_cgrp->ancestor_ids[0] = ret;
c6d57f33 1919
2aad2a86
TH
1920 ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0,
1921 GFP_KERNEL);
9d755d33
TH
1922 if (ret)
1923 goto out;
1924
d427dfeb 1925 /*
f0d9a5f1 1926 * We're accessing css_set_count without locking css_set_lock here,
d427dfeb
TH
1927 * but that's OK - it can only be increased by someone holding
1928 * cgroup_lock, and that's us. The worst that can happen is that we
1929 * have some link structures left over
1930 */
1931 ret = allocate_cgrp_cset_links(css_set_count, &tmp_links);
1932 if (ret)
9d755d33 1933 goto cancel_ref;
ddbcc7e8 1934
985ed670 1935 ret = cgroup_init_root_id(root);
ddbcc7e8 1936 if (ret)
9d755d33 1937 goto cancel_ref;
ddbcc7e8 1938
2bd59d48
TH
1939 root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops,
1940 KERNFS_ROOT_CREATE_DEACTIVATED,
1941 root_cgrp);
1942 if (IS_ERR(root->kf_root)) {
1943 ret = PTR_ERR(root->kf_root);
1944 goto exit_root_id;
1945 }
1946 root_cgrp->kn = root->kf_root->kn;
ddbcc7e8 1947
4df8dc90 1948 ret = css_populate_dir(&root_cgrp->self, NULL);
d427dfeb 1949 if (ret)
2bd59d48 1950 goto destroy_root;
ddbcc7e8 1951
5df36032 1952 ret = rebind_subsystems(root, ss_mask);
d427dfeb 1953 if (ret)
2bd59d48 1954 goto destroy_root;
ddbcc7e8 1955
d427dfeb
TH
1956 /*
1957 * There must be no failure case after here, since rebinding takes
1958 * care of subsystems' refcounts, which are explicitly dropped in
1959 * the failure exit path.
1960 */
1961 list_add(&root->root_list, &cgroup_roots);
1962 cgroup_root_count++;
0df6a63f 1963
d427dfeb 1964 /*
3dd06ffa 1965 * Link the root cgroup in this hierarchy into all the css_set
d427dfeb
TH
1966 * objects.
1967 */
f0d9a5f1 1968 spin_lock_bh(&css_set_lock);
0de0942d 1969 hash_for_each(css_set_table, i, cset, hlist) {
d427dfeb 1970 link_css_set(&tmp_links, cset, root_cgrp);
0de0942d
TH
1971 if (css_set_populated(cset))
1972 cgroup_update_populated(root_cgrp, true);
1973 }
f0d9a5f1 1974 spin_unlock_bh(&css_set_lock);
ddbcc7e8 1975
d5c419b6 1976 BUG_ON(!list_empty(&root_cgrp->self.children));
3c9c825b 1977 BUG_ON(atomic_read(&root->nr_cgrps) != 1);
ddbcc7e8 1978
2bd59d48 1979 kernfs_activate(root_cgrp->kn);
d427dfeb 1980 ret = 0;
2bd59d48 1981 goto out;
d427dfeb 1982
2bd59d48
TH
1983destroy_root:
1984 kernfs_destroy_root(root->kf_root);
1985 root->kf_root = NULL;
1986exit_root_id:
d427dfeb 1987 cgroup_exit_root_id(root);
9d755d33 1988cancel_ref:
9a1049da 1989 percpu_ref_exit(&root_cgrp->self.refcnt);
2bd59d48 1990out:
d427dfeb
TH
1991 free_cgrp_cset_links(&tmp_links);
1992 return ret;
ddbcc7e8
PM
1993}
1994
f7e83571 1995static struct dentry *cgroup_mount(struct file_system_type *fs_type,
ddbcc7e8 1996 int flags, const char *unused_dev_name,
f7e83571 1997 void *data)
ddbcc7e8 1998{
67e9c74b 1999 bool is_v2 = fs_type == &cgroup2_fs_type;
3a32bd72 2000 struct super_block *pinned_sb = NULL;
970317aa 2001 struct cgroup_subsys *ss;
3dd06ffa 2002 struct cgroup_root *root;
ddbcc7e8 2003 struct cgroup_sb_opts opts;
2bd59d48 2004 struct dentry *dentry;
8e30e2b8 2005 int ret;
970317aa 2006 int i;
c6b3d5bc 2007 bool new_sb;
ddbcc7e8 2008
56fde9e0
TH
2009 /*
2010 * The first time anyone tries to mount a cgroup, enable the list
2011 * linking each css_set to its tasks and fix up all existing tasks.
2012 */
2013 if (!use_task_css_set_links)
2014 cgroup_enable_task_cg_lists();
e37a06f1 2015
67e9c74b
TH
2016 if (is_v2) {
2017 if (data) {
2018 pr_err("cgroup2: unknown option \"%s\"\n", (char *)data);
2019 return ERR_PTR(-EINVAL);
2020 }
a7165264 2021 cgrp_dfl_visible = true;
67e9c74b
TH
2022 root = &cgrp_dfl_root;
2023 cgroup_get(&root->cgrp);
2024 goto out_mount;
2025 }
2026
aae8aab4 2027 mutex_lock(&cgroup_mutex);
8e30e2b8
TH
2028
2029 /* First find the desired set of subsystems */
ddbcc7e8 2030 ret = parse_cgroupfs_options(data, &opts);
c6d57f33 2031 if (ret)
8e30e2b8 2032 goto out_unlock;
a015edd2 2033
970317aa
LZ
2034 /*
2035 * Destruction of cgroup root is asynchronous, so subsystems may
2036 * still be dying after the previous unmount. Let's drain the
2037 * dying subsystems. We just need to ensure that the ones
2038 * unmounted previously finish dying and don't care about new ones
2039 * starting. Testing ref liveliness is good enough.
2040 */
2041 for_each_subsys(ss, i) {
2042 if (!(opts.subsys_mask & (1 << i)) ||
2043 ss->root == &cgrp_dfl_root)
2044 continue;
2045
2046 if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) {
2047 mutex_unlock(&cgroup_mutex);
2048 msleep(10);
2049 ret = restart_syscall();
2050 goto out_free;
2051 }
2052 cgroup_put(&ss->root->cgrp);
2053 }
2054
985ed670 2055 for_each_root(root) {
2bd59d48 2056 bool name_match = false;
3126121f 2057
3dd06ffa 2058 if (root == &cgrp_dfl_root)
985ed670 2059 continue;
3126121f 2060
cf5d5941 2061 /*
2bd59d48
TH
2062 * If we asked for a name then it must match. Also, if
2063 * name matches but sybsys_mask doesn't, we should fail.
2064 * Remember whether name matched.
cf5d5941 2065 */
2bd59d48
TH
2066 if (opts.name) {
2067 if (strcmp(opts.name, root->name))
2068 continue;
2069 name_match = true;
2070 }
ddbcc7e8 2071
c6d57f33 2072 /*
2bd59d48
TH
2073 * If we asked for subsystems (or explicitly for no
2074 * subsystems) then they must match.
c6d57f33 2075 */
2bd59d48 2076 if ((opts.subsys_mask || opts.none) &&
f392e51c 2077 (opts.subsys_mask != root->subsys_mask)) {
2bd59d48
TH
2078 if (!name_match)
2079 continue;
2080 ret = -EBUSY;
2081 goto out_unlock;
2082 }
873fe09e 2083
7b9a6ba5
TH
2084 if (root->flags ^ opts.flags)
2085 pr_warn("new mount options do not match the existing superblock, will be ignored\n");
ddbcc7e8 2086
776f02fa 2087 /*
3a32bd72
LZ
2088 * We want to reuse @root whose lifetime is governed by its
2089 * ->cgrp. Let's check whether @root is alive and keep it
2090 * that way. As cgroup_kill_sb() can happen anytime, we
2091 * want to block it by pinning the sb so that @root doesn't
2092 * get killed before mount is complete.
2093 *
2094 * With the sb pinned, tryget_live can reliably indicate
2095 * whether @root can be reused. If it's being killed,
2096 * drain it. We can use wait_queue for the wait but this
2097 * path is super cold. Let's just sleep a bit and retry.
776f02fa 2098 */
3a32bd72
LZ
2099 pinned_sb = kernfs_pin_sb(root->kf_root, NULL);
2100 if (IS_ERR(pinned_sb) ||
2101 !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) {
776f02fa 2102 mutex_unlock(&cgroup_mutex);
3a32bd72
LZ
2103 if (!IS_ERR_OR_NULL(pinned_sb))
2104 deactivate_super(pinned_sb);
776f02fa 2105 msleep(10);
a015edd2
TH
2106 ret = restart_syscall();
2107 goto out_free;
776f02fa 2108 }
ddbcc7e8 2109
776f02fa 2110 ret = 0;
2bd59d48 2111 goto out_unlock;
ddbcc7e8 2112 }
ddbcc7e8 2113
817929ec 2114 /*
172a2c06
TH
2115 * No such thing, create a new one. name= matching without subsys
2116 * specification is allowed for already existing hierarchies but we
2117 * can't create new one without subsys specification.
817929ec 2118 */
172a2c06
TH
2119 if (!opts.subsys_mask && !opts.none) {
2120 ret = -EINVAL;
2121 goto out_unlock;
817929ec 2122 }
817929ec 2123
172a2c06
TH
2124 root = kzalloc(sizeof(*root), GFP_KERNEL);
2125 if (!root) {
2126 ret = -ENOMEM;
2bd59d48 2127 goto out_unlock;
839ec545 2128 }
e5f6a860 2129
172a2c06
TH
2130 init_cgroup_root(root, &opts);
2131
35585573 2132 ret = cgroup_setup_root(root, opts.subsys_mask);
2bd59d48
TH
2133 if (ret)
2134 cgroup_free_root(root);
fa3ca07e 2135
8e30e2b8 2136out_unlock:
ddbcc7e8 2137 mutex_unlock(&cgroup_mutex);
a015edd2 2138out_free:
c6d57f33
PM
2139 kfree(opts.release_agent);
2140 kfree(opts.name);
03b1cde6 2141
2bd59d48 2142 if (ret)
8e30e2b8 2143 return ERR_PTR(ret);
67e9c74b 2144out_mount:
c9482a5b 2145 dentry = kernfs_mount(fs_type, flags, root->kf_root,
67e9c74b
TH
2146 is_v2 ? CGROUP2_SUPER_MAGIC : CGROUP_SUPER_MAGIC,
2147 &new_sb);
c6b3d5bc 2148 if (IS_ERR(dentry) || !new_sb)
3dd06ffa 2149 cgroup_put(&root->cgrp);
3a32bd72
LZ
2150
2151 /*
2152 * If @pinned_sb, we're reusing an existing root and holding an
2153 * extra ref on its sb. Mount is complete. Put the extra ref.
2154 */
2155 if (pinned_sb) {
2156 WARN_ON(new_sb);
2157 deactivate_super(pinned_sb);
2158 }
2159
2bd59d48
TH
2160 return dentry;
2161}
2162
2163static void cgroup_kill_sb(struct super_block *sb)
2164{
2165 struct kernfs_root *kf_root = kernfs_root_from_sb(sb);
3dd06ffa 2166 struct cgroup_root *root = cgroup_root_from_kf(kf_root);
2bd59d48 2167
9d755d33
TH
2168 /*
2169 * If @root doesn't have any mounts or children, start killing it.
2170 * This prevents new mounts by disabling percpu_ref_tryget_live().
2171 * cgroup_mount() may wait for @root's release.
1f779fb2
LZ
2172 *
2173 * And don't kill the default root.
9d755d33 2174 */
3c606d35 2175 if (!list_empty(&root->cgrp.self.children) ||
1f779fb2 2176 root == &cgrp_dfl_root)
9d755d33
TH
2177 cgroup_put(&root->cgrp);
2178 else
2179 percpu_ref_kill(&root->cgrp.self.refcnt);
2180
2bd59d48 2181 kernfs_kill_sb(sb);
ddbcc7e8
PM
2182}
2183
2184static struct file_system_type cgroup_fs_type = {
2185 .name = "cgroup",
f7e83571 2186 .mount = cgroup_mount,
ddbcc7e8
PM
2187 .kill_sb = cgroup_kill_sb,
2188};
2189
67e9c74b
TH
2190static struct file_system_type cgroup2_fs_type = {
2191 .name = "cgroup2",
2192 .mount = cgroup_mount,
2193 .kill_sb = cgroup_kill_sb,
2194};
2195
857a2beb 2196/**
913ffdb5 2197 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
857a2beb 2198 * @task: target task
857a2beb
TH
2199 * @buf: the buffer to write the path into
2200 * @buflen: the length of the buffer
2201 *
913ffdb5
TH
2202 * Determine @task's cgroup on the first (the one with the lowest non-zero
2203 * hierarchy_id) cgroup hierarchy and copy its path into @buf. This
2204 * function grabs cgroup_mutex and shouldn't be used inside locks used by
2205 * cgroup controller callbacks.
2206 *
e61734c5 2207 * Return value is the same as kernfs_path().
857a2beb 2208 */
e61734c5 2209char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen)
857a2beb 2210{
3dd06ffa 2211 struct cgroup_root *root;
913ffdb5 2212 struct cgroup *cgrp;
e61734c5
TH
2213 int hierarchy_id = 1;
2214 char *path = NULL;
857a2beb
TH
2215
2216 mutex_lock(&cgroup_mutex);
f0d9a5f1 2217 spin_lock_bh(&css_set_lock);
857a2beb 2218
913ffdb5
TH
2219 root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id);
2220
857a2beb
TH
2221 if (root) {
2222 cgrp = task_cgroup_from_root(task, root);
e61734c5 2223 path = cgroup_path(cgrp, buf, buflen);
913ffdb5
TH
2224 } else {
2225 /* if no hierarchy exists, everyone is in "/" */
e61734c5
TH
2226 if (strlcpy(buf, "/", buflen) < buflen)
2227 path = buf;
857a2beb
TH
2228 }
2229
f0d9a5f1 2230 spin_unlock_bh(&css_set_lock);
857a2beb 2231 mutex_unlock(&cgroup_mutex);
e61734c5 2232 return path;
857a2beb 2233}
913ffdb5 2234EXPORT_SYMBOL_GPL(task_cgroup_path);
857a2beb 2235
b3dc094e 2236/* used to track tasks and other necessary states during migration */
2f7ee569 2237struct cgroup_taskset {
b3dc094e
TH
2238 /* the src and dst cset list running through cset->mg_node */
2239 struct list_head src_csets;
2240 struct list_head dst_csets;
2241
1f7dd3e5
TH
2242 /* the subsys currently being processed */
2243 int ssid;
2244
b3dc094e
TH
2245 /*
2246 * Fields for cgroup_taskset_*() iteration.
2247 *
2248 * Before migration is committed, the target migration tasks are on
2249 * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of
2250 * the csets on ->dst_csets. ->csets point to either ->src_csets
2251 * or ->dst_csets depending on whether migration is committed.
2252 *
2253 * ->cur_csets and ->cur_task point to the current task position
2254 * during iteration.
2255 */
2256 struct list_head *csets;
2257 struct css_set *cur_cset;
2258 struct task_struct *cur_task;
2f7ee569
TH
2259};
2260
adaae5dc
TH
2261#define CGROUP_TASKSET_INIT(tset) (struct cgroup_taskset){ \
2262 .src_csets = LIST_HEAD_INIT(tset.src_csets), \
2263 .dst_csets = LIST_HEAD_INIT(tset.dst_csets), \
2264 .csets = &tset.src_csets, \
2265}
2266
2267/**
2268 * cgroup_taskset_add - try to add a migration target task to a taskset
2269 * @task: target task
2270 * @tset: target taskset
2271 *
2272 * Add @task, which is a migration target, to @tset. This function becomes
2273 * noop if @task doesn't need to be migrated. @task's css_set should have
2274 * been added as a migration source and @task->cg_list will be moved from
2275 * the css_set's tasks list to mg_tasks one.
2276 */
2277static void cgroup_taskset_add(struct task_struct *task,
2278 struct cgroup_taskset *tset)
2279{
2280 struct css_set *cset;
2281
f0d9a5f1 2282 lockdep_assert_held(&css_set_lock);
adaae5dc
TH
2283
2284 /* @task either already exited or can't exit until the end */
2285 if (task->flags & PF_EXITING)
2286 return;
2287
2288 /* leave @task alone if post_fork() hasn't linked it yet */
2289 if (list_empty(&task->cg_list))
2290 return;
2291
2292 cset = task_css_set(task);
2293 if (!cset->mg_src_cgrp)
2294 return;
2295
2296 list_move_tail(&task->cg_list, &cset->mg_tasks);
2297 if (list_empty(&cset->mg_node))
2298 list_add_tail(&cset->mg_node, &tset->src_csets);
2299 if (list_empty(&cset->mg_dst_cset->mg_node))
2300 list_move_tail(&cset->mg_dst_cset->mg_node,
2301 &tset->dst_csets);
2302}
2303
2f7ee569
TH
2304/**
2305 * cgroup_taskset_first - reset taskset and return the first task
2306 * @tset: taskset of interest
1f7dd3e5 2307 * @dst_cssp: output variable for the destination css
2f7ee569
TH
2308 *
2309 * @tset iteration is initialized and the first task is returned.
2310 */
1f7dd3e5
TH
2311struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset,
2312 struct cgroup_subsys_state **dst_cssp)
2f7ee569 2313{
b3dc094e
TH
2314 tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node);
2315 tset->cur_task = NULL;
2316
1f7dd3e5 2317 return cgroup_taskset_next(tset, dst_cssp);
2f7ee569 2318}
2f7ee569
TH
2319
2320/**
2321 * cgroup_taskset_next - iterate to the next task in taskset
2322 * @tset: taskset of interest
1f7dd3e5 2323 * @dst_cssp: output variable for the destination css
2f7ee569
TH
2324 *
2325 * Return the next task in @tset. Iteration must have been initialized
2326 * with cgroup_taskset_first().
2327 */
1f7dd3e5
TH
2328struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset,
2329 struct cgroup_subsys_state **dst_cssp)
2f7ee569 2330{
b3dc094e
TH
2331 struct css_set *cset = tset->cur_cset;
2332 struct task_struct *task = tset->cur_task;
2f7ee569 2333
b3dc094e
TH
2334 while (&cset->mg_node != tset->csets) {
2335 if (!task)
2336 task = list_first_entry(&cset->mg_tasks,
2337 struct task_struct, cg_list);
2338 else
2339 task = list_next_entry(task, cg_list);
2f7ee569 2340
b3dc094e
TH
2341 if (&task->cg_list != &cset->mg_tasks) {
2342 tset->cur_cset = cset;
2343 tset->cur_task = task;
1f7dd3e5
TH
2344
2345 /*
2346 * This function may be called both before and
2347 * after cgroup_taskset_migrate(). The two cases
2348 * can be distinguished by looking at whether @cset
2349 * has its ->mg_dst_cset set.
2350 */
2351 if (cset->mg_dst_cset)
2352 *dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];
2353 else
2354 *dst_cssp = cset->subsys[tset->ssid];
2355
b3dc094e
TH
2356 return task;
2357 }
2f7ee569 2358
b3dc094e
TH
2359 cset = list_next_entry(cset, mg_node);
2360 task = NULL;
2361 }
2f7ee569 2362
b3dc094e 2363 return NULL;
2f7ee569 2364}
2f7ee569 2365
adaae5dc
TH
2366/**
2367 * cgroup_taskset_migrate - migrate a taskset to a cgroup
2368 * @tset: taget taskset
2369 * @dst_cgrp: destination cgroup
2370 *
2371 * Migrate tasks in @tset to @dst_cgrp. This function fails iff one of the
2372 * ->can_attach callbacks fails and guarantees that either all or none of
2373 * the tasks in @tset are migrated. @tset is consumed regardless of
2374 * success.
2375 */
2376static int cgroup_taskset_migrate(struct cgroup_taskset *tset,
2377 struct cgroup *dst_cgrp)
2378{
2379 struct cgroup_subsys_state *css, *failed_css = NULL;
2380 struct task_struct *task, *tmp_task;
2381 struct css_set *cset, *tmp_cset;
2382 int i, ret;
2383
2384 /* methods shouldn't be called if no task is actually migrating */
2385 if (list_empty(&tset->src_csets))
2386 return 0;
2387
2388 /* check that we can legitimately attach to the cgroup */
2389 for_each_e_css(css, i, dst_cgrp) {
2390 if (css->ss->can_attach) {
1f7dd3e5
TH
2391 tset->ssid = i;
2392 ret = css->ss->can_attach(tset);
adaae5dc
TH
2393 if (ret) {
2394 failed_css = css;
2395 goto out_cancel_attach;
2396 }
2397 }
2398 }
2399
2400 /*
2401 * Now that we're guaranteed success, proceed to move all tasks to
2402 * the new cgroup. There are no failure cases after here, so this
2403 * is the commit point.
2404 */
f0d9a5f1 2405 spin_lock_bh(&css_set_lock);
adaae5dc 2406 list_for_each_entry(cset, &tset->src_csets, mg_node) {
f6d7d049
TH
2407 list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {
2408 struct css_set *from_cset = task_css_set(task);
2409 struct css_set *to_cset = cset->mg_dst_cset;
2410
2411 get_css_set(to_cset);
2412 css_set_move_task(task, from_cset, to_cset, true);
2413 put_css_set_locked(from_cset);
2414 }
adaae5dc 2415 }
f0d9a5f1 2416 spin_unlock_bh(&css_set_lock);
adaae5dc
TH
2417
2418 /*
2419 * Migration is committed, all target tasks are now on dst_csets.
2420 * Nothing is sensitive to fork() after this point. Notify
2421 * controllers that migration is complete.
2422 */
2423 tset->csets = &tset->dst_csets;
2424
1f7dd3e5
TH
2425 for_each_e_css(css, i, dst_cgrp) {
2426 if (css->ss->attach) {
2427 tset->ssid = i;
2428 css->ss->attach(tset);
2429 }
2430 }
adaae5dc
TH
2431
2432 ret = 0;
2433 goto out_release_tset;
2434
2435out_cancel_attach:
2436 for_each_e_css(css, i, dst_cgrp) {
2437 if (css == failed_css)
2438 break;
1f7dd3e5
TH
2439 if (css->ss->cancel_attach) {
2440 tset->ssid = i;
2441 css->ss->cancel_attach(tset);
2442 }
adaae5dc
TH
2443 }
2444out_release_tset:
f0d9a5f1 2445 spin_lock_bh(&css_set_lock);
adaae5dc
TH
2446 list_splice_init(&tset->dst_csets, &tset->src_csets);
2447 list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {
2448 list_splice_tail_init(&cset->mg_tasks, &cset->tasks);
2449 list_del_init(&cset->mg_node);
2450 }
f0d9a5f1 2451 spin_unlock_bh(&css_set_lock);
adaae5dc
TH
2452 return ret;
2453}
2454
a043e3b2 2455/**
1958d2d5
TH
2456 * cgroup_migrate_finish - cleanup after attach
2457 * @preloaded_csets: list of preloaded css_sets
74a1166d 2458 *
1958d2d5
TH
2459 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See
2460 * those functions for details.
74a1166d 2461 */
1958d2d5 2462static void cgroup_migrate_finish(struct list_head *preloaded_csets)
74a1166d 2463{
1958d2d5 2464 struct css_set *cset, *tmp_cset;
74a1166d 2465
1958d2d5
TH
2466 lockdep_assert_held(&cgroup_mutex);
2467
f0d9a5f1 2468 spin_lock_bh(&css_set_lock);
1958d2d5
TH
2469 list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) {
2470 cset->mg_src_cgrp = NULL;
2471 cset->mg_dst_cset = NULL;
2472 list_del_init(&cset->mg_preload_node);
a25eb52e 2473 put_css_set_locked(cset);
1958d2d5 2474 }
f0d9a5f1 2475 spin_unlock_bh(&css_set_lock);
1958d2d5
TH
2476}
2477
2478/**
2479 * cgroup_migrate_add_src - add a migration source css_set
2480 * @src_cset: the source css_set to add
2481 * @dst_cgrp: the destination cgroup
2482 * @preloaded_csets: list of preloaded css_sets
2483 *
2484 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin
2485 * @src_cset and add it to @preloaded_csets, which should later be cleaned
2486 * up by cgroup_migrate_finish().
2487 *
1ed13287
TH
2488 * This function may be called without holding cgroup_threadgroup_rwsem
2489 * even if the target is a process. Threads may be created and destroyed
2490 * but as long as cgroup_mutex is not dropped, no new css_set can be put
2491 * into play and the preloaded css_sets are guaranteed to cover all
2492 * migrations.
1958d2d5
TH
2493 */
2494static void cgroup_migrate_add_src(struct css_set *src_cset,
2495 struct cgroup *dst_cgrp,
2496 struct list_head *preloaded_csets)
2497{
2498 struct cgroup *src_cgrp;
2499
2500 lockdep_assert_held(&cgroup_mutex);
f0d9a5f1 2501 lockdep_assert_held(&css_set_lock);
1958d2d5
TH
2502
2503 src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root);
2504
1958d2d5
TH
2505 if (!list_empty(&src_cset->mg_preload_node))
2506 return;
2507
2508 WARN_ON(src_cset->mg_src_cgrp);
2509 WARN_ON(!list_empty(&src_cset->mg_tasks));
2510 WARN_ON(!list_empty(&src_cset->mg_node));
2511
2512 src_cset->mg_src_cgrp = src_cgrp;
2513 get_css_set(src_cset);
2514 list_add(&src_cset->mg_preload_node, preloaded_csets);
2515}
2516
2517/**
2518 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
f817de98 2519 * @dst_cgrp: the destination cgroup (may be %NULL)
1958d2d5
TH
2520 * @preloaded_csets: list of preloaded source css_sets
2521 *
2522 * Tasks are about to be moved to @dst_cgrp and all the source css_sets
2523 * have been preloaded to @preloaded_csets. This function looks up and
f817de98
TH
2524 * pins all destination css_sets, links each to its source, and append them
2525 * to @preloaded_csets. If @dst_cgrp is %NULL, the destination of each
2526 * source css_set is assumed to be its cgroup on the default hierarchy.
1958d2d5
TH
2527 *
2528 * This function must be called after cgroup_migrate_add_src() has been
2529 * called on each migration source css_set. After migration is performed
2530 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2531 * @preloaded_csets.
2532 */
2533static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp,
2534 struct list_head *preloaded_csets)
2535{
2536 LIST_HEAD(csets);
f817de98 2537 struct css_set *src_cset, *tmp_cset;
1958d2d5
TH
2538
2539 lockdep_assert_held(&cgroup_mutex);
2540
f8f22e53 2541 /*
62716ea0 2542 * Except for the root, subtree_control must be zero for a cgroup
f8f22e53
TH
2543 * with tasks so that child cgroups don't compete against tasks.
2544 */
d51f39b0 2545 if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) &&
62716ea0 2546 dst_cgrp->subtree_control)
f8f22e53
TH
2547 return -EBUSY;
2548
1958d2d5 2549 /* look up the dst cset for each src cset and link it to src */
f817de98 2550 list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) {
1958d2d5
TH
2551 struct css_set *dst_cset;
2552
f817de98
TH
2553 dst_cset = find_css_set(src_cset,
2554 dst_cgrp ?: src_cset->dfl_cgrp);
1958d2d5
TH
2555 if (!dst_cset)
2556 goto err;
2557
2558 WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
f817de98
TH
2559
2560 /*
2561 * If src cset equals dst, it's noop. Drop the src.
2562 * cgroup_migrate() will skip the cset too. Note that we
2563 * can't handle src == dst as some nodes are used by both.
2564 */
2565 if (src_cset == dst_cset) {
2566 src_cset->mg_src_cgrp = NULL;
2567 list_del_init(&src_cset->mg_preload_node);
a25eb52e
ZL
2568 put_css_set(src_cset);
2569 put_css_set(dst_cset);
f817de98
TH
2570 continue;
2571 }
2572
1958d2d5
TH
2573 src_cset->mg_dst_cset = dst_cset;
2574
2575 if (list_empty(&dst_cset->mg_preload_node))
2576 list_add(&dst_cset->mg_preload_node, &csets);
2577 else
a25eb52e 2578 put_css_set(dst_cset);
1958d2d5
TH
2579 }
2580
f817de98 2581 list_splice_tail(&csets, preloaded_csets);
1958d2d5
TH
2582 return 0;
2583err:
2584 cgroup_migrate_finish(&csets);
2585 return -ENOMEM;
2586}
2587
2588/**
2589 * cgroup_migrate - migrate a process or task to a cgroup
1958d2d5
TH
2590 * @leader: the leader of the process or the task to migrate
2591 * @threadgroup: whether @leader points to the whole process or a single task
9af2ec45 2592 * @cgrp: the destination cgroup
1958d2d5
TH
2593 *
2594 * Migrate a process or task denoted by @leader to @cgrp. If migrating a
1ed13287 2595 * process, the caller must be holding cgroup_threadgroup_rwsem. The
1958d2d5
TH
2596 * caller is also responsible for invoking cgroup_migrate_add_src() and
2597 * cgroup_migrate_prepare_dst() on the targets before invoking this
2598 * function and following up with cgroup_migrate_finish().
2599 *
2600 * As long as a controller's ->can_attach() doesn't fail, this function is
2601 * guaranteed to succeed. This means that, excluding ->can_attach()
2602 * failure, when migrating multiple targets, the success or failure can be
2603 * decided for all targets by invoking group_migrate_prepare_dst() before
2604 * actually starting migrating.
2605 */
9af2ec45
TH
2606static int cgroup_migrate(struct task_struct *leader, bool threadgroup,
2607 struct cgroup *cgrp)
74a1166d 2608{
adaae5dc
TH
2609 struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset);
2610 struct task_struct *task;
74a1166d 2611
fb5d2b4c
MSB
2612 /*
2613 * Prevent freeing of tasks while we take a snapshot. Tasks that are
2614 * already PF_EXITING could be freed from underneath us unless we
2615 * take an rcu_read_lock.
2616 */
f0d9a5f1 2617 spin_lock_bh(&css_set_lock);
fb5d2b4c 2618 rcu_read_lock();
9db8de37 2619 task = leader;
74a1166d 2620 do {
adaae5dc 2621 cgroup_taskset_add(task, &tset);
081aa458
LZ
2622 if (!threadgroup)
2623 break;
9db8de37 2624 } while_each_thread(leader, task);
fb5d2b4c 2625 rcu_read_unlock();
f0d9a5f1 2626 spin_unlock_bh(&css_set_lock);
74a1166d 2627
adaae5dc 2628 return cgroup_taskset_migrate(&tset, cgrp);
74a1166d
BB
2629}
2630
1958d2d5
TH
2631/**
2632 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2633 * @dst_cgrp: the cgroup to attach to
2634 * @leader: the task or the leader of the threadgroup to be attached
2635 * @threadgroup: attach the whole threadgroup?
2636 *
1ed13287 2637 * Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
1958d2d5
TH
2638 */
2639static int cgroup_attach_task(struct cgroup *dst_cgrp,
2640 struct task_struct *leader, bool threadgroup)
2641{
2642 LIST_HEAD(preloaded_csets);
2643 struct task_struct *task;
2644 int ret;
2645
2646 /* look up all src csets */
f0d9a5f1 2647 spin_lock_bh(&css_set_lock);
1958d2d5
TH
2648 rcu_read_lock();
2649 task = leader;
2650 do {
2651 cgroup_migrate_add_src(task_css_set(task), dst_cgrp,
2652 &preloaded_csets);
2653 if (!threadgroup)
2654 break;
2655 } while_each_thread(leader, task);
2656 rcu_read_unlock();
f0d9a5f1 2657 spin_unlock_bh(&css_set_lock);
1958d2d5
TH
2658
2659 /* prepare dst csets and commit */
2660 ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets);
2661 if (!ret)
9af2ec45 2662 ret = cgroup_migrate(leader, threadgroup, dst_cgrp);
1958d2d5
TH
2663
2664 cgroup_migrate_finish(&preloaded_csets);
2665 return ret;
74a1166d
BB
2666}
2667
187fe840
TH
2668static int cgroup_procs_write_permission(struct task_struct *task,
2669 struct cgroup *dst_cgrp,
2670 struct kernfs_open_file *of)
dedf22e9
TH
2671{
2672 const struct cred *cred = current_cred();
2673 const struct cred *tcred = get_task_cred(task);
2674 int ret = 0;
2675
2676 /*
2677 * even if we're attaching all tasks in the thread group, we only
2678 * need to check permissions on one of them.
2679 */
2680 if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) &&
2681 !uid_eq(cred->euid, tcred->uid) &&
2682 !uid_eq(cred->euid, tcred->suid))
2683 ret = -EACCES;
2684
187fe840
TH
2685 if (!ret && cgroup_on_dfl(dst_cgrp)) {
2686 struct super_block *sb = of->file->f_path.dentry->d_sb;
2687 struct cgroup *cgrp;
2688 struct inode *inode;
2689
f0d9a5f1 2690 spin_lock_bh(&css_set_lock);
187fe840 2691 cgrp = task_cgroup_from_root(task, &cgrp_dfl_root);
f0d9a5f1 2692 spin_unlock_bh(&css_set_lock);
187fe840
TH
2693
2694 while (!cgroup_is_descendant(dst_cgrp, cgrp))
2695 cgrp = cgroup_parent(cgrp);
2696
2697 ret = -ENOMEM;
6f60eade 2698 inode = kernfs_get_inode(sb, cgrp->procs_file.kn);
187fe840
TH
2699 if (inode) {
2700 ret = inode_permission(inode, MAY_WRITE);
2701 iput(inode);
2702 }
2703 }
2704
dedf22e9
TH
2705 put_cred(tcred);
2706 return ret;
2707}
2708
74a1166d
BB
2709/*
2710 * Find the task_struct of the task to attach by vpid and pass it along to the
cd3d0952 2711 * function to attach either it or all tasks in its threadgroup. Will lock
0e1d768f 2712 * cgroup_mutex and threadgroup.
bbcb81d0 2713 */
acbef755
TH
2714static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
2715 size_t nbytes, loff_t off, bool threadgroup)
bbcb81d0 2716{
bbcb81d0 2717 struct task_struct *tsk;
e76ecaee 2718 struct cgroup *cgrp;
acbef755 2719 pid_t pid;
bbcb81d0
PM
2720 int ret;
2721
acbef755
TH
2722 if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
2723 return -EINVAL;
2724
e76ecaee
TH
2725 cgrp = cgroup_kn_lock_live(of->kn);
2726 if (!cgrp)
74a1166d
BB
2727 return -ENODEV;
2728
3014dde7 2729 percpu_down_write(&cgroup_threadgroup_rwsem);
b78949eb 2730 rcu_read_lock();
bbcb81d0 2731 if (pid) {
73507f33 2732 tsk = find_task_by_vpid(pid);
74a1166d 2733 if (!tsk) {
dd4b0a46 2734 ret = -ESRCH;
3014dde7 2735 goto out_unlock_rcu;
bbcb81d0 2736 }
dedf22e9 2737 } else {
b78949eb 2738 tsk = current;
dedf22e9 2739 }
cd3d0952
TH
2740
2741 if (threadgroup)
b78949eb 2742 tsk = tsk->group_leader;
c4c27fbd
MG
2743
2744 /*
14a40ffc 2745 * Workqueue threads may acquire PF_NO_SETAFFINITY and become
c4c27fbd
MG
2746 * trapped in a cpuset, or RT worker may be born in a cgroup
2747 * with no rt_runtime allocated. Just say no.
2748 */
14a40ffc 2749 if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) {
c4c27fbd 2750 ret = -EINVAL;
3014dde7 2751 goto out_unlock_rcu;
c4c27fbd
MG
2752 }
2753
b78949eb
MSB
2754 get_task_struct(tsk);
2755 rcu_read_unlock();
2756
187fe840 2757 ret = cgroup_procs_write_permission(tsk, cgrp, of);
dedf22e9
TH
2758 if (!ret)
2759 ret = cgroup_attach_task(cgrp, tsk, threadgroup);
081aa458 2760
f9f9e7b7 2761 put_task_struct(tsk);
3014dde7
TH
2762 goto out_unlock_threadgroup;
2763
2764out_unlock_rcu:
2765 rcu_read_unlock();
2766out_unlock_threadgroup:
2767 percpu_up_write(&cgroup_threadgroup_rwsem);
e76ecaee 2768 cgroup_kn_unlock(of->kn);
e93ad19d 2769 cpuset_post_attach_flush();
acbef755 2770 return ret ?: nbytes;
bbcb81d0
PM
2771}
2772
7ae1bad9
TH
2773/**
2774 * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from'
2775 * @from: attach to all cgroups of a given task
2776 * @tsk: the task to be attached
2777 */
2778int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
2779{
3dd06ffa 2780 struct cgroup_root *root;
7ae1bad9
TH
2781 int retval = 0;
2782
47cfcd09 2783 mutex_lock(&cgroup_mutex);
985ed670 2784 for_each_root(root) {
96d365e0
TH
2785 struct cgroup *from_cgrp;
2786
3dd06ffa 2787 if (root == &cgrp_dfl_root)
985ed670
TH
2788 continue;
2789
f0d9a5f1 2790 spin_lock_bh(&css_set_lock);
96d365e0 2791 from_cgrp = task_cgroup_from_root(from, root);
f0d9a5f1 2792 spin_unlock_bh(&css_set_lock);
7ae1bad9 2793
6f4b7e63 2794 retval = cgroup_attach_task(from_cgrp, tsk, false);
7ae1bad9
TH
2795 if (retval)
2796 break;
2797 }
47cfcd09 2798 mutex_unlock(&cgroup_mutex);
7ae1bad9
TH
2799
2800 return retval;
2801}
2802EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
2803
acbef755
TH
2804static ssize_t cgroup_tasks_write(struct kernfs_open_file *of,
2805 char *buf, size_t nbytes, loff_t off)
74a1166d 2806{
acbef755 2807 return __cgroup_procs_write(of, buf, nbytes, off, false);
74a1166d
BB
2808}
2809
acbef755
TH
2810static ssize_t cgroup_procs_write(struct kernfs_open_file *of,
2811 char *buf, size_t nbytes, loff_t off)
af351026 2812{
acbef755 2813 return __cgroup_procs_write(of, buf, nbytes, off, true);
af351026
PM
2814}
2815
451af504
TH
2816static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
2817 char *buf, size_t nbytes, loff_t off)
e788e066 2818{
e76ecaee 2819 struct cgroup *cgrp;
5f469907 2820
e76ecaee 2821 BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
5f469907 2822
e76ecaee
TH
2823 cgrp = cgroup_kn_lock_live(of->kn);
2824 if (!cgrp)
e788e066 2825 return -ENODEV;
69e943b7 2826 spin_lock(&release_agent_path_lock);
e76ecaee
TH
2827 strlcpy(cgrp->root->release_agent_path, strstrip(buf),
2828 sizeof(cgrp->root->release_agent_path));
69e943b7 2829 spin_unlock(&release_agent_path_lock);
e76ecaee 2830 cgroup_kn_unlock(of->kn);
451af504 2831 return nbytes;
e788e066
PM
2832}
2833
2da8ca82 2834static int cgroup_release_agent_show(struct seq_file *seq, void *v)
e788e066 2835{
2da8ca82 2836 struct cgroup *cgrp = seq_css(seq)->cgroup;
182446d0 2837
46cfeb04 2838 spin_lock(&release_agent_path_lock);
e788e066 2839 seq_puts(seq, cgrp->root->release_agent_path);
46cfeb04 2840 spin_unlock(&release_agent_path_lock);
e788e066 2841 seq_putc(seq, '\n');
e788e066
PM
2842 return 0;
2843}
2844
2da8ca82 2845static int cgroup_sane_behavior_show(struct seq_file *seq, void *v)
873fe09e 2846{
c1d5d42e 2847 seq_puts(seq, "0\n");
e788e066
PM
2848 return 0;
2849}
2850
6e5c8307 2851static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask)
355e0c48 2852{
f8f22e53
TH
2853 struct cgroup_subsys *ss;
2854 bool printed = false;
2855 int ssid;
a742c59d 2856
b4e0eeaf 2857 do_each_subsys_mask(ss, ssid, ss_mask) {
a966a4ed
AS
2858 if (printed)
2859 seq_putc(seq, ' ');
2860 seq_printf(seq, "%s", ss->name);
2861 printed = true;
b4e0eeaf 2862 } while_each_subsys_mask();
f8f22e53
TH
2863 if (printed)
2864 seq_putc(seq, '\n');
355e0c48
PM
2865}
2866
f8f22e53
TH
2867/* show controllers which are currently attached to the default hierarchy */
2868static int cgroup_root_controllers_show(struct seq_file *seq, void *v)
db3b1497 2869{
f8f22e53
TH
2870 struct cgroup *cgrp = seq_css(seq)->cgroup;
2871
5533e011 2872 cgroup_print_ss_mask(seq, cgrp->root->subsys_mask &
a7165264 2873 ~cgrp_dfl_inhibit_ss_mask);
f8f22e53 2874 return 0;
db3b1497
PM
2875}
2876
f8f22e53
TH
2877/* show controllers which are enabled from the parent */
2878static int cgroup_controllers_show(struct seq_file *seq, void *v)
ddbcc7e8 2879{
f8f22e53
TH
2880 struct cgroup *cgrp = seq_css(seq)->cgroup;
2881
667c2491 2882 cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control);
f8f22e53 2883 return 0;
ddbcc7e8
PM
2884}
2885
f8f22e53
TH
2886/* show controllers which are enabled for a given cgroup's children */
2887static int cgroup_subtree_control_show(struct seq_file *seq, void *v)
ddbcc7e8 2888{
f8f22e53
TH
2889 struct cgroup *cgrp = seq_css(seq)->cgroup;
2890
667c2491 2891 cgroup_print_ss_mask(seq, cgrp->subtree_control);
f8f22e53
TH
2892 return 0;
2893}
2894
2895/**
2896 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
2897 * @cgrp: root of the subtree to update csses for
2898 *
8699b776 2899 * @cgrp's subtree_ss_mask has changed and its subtree's (self excluded)
f8f22e53
TH
2900 * css associations need to be updated accordingly. This function looks up
2901 * all css_sets which are attached to the subtree, creates the matching
2902 * updated css_sets and migrates the tasks to the new ones.
2903 */
2904static int cgroup_update_dfl_csses(struct cgroup *cgrp)
2905{
2906 LIST_HEAD(preloaded_csets);
10265075 2907 struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset);
f8f22e53
TH
2908 struct cgroup_subsys_state *css;
2909 struct css_set *src_cset;
2910 int ret;
2911
f8f22e53
TH
2912 lockdep_assert_held(&cgroup_mutex);
2913
3014dde7
TH
2914 percpu_down_write(&cgroup_threadgroup_rwsem);
2915
f8f22e53 2916 /* look up all csses currently attached to @cgrp's subtree */
f0d9a5f1 2917 spin_lock_bh(&css_set_lock);
f8f22e53
TH
2918 css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) {
2919 struct cgrp_cset_link *link;
2920
8699b776 2921 /* self is not affected by subtree_ss_mask change */
f8f22e53
TH
2922 if (css->cgroup == cgrp)
2923 continue;
2924
2925 list_for_each_entry(link, &css->cgroup->cset_links, cset_link)
2926 cgroup_migrate_add_src(link->cset, cgrp,
2927 &preloaded_csets);
2928 }
f0d9a5f1 2929 spin_unlock_bh(&css_set_lock);
f8f22e53
TH
2930
2931 /* NULL dst indicates self on default hierarchy */
2932 ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets);
2933 if (ret)
2934 goto out_finish;
2935
f0d9a5f1 2936 spin_lock_bh(&css_set_lock);
f8f22e53 2937 list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) {
10265075 2938 struct task_struct *task, *ntask;
f8f22e53
TH
2939
2940 /* src_csets precede dst_csets, break on the first dst_cset */
2941 if (!src_cset->mg_src_cgrp)
2942 break;
2943
10265075
TH
2944 /* all tasks in src_csets need to be migrated */
2945 list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list)
2946 cgroup_taskset_add(task, &tset);
f8f22e53 2947 }
f0d9a5f1 2948 spin_unlock_bh(&css_set_lock);
f8f22e53 2949
10265075 2950 ret = cgroup_taskset_migrate(&tset, cgrp);
f8f22e53
TH
2951out_finish:
2952 cgroup_migrate_finish(&preloaded_csets);
3014dde7 2953 percpu_up_write(&cgroup_threadgroup_rwsem);
f8f22e53
TH
2954 return ret;
2955}
2956
2957/* change the enabled child controllers for a cgroup in the default hierarchy */
451af504
TH
2958static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
2959 char *buf, size_t nbytes,
2960 loff_t off)
f8f22e53 2961{
6e5c8307
TH
2962 u16 enable = 0, disable = 0;
2963 u16 css_enable, css_disable, old_sc, new_sc, old_ss, new_ss;
a9746d8d 2964 struct cgroup *cgrp, *child;
f8f22e53 2965 struct cgroup_subsys *ss;
451af504 2966 char *tok;
f8f22e53
TH
2967 int ssid, ret;
2968
2969 /*
d37167ab
TH
2970 * Parse input - space separated list of subsystem names prefixed
2971 * with either + or -.
f8f22e53 2972 */
451af504
TH
2973 buf = strstrip(buf);
2974 while ((tok = strsep(&buf, " "))) {
d37167ab
TH
2975 if (tok[0] == '\0')
2976 continue;
a7165264 2977 do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) {
fc5ed1e9
TH
2978 if (!cgroup_ssid_enabled(ssid) ||
2979 strcmp(tok + 1, ss->name))
f8f22e53
TH
2980 continue;
2981
2982 if (*tok == '+') {
7d331fa9
TH
2983 enable |= 1 << ssid;
2984 disable &= ~(1 << ssid);
f8f22e53 2985 } else if (*tok == '-') {
7d331fa9
TH
2986 disable |= 1 << ssid;
2987 enable &= ~(1 << ssid);
f8f22e53
TH
2988 } else {
2989 return -EINVAL;
2990 }
2991 break;
b4e0eeaf 2992 } while_each_subsys_mask();
f8f22e53
TH
2993 if (ssid == CGROUP_SUBSYS_COUNT)
2994 return -EINVAL;
2995 }
2996
a9746d8d
TH
2997 cgrp = cgroup_kn_lock_live(of->kn);
2998 if (!cgrp)
2999 return -ENODEV;
f8f22e53
TH
3000
3001 for_each_subsys(ss, ssid) {
3002 if (enable & (1 << ssid)) {
667c2491 3003 if (cgrp->subtree_control & (1 << ssid)) {
f8f22e53
TH
3004 enable &= ~(1 << ssid);
3005 continue;
3006 }
3007
c29adf24
TH
3008 /* unavailable or not enabled on the parent? */
3009 if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) ||
3010 (cgroup_parent(cgrp) &&
667c2491 3011 !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) {
c29adf24
TH
3012 ret = -ENOENT;
3013 goto out_unlock;
3014 }
f8f22e53 3015 } else if (disable & (1 << ssid)) {
667c2491 3016 if (!(cgrp->subtree_control & (1 << ssid))) {
f8f22e53
TH
3017 disable &= ~(1 << ssid);
3018 continue;
3019 }
3020
3021 /* a child has it enabled? */
3022 cgroup_for_each_live_child(child, cgrp) {
667c2491 3023 if (child->subtree_control & (1 << ssid)) {
f8f22e53 3024 ret = -EBUSY;
ddab2b6e 3025 goto out_unlock;
f8f22e53
TH
3026 }
3027 }
3028 }
3029 }
3030
3031 if (!enable && !disable) {
3032 ret = 0;
ddab2b6e 3033 goto out_unlock;
f8f22e53
TH
3034 }
3035
3036 /*
667c2491 3037 * Except for the root, subtree_control must be zero for a cgroup
f8f22e53
TH
3038 * with tasks so that child cgroups don't compete against tasks.
3039 */
d51f39b0 3040 if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) {
f8f22e53
TH
3041 ret = -EBUSY;
3042 goto out_unlock;
3043 }
3044
3045 /*
f63070d3
TH
3046 * Update subsys masks and calculate what needs to be done. More
3047 * subsystems than specified may need to be enabled or disabled
3048 * depending on subsystem dependencies.
3049 */
755bf5ee 3050 old_sc = cgrp->subtree_control;
8699b776 3051 old_ss = cgrp->subtree_ss_mask;
755bf5ee 3052 new_sc = (old_sc | enable) & ~disable;
8699b776 3053 new_ss = cgroup_calc_subtree_ss_mask(cgrp, new_sc);
f63070d3 3054
755bf5ee
TH
3055 css_enable = ~old_ss & new_ss;
3056 css_disable = old_ss & ~new_ss;
f63070d3
TH
3057 enable |= css_enable;
3058 disable |= css_disable;
c29adf24 3059
db6e3053
TH
3060 /*
3061 * Because css offlining is asynchronous, userland might try to
3062 * re-enable the same controller while the previous instance is
3063 * still around. In such cases, wait till it's gone using
3064 * offline_waitq.
3065 */
b4e0eeaf 3066 do_each_subsys_mask(ss, ssid, css_enable) {
db6e3053
TH
3067 cgroup_for_each_live_child(child, cgrp) {
3068 DEFINE_WAIT(wait);
3069
3070 if (!cgroup_css(child, ss))
3071 continue;
3072
3073 cgroup_get(child);
3074 prepare_to_wait(&child->offline_waitq, &wait,
3075 TASK_UNINTERRUPTIBLE);
3076 cgroup_kn_unlock(of->kn);
3077 schedule();
3078 finish_wait(&child->offline_waitq, &wait);
3079 cgroup_put(child);
3080
3081 return restart_syscall();
3082 }
b4e0eeaf 3083 } while_each_subsys_mask();
db6e3053 3084
755bf5ee 3085 cgrp->subtree_control = new_sc;
8699b776 3086 cgrp->subtree_ss_mask = new_ss;
755bf5ee 3087
f63070d3
TH
3088 /*
3089 * Create new csses or make the existing ones visible. A css is
3090 * created invisible if it's being implicitly enabled through
3091 * dependency. An invisible css is made visible when the userland
3092 * explicitly enables it.
f8f22e53 3093 */
996cd1fb 3094 do_each_subsys_mask(ss, ssid, enable) {
f8f22e53 3095 cgroup_for_each_live_child(child, cgrp) {
6cd0f5bb
TH
3096 if (css_enable & (1 << ssid)) {
3097 struct cgroup_subsys_state *css;
3098
3099 css = css_create(child, ss);
3100 if (IS_ERR(css)) {
3101 ret = PTR_ERR(css);
3102 goto err_undo_css;
3103 }
3104
3105 if (cgrp->subtree_control & (1 << ssid)) {
3106 ret = css_populate_dir(css, NULL);
3107 if (ret)
3108 goto err_undo_css;
3109 }
3110 } else {
4df8dc90
TH
3111 ret = css_populate_dir(cgroup_css(child, ss),
3112 NULL);
6cd0f5bb
TH
3113 if (ret)
3114 goto err_undo_css;
3115 }
f8f22e53 3116 }
996cd1fb 3117 } while_each_subsys_mask();
f8f22e53 3118
c29adf24
TH
3119 /*
3120 * At this point, cgroup_e_css() results reflect the new csses
3121 * making the following cgroup_update_dfl_csses() properly update
3122 * css associations of all tasks in the subtree.
3123 */
f8f22e53
TH
3124 ret = cgroup_update_dfl_csses(cgrp);
3125 if (ret)
3126 goto err_undo_css;
3127
f63070d3
TH
3128 /*
3129 * All tasks are migrated out of disabled csses. Kill or hide
3130 * them. A css is hidden when the userland requests it to be
b4536f0c
TH
3131 * disabled while other subsystems are still depending on it. The
3132 * css must not actively control resources and be in the vanilla
3133 * state if it's made visible again later. Controllers which may
3134 * be depended upon should provide ->css_reset() for this purpose.
f63070d3 3135 */
996cd1fb 3136 do_each_subsys_mask(ss, ssid, disable) {
f63070d3 3137 cgroup_for_each_live_child(child, cgrp) {
b4536f0c
TH
3138 struct cgroup_subsys_state *css = cgroup_css(child, ss);
3139
3140 if (css_disable & (1 << ssid)) {
3141 kill_css(css);
3142 } else {
4df8dc90 3143 css_clear_dir(css, NULL);
b4536f0c
TH
3144 if (ss->css_reset)
3145 ss->css_reset(css);
3146 }
f63070d3 3147 }
996cd1fb 3148 } while_each_subsys_mask();
f8f22e53
TH
3149
3150 kernfs_activate(cgrp->kn);
3151 ret = 0;
3152out_unlock:
a9746d8d 3153 cgroup_kn_unlock(of->kn);
451af504 3154 return ret ?: nbytes;
f8f22e53
TH
3155
3156err_undo_css:
755bf5ee 3157 cgrp->subtree_control = old_sc;
8699b776 3158 cgrp->subtree_ss_mask = old_ss;
f8f22e53 3159
996cd1fb 3160 do_each_subsys_mask(ss, ssid, enable) {
f8f22e53
TH
3161 cgroup_for_each_live_child(child, cgrp) {
3162 struct cgroup_subsys_state *css = cgroup_css(child, ss);
f63070d3
TH
3163
3164 if (!css)
3165 continue;
3166
3167 if (css_enable & (1 << ssid))
f8f22e53 3168 kill_css(css);
f63070d3 3169 else
4df8dc90 3170 css_clear_dir(css, NULL);
f8f22e53 3171 }
996cd1fb 3172 } while_each_subsys_mask();
f8f22e53
TH
3173 goto out_unlock;
3174}
3175
4a07c222 3176static int cgroup_events_show(struct seq_file *seq, void *v)
842b597e 3177{
4a07c222 3178 seq_printf(seq, "populated %d\n",
27bd4dbb 3179 cgroup_is_populated(seq_css(seq)->cgroup));
842b597e
TH
3180 return 0;
3181}
3182
2bd59d48
TH
3183static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
3184 size_t nbytes, loff_t off)
355e0c48 3185{
2bd59d48
TH
3186 struct cgroup *cgrp = of->kn->parent->priv;
3187 struct cftype *cft = of->kn->priv;
3188 struct cgroup_subsys_state *css;
a742c59d 3189 int ret;
355e0c48 3190
b4168640
TH
3191 if (cft->write)
3192 return cft->write(of, buf, nbytes, off);
3193
2bd59d48
TH
3194 /*
3195 * kernfs guarantees that a file isn't deleted with operations in
3196 * flight, which means that the matching css is and stays alive and
3197 * doesn't need to be pinned. The RCU locking is not necessary
3198 * either. It's just for the convenience of using cgroup_css().
3199 */
3200 rcu_read_lock();
3201 css = cgroup_css(cgrp, cft->ss);
3202 rcu_read_unlock();
a742c59d 3203
451af504 3204 if (cft->write_u64) {
a742c59d
TH
3205 unsigned long long v;
3206 ret = kstrtoull(buf, 0, &v);
3207 if (!ret)
3208 ret = cft->write_u64(css, cft, v);
3209 } else if (cft->write_s64) {
3210 long long v;
3211 ret = kstrtoll(buf, 0, &v);
3212 if (!ret)
3213 ret = cft->write_s64(css, cft, v);
e73d2c61 3214 } else {
a742c59d 3215 ret = -EINVAL;
e73d2c61 3216 }
2bd59d48 3217
a742c59d 3218 return ret ?: nbytes;
355e0c48
PM
3219}
3220
6612f05b 3221static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos)
db3b1497 3222{
2bd59d48 3223 return seq_cft(seq)->seq_start(seq, ppos);
db3b1497
PM
3224}
3225
6612f05b 3226static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos)
ddbcc7e8 3227{
2bd59d48 3228 return seq_cft(seq)->seq_next(seq, v, ppos);
ddbcc7e8
PM
3229}
3230
6612f05b 3231static void cgroup_seqfile_stop(struct seq_file *seq, void *v)
ddbcc7e8 3232{
2bd59d48 3233 seq_cft(seq)->seq_stop(seq, v);
ddbcc7e8
PM
3234}
3235
91796569 3236static int cgroup_seqfile_show(struct seq_file *m, void *arg)
e73d2c61 3237{
7da11279
TH
3238 struct cftype *cft = seq_cft(m);
3239 struct cgroup_subsys_state *css = seq_css(m);
e73d2c61 3240
2da8ca82
TH
3241 if (cft->seq_show)
3242 return cft->seq_show(m, arg);
e73d2c61 3243
f4c753b7 3244 if (cft->read_u64)
896f5199
TH
3245 seq_printf(m, "%llu\n", cft->read_u64(css, cft));
3246 else if (cft->read_s64)
3247 seq_printf(m, "%lld\n", cft->read_s64(css, cft));
3248 else
3249 return -EINVAL;
3250 return 0;
91796569
PM
3251}
3252
2bd59d48
TH
3253static struct kernfs_ops cgroup_kf_single_ops = {
3254 .atomic_write_len = PAGE_SIZE,
3255 .write = cgroup_file_write,
3256 .seq_show = cgroup_seqfile_show,
91796569
PM
3257};
3258
2bd59d48
TH
3259static struct kernfs_ops cgroup_kf_ops = {
3260 .atomic_write_len = PAGE_SIZE,
3261 .write = cgroup_file_write,
3262 .seq_start = cgroup_seqfile_start,
3263 .seq_next = cgroup_seqfile_next,
3264 .seq_stop = cgroup_seqfile_stop,
3265 .seq_show = cgroup_seqfile_show,
3266};
ddbcc7e8
PM
3267
3268/*
3269 * cgroup_rename - Only allow simple rename of directories in place.
3270 */
2bd59d48
TH
3271static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent,
3272 const char *new_name_str)
ddbcc7e8 3273{
2bd59d48 3274 struct cgroup *cgrp = kn->priv;
65dff759 3275 int ret;
65dff759 3276
2bd59d48 3277 if (kernfs_type(kn) != KERNFS_DIR)
ddbcc7e8 3278 return -ENOTDIR;
2bd59d48 3279 if (kn->parent != new_parent)
ddbcc7e8 3280 return -EIO;
65dff759 3281
6db8e85c
TH
3282 /*
3283 * This isn't a proper migration and its usefulness is very
aa6ec29b 3284 * limited. Disallow on the default hierarchy.
6db8e85c 3285 */
aa6ec29b 3286 if (cgroup_on_dfl(cgrp))
6db8e85c 3287 return -EPERM;
099fca32 3288
e1b2dc17 3289 /*
8353da1f 3290 * We're gonna grab cgroup_mutex which nests outside kernfs
e1b2dc17 3291 * active_ref. kernfs_rename() doesn't require active_ref
8353da1f 3292 * protection. Break them before grabbing cgroup_mutex.
e1b2dc17
TH
3293 */
3294 kernfs_break_active_protection(new_parent);
3295 kernfs_break_active_protection(kn);
099fca32 3296
2bd59d48 3297 mutex_lock(&cgroup_mutex);
099fca32 3298
2bd59d48 3299 ret = kernfs_rename(kn, new_parent, new_name_str);
099fca32 3300
2bd59d48 3301 mutex_unlock(&cgroup_mutex);
e1b2dc17
TH
3302
3303 kernfs_unbreak_active_protection(kn);
3304 kernfs_unbreak_active_protection(new_parent);
2bd59d48 3305 return ret;
099fca32
LZ
3306}
3307
49957f8e
TH
3308/* set uid and gid of cgroup dirs and files to that of the creator */
3309static int cgroup_kn_set_ugid(struct kernfs_node *kn)
3310{
3311 struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID,
3312 .ia_uid = current_fsuid(),
3313 .ia_gid = current_fsgid(), };
3314
3315 if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) &&
3316 gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID))
3317 return 0;
3318
3319 return kernfs_setattr(kn, &iattr);
3320}
3321
4df8dc90
TH
3322static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,
3323 struct cftype *cft)
ddbcc7e8 3324{
8d7e6fb0 3325 char name[CGROUP_FILE_NAME_MAX];
2bd59d48
TH
3326 struct kernfs_node *kn;
3327 struct lock_class_key *key = NULL;
49957f8e 3328 int ret;
05ef1d7c 3329
2bd59d48
TH
3330#ifdef CONFIG_DEBUG_LOCK_ALLOC
3331 key = &cft->lockdep_key;
3332#endif
3333 kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name),
3334 cgroup_file_mode(cft), 0, cft->kf_ops, cft,
dfeb0750 3335 NULL, key);
49957f8e
TH
3336 if (IS_ERR(kn))
3337 return PTR_ERR(kn);
3338
3339 ret = cgroup_kn_set_ugid(kn);
f8f22e53 3340 if (ret) {
49957f8e 3341 kernfs_remove(kn);
f8f22e53
TH
3342 return ret;
3343 }
3344
6f60eade
TH
3345 if (cft->file_offset) {
3346 struct cgroup_file *cfile = (void *)css + cft->file_offset;
3347
34c06254 3348 spin_lock_irq(&cgroup_file_kn_lock);
6f60eade 3349 cfile->kn = kn;
34c06254 3350 spin_unlock_irq(&cgroup_file_kn_lock);
6f60eade
TH
3351 }
3352
f8f22e53 3353 return 0;
ddbcc7e8
PM
3354}
3355
b1f28d31
TH
3356/**
3357 * cgroup_addrm_files - add or remove files to a cgroup directory
4df8dc90
TH
3358 * @css: the target css
3359 * @cgrp: the target cgroup (usually css->cgroup)
b1f28d31
TH
3360 * @cfts: array of cftypes to be added
3361 * @is_add: whether to add or remove
3362 *
3363 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
6732ed85 3364 * For removals, this function never fails.
b1f28d31 3365 */
4df8dc90
TH
3366static int cgroup_addrm_files(struct cgroup_subsys_state *css,
3367 struct cgroup *cgrp, struct cftype cfts[],
2bb566cb 3368 bool is_add)
ddbcc7e8 3369{
6732ed85 3370 struct cftype *cft, *cft_end = NULL;
b598dde3 3371 int ret = 0;
b1f28d31 3372
01f6474c 3373 lockdep_assert_held(&cgroup_mutex);
db0416b6 3374
6732ed85
TH
3375restart:
3376 for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) {
f33fddc2 3377 /* does cft->flags tell us to skip this file on @cgrp? */
05ebb6e6 3378 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp))
8cbbf2c9 3379 continue;
05ebb6e6 3380 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp))
873fe09e 3381 continue;
d51f39b0 3382 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp))
f33fddc2 3383 continue;
d51f39b0 3384 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp))
f33fddc2
G
3385 continue;
3386
2739d3cc 3387 if (is_add) {
4df8dc90 3388 ret = cgroup_add_file(css, cgrp, cft);
b1f28d31 3389 if (ret) {
ed3d261b
JP
3390 pr_warn("%s: failed to add %s, err=%d\n",
3391 __func__, cft->name, ret);
6732ed85
TH
3392 cft_end = cft;
3393 is_add = false;
3394 goto restart;
b1f28d31 3395 }
2739d3cc
LZ
3396 } else {
3397 cgroup_rm_file(cgrp, cft);
db0416b6 3398 }
ddbcc7e8 3399 }
b598dde3 3400 return ret;
ddbcc7e8
PM
3401}
3402
21a2d343 3403static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add)
8e3f6541
TH
3404{
3405 LIST_HEAD(pending);
2bb566cb 3406 struct cgroup_subsys *ss = cfts[0].ss;
3dd06ffa 3407 struct cgroup *root = &ss->root->cgrp;
492eb21b 3408 struct cgroup_subsys_state *css;
9ccece80 3409 int ret = 0;
8e3f6541 3410
01f6474c 3411 lockdep_assert_held(&cgroup_mutex);
e8c82d20 3412
e8c82d20 3413 /* add/rm files for all cgroups created before */
ca8bdcaf 3414 css_for_each_descendant_pre(css, cgroup_css(root, ss)) {
492eb21b
TH
3415 struct cgroup *cgrp = css->cgroup;
3416
88cb04b9 3417 if (!(css->flags & CSS_VISIBLE))
e8c82d20
LZ
3418 continue;
3419
4df8dc90 3420 ret = cgroup_addrm_files(css, cgrp, cfts, is_add);
9ccece80
TH
3421 if (ret)
3422 break;
8e3f6541 3423 }
21a2d343
TH
3424
3425 if (is_add && !ret)
3426 kernfs_activate(root->kn);
9ccece80 3427 return ret;
8e3f6541
TH
3428}
3429
2da440a2 3430static void cgroup_exit_cftypes(struct cftype *cfts)
8e3f6541 3431{
2bb566cb 3432 struct cftype *cft;
8e3f6541 3433
2bd59d48
TH
3434 for (cft = cfts; cft->name[0] != '\0'; cft++) {
3435 /* free copy for custom atomic_write_len, see init_cftypes() */
3436 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE)
3437 kfree(cft->kf_ops);
3438 cft->kf_ops = NULL;
2da440a2 3439 cft->ss = NULL;
a8ddc821
TH
3440
3441 /* revert flags set by cgroup core while adding @cfts */
05ebb6e6 3442 cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL);
2bd59d48 3443 }
2da440a2
TH
3444}
3445
2bd59d48 3446static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
2da440a2
TH
3447{
3448 struct cftype *cft;
3449
2bd59d48
TH
3450 for (cft = cfts; cft->name[0] != '\0'; cft++) {
3451 struct kernfs_ops *kf_ops;
3452
0adb0704
TH
3453 WARN_ON(cft->ss || cft->kf_ops);
3454
2bd59d48
TH
3455 if (cft->seq_start)
3456 kf_ops = &cgroup_kf_ops;
3457 else
3458 kf_ops = &cgroup_kf_single_ops;
3459
3460 /*
3461 * Ugh... if @cft wants a custom max_write_len, we need to
3462 * make a copy of kf_ops to set its atomic_write_len.
3463 */
3464 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) {
3465 kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL);
3466 if (!kf_ops) {
3467 cgroup_exit_cftypes(cfts);
3468 return -ENOMEM;
3469 }
3470 kf_ops->atomic_write_len = cft->max_write_len;
3471 }
8e3f6541 3472
2bd59d48 3473 cft->kf_ops = kf_ops;
2bb566cb 3474 cft->ss = ss;
2bd59d48 3475 }
2bb566cb 3476
2bd59d48 3477 return 0;
2da440a2
TH
3478}
3479
21a2d343
TH
3480static int cgroup_rm_cftypes_locked(struct cftype *cfts)
3481{
01f6474c 3482 lockdep_assert_held(&cgroup_mutex);
21a2d343
TH
3483
3484 if (!cfts || !cfts[0].ss)
3485 return -ENOENT;
3486
3487 list_del(&cfts->node);
3488 cgroup_apply_cftypes(cfts, false);
3489 cgroup_exit_cftypes(cfts);
3490 return 0;
8e3f6541 3491}
8e3f6541 3492
79578621
TH
3493/**
3494 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
79578621
TH
3495 * @cfts: zero-length name terminated array of cftypes
3496 *
2bb566cb
TH
3497 * Unregister @cfts. Files described by @cfts are removed from all
3498 * existing cgroups and all future cgroups won't have them either. This
3499 * function can be called anytime whether @cfts' subsys is attached or not.
79578621
TH
3500 *
3501 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
2bb566cb 3502 * registered.
79578621 3503 */
2bb566cb 3504int cgroup_rm_cftypes(struct cftype *cfts)
79578621 3505{
21a2d343 3506 int ret;
79578621 3507
01f6474c 3508 mutex_lock(&cgroup_mutex);
21a2d343 3509 ret = cgroup_rm_cftypes_locked(cfts);
01f6474c 3510 mutex_unlock(&cgroup_mutex);
21a2d343 3511 return ret;
80b13586
TH
3512}
3513
8e3f6541
TH
3514/**
3515 * cgroup_add_cftypes - add an array of cftypes to a subsystem
3516 * @ss: target cgroup subsystem
3517 * @cfts: zero-length name terminated array of cftypes
3518 *
3519 * Register @cfts to @ss. Files described by @cfts are created for all
3520 * existing cgroups to which @ss is attached and all future cgroups will
3521 * have them too. This function can be called anytime whether @ss is
3522 * attached or not.
3523 *
3524 * Returns 0 on successful registration, -errno on failure. Note that this
3525 * function currently returns 0 as long as @cfts registration is successful
3526 * even if some file creation attempts on existing cgroups fail.
3527 */
2cf669a5 3528static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
8e3f6541 3529{
9ccece80 3530 int ret;
8e3f6541 3531
fc5ed1e9 3532 if (!cgroup_ssid_enabled(ss->id))
c731ae1d
LZ
3533 return 0;
3534
dc5736ed
LZ
3535 if (!cfts || cfts[0].name[0] == '\0')
3536 return 0;
2bb566cb 3537
2bd59d48
TH
3538 ret = cgroup_init_cftypes(ss, cfts);
3539 if (ret)
3540 return ret;
79578621 3541
01f6474c 3542 mutex_lock(&cgroup_mutex);
21a2d343 3543
0adb0704 3544 list_add_tail(&cfts->node, &ss->cfts);
21a2d343 3545 ret = cgroup_apply_cftypes(cfts, true);
9ccece80 3546 if (ret)
21a2d343 3547 cgroup_rm_cftypes_locked(cfts);
79578621 3548
01f6474c 3549 mutex_unlock(&cgroup_mutex);
9ccece80 3550 return ret;
79578621
TH
3551}
3552
a8ddc821
TH
3553/**
3554 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
3555 * @ss: target cgroup subsystem
3556 * @cfts: zero-length name terminated array of cftypes
3557 *
3558 * Similar to cgroup_add_cftypes() but the added files are only used for
3559 * the default hierarchy.
3560 */
3561int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3562{
3563 struct cftype *cft;
3564
3565 for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
05ebb6e6 3566 cft->flags |= __CFTYPE_ONLY_ON_DFL;
a8ddc821
TH
3567 return cgroup_add_cftypes(ss, cfts);
3568}
3569
3570/**
3571 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
3572 * @ss: target cgroup subsystem
3573 * @cfts: zero-length name terminated array of cftypes
3574 *
3575 * Similar to cgroup_add_cftypes() but the added files are only used for
3576 * the legacy hierarchies.
3577 */
2cf669a5
TH
3578int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)
3579{
a8ddc821
TH
3580 struct cftype *cft;
3581
e4b7037c
TH
3582 for (cft = cfts; cft && cft->name[0] != '\0'; cft++)
3583 cft->flags |= __CFTYPE_NOT_ON_DFL;
2cf669a5
TH
3584 return cgroup_add_cftypes(ss, cfts);
3585}
3586
34c06254
TH
3587/**
3588 * cgroup_file_notify - generate a file modified event for a cgroup_file
3589 * @cfile: target cgroup_file
3590 *
3591 * @cfile must have been obtained by setting cftype->file_offset.
3592 */
3593void cgroup_file_notify(struct cgroup_file *cfile)
3594{
3595 unsigned long flags;
3596
3597 spin_lock_irqsave(&cgroup_file_kn_lock, flags);
3598 if (cfile->kn)
3599 kernfs_notify(cfile->kn);
3600 spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
3601}
3602
a043e3b2
LZ
3603/**
3604 * cgroup_task_count - count the number of tasks in a cgroup.
3605 * @cgrp: the cgroup in question
3606 *
3607 * Return the number of tasks in the cgroup.
3608 */
07bc356e 3609static int cgroup_task_count(const struct cgroup *cgrp)
bbcb81d0
PM
3610{
3611 int count = 0;
69d0206c 3612 struct cgrp_cset_link *link;
817929ec 3613
f0d9a5f1 3614 spin_lock_bh(&css_set_lock);
69d0206c
TH
3615 list_for_each_entry(link, &cgrp->cset_links, cset_link)
3616 count += atomic_read(&link->cset->refcount);
f0d9a5f1 3617 spin_unlock_bh(&css_set_lock);
bbcb81d0
PM
3618 return count;
3619}
3620
53fa5261 3621/**
492eb21b 3622 * css_next_child - find the next child of a given css
c2931b70
TH
3623 * @pos: the current position (%NULL to initiate traversal)
3624 * @parent: css whose children to walk
53fa5261 3625 *
c2931b70 3626 * This function returns the next child of @parent and should be called
87fb54f1 3627 * under either cgroup_mutex or RCU read lock. The only requirement is
c2931b70
TH
3628 * that @parent and @pos are accessible. The next sibling is guaranteed to
3629 * be returned regardless of their states.
3630 *
3631 * If a subsystem synchronizes ->css_online() and the start of iteration, a
3632 * css which finished ->css_online() is guaranteed to be visible in the
3633 * future iterations and will stay visible until the last reference is put.
3634 * A css which hasn't finished ->css_online() or already finished
3635 * ->css_offline() may show up during traversal. It's each subsystem's
3636 * responsibility to synchronize against on/offlining.
53fa5261 3637 */
c2931b70
TH
3638struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos,
3639 struct cgroup_subsys_state *parent)
53fa5261 3640{
c2931b70 3641 struct cgroup_subsys_state *next;
53fa5261 3642
8353da1f 3643 cgroup_assert_mutex_or_rcu_locked();
53fa5261
TH
3644
3645 /*
de3f0341
TH
3646 * @pos could already have been unlinked from the sibling list.
3647 * Once a cgroup is removed, its ->sibling.next is no longer
3648 * updated when its next sibling changes. CSS_RELEASED is set when
3649 * @pos is taken off list, at which time its next pointer is valid,
3650 * and, as releases are serialized, the one pointed to by the next
3651 * pointer is guaranteed to not have started release yet. This
3652 * implies that if we observe !CSS_RELEASED on @pos in this RCU
3653 * critical section, the one pointed to by its next pointer is
3654 * guaranteed to not have finished its RCU grace period even if we
3655 * have dropped rcu_read_lock() inbetween iterations.
3b287a50 3656 *
de3f0341
TH
3657 * If @pos has CSS_RELEASED set, its next pointer can't be
3658 * dereferenced; however, as each css is given a monotonically
3659 * increasing unique serial number and always appended to the
3660 * sibling list, the next one can be found by walking the parent's
3661 * children until the first css with higher serial number than
3662 * @pos's. While this path can be slower, it happens iff iteration
3663 * races against release and the race window is very small.
53fa5261 3664 */
3b287a50 3665 if (!pos) {
c2931b70
TH
3666 next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling);
3667 } else if (likely(!(pos->flags & CSS_RELEASED))) {
3668 next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling);
3b287a50 3669 } else {
c2931b70 3670 list_for_each_entry_rcu(next, &parent->children, sibling)
3b287a50
TH
3671 if (next->serial_nr > pos->serial_nr)
3672 break;
53fa5261
TH
3673 }
3674
3b281afb
TH
3675 /*
3676 * @next, if not pointing to the head, can be dereferenced and is
c2931b70 3677 * the next sibling.
3b281afb 3678 */
c2931b70
TH
3679 if (&next->sibling != &parent->children)
3680 return next;
3b281afb 3681 return NULL;
53fa5261 3682}
53fa5261 3683
574bd9f7 3684/**
492eb21b 3685 * css_next_descendant_pre - find the next descendant for pre-order walk
574bd9f7 3686 * @pos: the current position (%NULL to initiate traversal)
492eb21b 3687 * @root: css whose descendants to walk
574bd9f7 3688 *
492eb21b 3689 * To be used by css_for_each_descendant_pre(). Find the next descendant
bd8815a6
TH
3690 * to visit for pre-order traversal of @root's descendants. @root is
3691 * included in the iteration and the first node to be visited.
75501a6d 3692 *
87fb54f1
TH
3693 * While this function requires cgroup_mutex or RCU read locking, it
3694 * doesn't require the whole traversal to be contained in a single critical
3695 * section. This function will return the correct next descendant as long
3696 * as both @pos and @root are accessible and @pos is a descendant of @root.
c2931b70
TH
3697 *
3698 * If a subsystem synchronizes ->css_online() and the start of iteration, a
3699 * css which finished ->css_online() is guaranteed to be visible in the
3700 * future iterations and will stay visible until the last reference is put.
3701 * A css which hasn't finished ->css_online() or already finished
3702 * ->css_offline() may show up during traversal. It's each subsystem's
3703 * responsibility to synchronize against on/offlining.
574bd9f7 3704 */
492eb21b
TH
3705struct cgroup_subsys_state *
3706css_next_descendant_pre(struct cgroup_subsys_state *pos,
3707 struct cgroup_subsys_state *root)
574bd9f7 3708{
492eb21b 3709 struct cgroup_subsys_state *next;
574bd9f7 3710
8353da1f 3711 cgroup_assert_mutex_or_rcu_locked();
574bd9f7 3712
bd8815a6 3713 /* if first iteration, visit @root */
7805d000 3714 if (!pos)
bd8815a6 3715 return root;
574bd9f7
TH
3716
3717 /* visit the first child if exists */
492eb21b 3718 next = css_next_child(NULL, pos);
574bd9f7
TH
3719 if (next)
3720 return next;
3721
3722 /* no child, visit my or the closest ancestor's next sibling */
492eb21b 3723 while (pos != root) {
5c9d535b 3724 next = css_next_child(pos, pos->parent);
75501a6d 3725 if (next)
574bd9f7 3726 return next;
5c9d535b 3727 pos = pos->parent;
7805d000 3728 }
574bd9f7
TH
3729
3730 return NULL;
3731}
574bd9f7 3732
12a9d2fe 3733/**
492eb21b
TH
3734 * css_rightmost_descendant - return the rightmost descendant of a css
3735 * @pos: css of interest
12a9d2fe 3736 *
492eb21b
TH
3737 * Return the rightmost descendant of @pos. If there's no descendant, @pos
3738 * is returned. This can be used during pre-order traversal to skip
12a9d2fe 3739 * subtree of @pos.
75501a6d 3740 *
87fb54f1
TH
3741 * While this function requires cgroup_mutex or RCU read locking, it
3742 * doesn't require the whole traversal to be contained in a single critical
3743 * section. This function will return the correct rightmost descendant as
3744 * long as @pos is accessible.
12a9d2fe 3745 */
492eb21b
TH
3746struct cgroup_subsys_state *
3747css_rightmost_descendant(struct cgroup_subsys_state *pos)
12a9d2fe 3748{
492eb21b 3749 struct cgroup_subsys_state *last, *tmp;
12a9d2fe 3750
8353da1f 3751 cgroup_assert_mutex_or_rcu_locked();
12a9d2fe
TH
3752
3753 do {
3754 last = pos;
3755 /* ->prev isn't RCU safe, walk ->next till the end */
3756 pos = NULL;
492eb21b 3757 css_for_each_child(tmp, last)
12a9d2fe
TH
3758 pos = tmp;
3759 } while (pos);
3760
3761 return last;
3762}
12a9d2fe 3763
492eb21b
TH
3764static struct cgroup_subsys_state *
3765css_leftmost_descendant(struct cgroup_subsys_state *pos)
574bd9f7 3766{
492eb21b 3767 struct cgroup_subsys_state *last;
574bd9f7
TH
3768
3769 do {
3770 last = pos;
492eb21b 3771 pos = css_next_child(NULL, pos);
574bd9f7
TH
3772 } while (pos);
3773
3774 return last;
3775}
3776
3777/**
492eb21b 3778 * css_next_descendant_post - find the next descendant for post-order walk
574bd9f7 3779 * @pos: the current position (%NULL to initiate traversal)
492eb21b 3780 * @root: css whose descendants to walk
574bd9f7 3781 *
492eb21b 3782 * To be used by css_for_each_descendant_post(). Find the next descendant
bd8815a6
TH
3783 * to visit for post-order traversal of @root's descendants. @root is
3784 * included in the iteration and the last node to be visited.
75501a6d 3785 *
87fb54f1
TH
3786 * While this function requires cgroup_mutex or RCU read locking, it
3787 * doesn't require the whole traversal to be contained in a single critical
3788 * section. This function will return the correct next descendant as long
3789 * as both @pos and @cgroup are accessible and @pos is a descendant of
3790 * @cgroup.
c2931b70
TH
3791 *
3792 * If a subsystem synchronizes ->css_online() and the start of iteration, a
3793 * css which finished ->css_online() is guaranteed to be visible in the
3794 * future iterations and will stay visible until the last reference is put.
3795 * A css which hasn't finished ->css_online() or already finished
3796 * ->css_offline() may show up during traversal. It's each subsystem's
3797 * responsibility to synchronize against on/offlining.
574bd9f7 3798 */
492eb21b
TH
3799struct cgroup_subsys_state *
3800css_next_descendant_post(struct cgroup_subsys_state *pos,
3801 struct cgroup_subsys_state *root)
574bd9f7 3802{
492eb21b 3803 struct cgroup_subsys_state *next;
574bd9f7 3804
8353da1f 3805 cgroup_assert_mutex_or_rcu_locked();
574bd9f7 3806
58b79a91
TH
3807 /* if first iteration, visit leftmost descendant which may be @root */
3808 if (!pos)
3809 return css_leftmost_descendant(root);
574bd9f7 3810
bd8815a6
TH
3811 /* if we visited @root, we're done */
3812 if (pos == root)
3813 return NULL;
3814
574bd9f7 3815 /* if there's an unvisited sibling, visit its leftmost descendant */
5c9d535b 3816 next = css_next_child(pos, pos->parent);
75501a6d 3817 if (next)
492eb21b 3818 return css_leftmost_descendant(next);
574bd9f7
TH
3819
3820 /* no sibling left, visit parent */
5c9d535b 3821 return pos->parent;
574bd9f7 3822}
574bd9f7 3823
f3d46500
TH
3824/**
3825 * css_has_online_children - does a css have online children
3826 * @css: the target css
3827 *
3828 * Returns %true if @css has any online children; otherwise, %false. This
3829 * function can be called from any context but the caller is responsible
3830 * for synchronizing against on/offlining as necessary.
3831 */
3832bool css_has_online_children(struct cgroup_subsys_state *css)
cbc125ef 3833{
f3d46500
TH
3834 struct cgroup_subsys_state *child;
3835 bool ret = false;
cbc125ef
TH
3836
3837 rcu_read_lock();
f3d46500 3838 css_for_each_child(child, css) {
99bae5f9 3839 if (child->flags & CSS_ONLINE) {
f3d46500
TH
3840 ret = true;
3841 break;
cbc125ef
TH
3842 }
3843 }
3844 rcu_read_unlock();
f3d46500 3845 return ret;
574bd9f7 3846}
574bd9f7 3847
0942eeee 3848/**
ecb9d535 3849 * css_task_iter_advance_css_set - advance a task itererator to the next css_set
0942eeee
TH
3850 * @it: the iterator to advance
3851 *
3852 * Advance @it to the next css_set to walk.
d515876e 3853 */
ecb9d535 3854static void css_task_iter_advance_css_set(struct css_task_iter *it)
d515876e 3855{
0f0a2b4f 3856 struct list_head *l = it->cset_pos;
d515876e
TH
3857 struct cgrp_cset_link *link;
3858 struct css_set *cset;
3859
f0d9a5f1 3860 lockdep_assert_held(&css_set_lock);
ed27b9f7 3861
d515876e
TH
3862 /* Advance to the next non-empty css_set */
3863 do {
3864 l = l->next;
0f0a2b4f
TH
3865 if (l == it->cset_head) {
3866 it->cset_pos = NULL;
ecb9d535 3867 it->task_pos = NULL;
d515876e
TH
3868 return;
3869 }
3ebb2b6e
TH
3870
3871 if (it->ss) {
3872 cset = container_of(l, struct css_set,
3873 e_cset_node[it->ss->id]);
3874 } else {
3875 link = list_entry(l, struct cgrp_cset_link, cset_link);
3876 cset = link->cset;
3877 }
0de0942d 3878 } while (!css_set_populated(cset));
c7561128 3879
0f0a2b4f 3880 it->cset_pos = l;
c7561128
TH
3881
3882 if (!list_empty(&cset->tasks))
0f0a2b4f 3883 it->task_pos = cset->tasks.next;
c7561128 3884 else
0f0a2b4f
TH
3885 it->task_pos = cset->mg_tasks.next;
3886
3887 it->tasks_head = &cset->tasks;
3888 it->mg_tasks_head = &cset->mg_tasks;
ed27b9f7
TH
3889
3890 /*
3891 * We don't keep css_sets locked across iteration steps and thus
3892 * need to take steps to ensure that iteration can be resumed after
3893 * the lock is re-acquired. Iteration is performed at two levels -
3894 * css_sets and tasks in them.
3895 *
3896 * Once created, a css_set never leaves its cgroup lists, so a
3897 * pinned css_set is guaranteed to stay put and we can resume
3898 * iteration afterwards.
3899 *
3900 * Tasks may leave @cset across iteration steps. This is resolved
3901 * by registering each iterator with the css_set currently being
3902 * walked and making css_set_move_task() advance iterators whose
3903 * next task is leaving.
3904 */
3905 if (it->cur_cset) {
3906 list_del(&it->iters_node);
3907 put_css_set_locked(it->cur_cset);
3908 }
3909 get_css_set(cset);
3910 it->cur_cset = cset;
3911 list_add(&it->iters_node, &cset->task_iters);
d515876e
TH
3912}
3913
ecb9d535
TH
3914static void css_task_iter_advance(struct css_task_iter *it)
3915{
3916 struct list_head *l = it->task_pos;
3917
f0d9a5f1 3918 lockdep_assert_held(&css_set_lock);
ecb9d535
TH
3919 WARN_ON_ONCE(!l);
3920
3921 /*
3922 * Advance iterator to find next entry. cset->tasks is consumed
3923 * first and then ->mg_tasks. After ->mg_tasks, we move onto the
3924 * next cset.
3925 */
3926 l = l->next;
3927
3928 if (l == it->tasks_head)
3929 l = it->mg_tasks_head->next;
3930
3931 if (l == it->mg_tasks_head)
3932 css_task_iter_advance_css_set(it);
3933 else
3934 it->task_pos = l;
3935}
3936
0942eeee 3937/**
72ec7029
TH
3938 * css_task_iter_start - initiate task iteration
3939 * @css: the css to walk tasks of
0942eeee
TH
3940 * @it: the task iterator to use
3941 *
72ec7029
TH
3942 * Initiate iteration through the tasks of @css. The caller can call
3943 * css_task_iter_next() to walk through the tasks until the function
3944 * returns NULL. On completion of iteration, css_task_iter_end() must be
3945 * called.
0942eeee 3946 */
72ec7029
TH
3947void css_task_iter_start(struct cgroup_subsys_state *css,
3948 struct css_task_iter *it)
817929ec 3949{
56fde9e0
TH
3950 /* no one should try to iterate before mounting cgroups */
3951 WARN_ON_ONCE(!use_task_css_set_links);
31a7df01 3952
ed27b9f7
TH
3953 memset(it, 0, sizeof(*it));
3954
f0d9a5f1 3955 spin_lock_bh(&css_set_lock);
c59cd3d8 3956
3ebb2b6e
TH
3957 it->ss = css->ss;
3958
3959 if (it->ss)
3960 it->cset_pos = &css->cgroup->e_csets[css->ss->id];
3961 else
3962 it->cset_pos = &css->cgroup->cset_links;
3963
0f0a2b4f 3964 it->cset_head = it->cset_pos;
c59cd3d8 3965
ecb9d535 3966 css_task_iter_advance_css_set(it);
ed27b9f7 3967
f0d9a5f1 3968 spin_unlock_bh(&css_set_lock);
817929ec
PM
3969}
3970
0942eeee 3971/**
72ec7029 3972 * css_task_iter_next - return the next task for the iterator
0942eeee
TH
3973 * @it: the task iterator being iterated
3974 *
3975 * The "next" function for task iteration. @it should have been
72ec7029
TH
3976 * initialized via css_task_iter_start(). Returns NULL when the iteration
3977 * reaches the end.
0942eeee 3978 */
72ec7029 3979struct task_struct *css_task_iter_next(struct css_task_iter *it)
817929ec 3980{
d5745675 3981 if (it->cur_task) {
ed27b9f7 3982 put_task_struct(it->cur_task);
d5745675
TH
3983 it->cur_task = NULL;
3984 }
ed27b9f7 3985
f0d9a5f1 3986 spin_lock_bh(&css_set_lock);
ed27b9f7 3987
d5745675
TH
3988 if (it->task_pos) {
3989 it->cur_task = list_entry(it->task_pos, struct task_struct,
3990 cg_list);
3991 get_task_struct(it->cur_task);
3992 css_task_iter_advance(it);
3993 }
ed27b9f7 3994
f0d9a5f1 3995 spin_unlock_bh(&css_set_lock);
ed27b9f7
TH
3996
3997 return it->cur_task;
817929ec
PM
3998}
3999
0942eeee 4000/**
72ec7029 4001 * css_task_iter_end - finish task iteration
0942eeee
TH
4002 * @it: the task iterator to finish
4003 *
72ec7029 4004 * Finish task iteration started by css_task_iter_start().
0942eeee 4005 */
72ec7029 4006void css_task_iter_end(struct css_task_iter *it)
31a7df01 4007{
ed27b9f7 4008 if (it->cur_cset) {
f0d9a5f1 4009 spin_lock_bh(&css_set_lock);
ed27b9f7
TH
4010 list_del(&it->iters_node);
4011 put_css_set_locked(it->cur_cset);
f0d9a5f1 4012 spin_unlock_bh(&css_set_lock);
ed27b9f7
TH
4013 }
4014
4015 if (it->cur_task)
4016 put_task_struct(it->cur_task);
31a7df01
CW
4017}
4018
4019/**
8cc99345
TH
4020 * cgroup_trasnsfer_tasks - move tasks from one cgroup to another
4021 * @to: cgroup to which the tasks will be moved
4022 * @from: cgroup in which the tasks currently reside
31a7df01 4023 *
eaf797ab
TH
4024 * Locking rules between cgroup_post_fork() and the migration path
4025 * guarantee that, if a task is forking while being migrated, the new child
4026 * is guaranteed to be either visible in the source cgroup after the
4027 * parent's migration is complete or put into the target cgroup. No task
4028 * can slip out of migration through forking.
31a7df01 4029 */
8cc99345 4030int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from)
31a7df01 4031{
952aaa12
TH
4032 LIST_HEAD(preloaded_csets);
4033 struct cgrp_cset_link *link;
72ec7029 4034 struct css_task_iter it;
e406d1cf 4035 struct task_struct *task;
952aaa12 4036 int ret;
31a7df01 4037
952aaa12 4038 mutex_lock(&cgroup_mutex);
31a7df01 4039
952aaa12 4040 /* all tasks in @from are being moved, all csets are source */
f0d9a5f1 4041 spin_lock_bh(&css_set_lock);
952aaa12
TH
4042 list_for_each_entry(link, &from->cset_links, cset_link)
4043 cgroup_migrate_add_src(link->cset, to, &preloaded_csets);
f0d9a5f1 4044 spin_unlock_bh(&css_set_lock);
31a7df01 4045
952aaa12
TH
4046 ret = cgroup_migrate_prepare_dst(to, &preloaded_csets);
4047 if (ret)
4048 goto out_err;
8cc99345 4049
952aaa12 4050 /*
2cfa2b19 4051 * Migrate tasks one-by-one until @from is empty. This fails iff
952aaa12
TH
4052 * ->can_attach() fails.
4053 */
e406d1cf 4054 do {
9d800df1 4055 css_task_iter_start(&from->self, &it);
e406d1cf
TH
4056 task = css_task_iter_next(&it);
4057 if (task)
4058 get_task_struct(task);
4059 css_task_iter_end(&it);
4060
4061 if (task) {
9af2ec45 4062 ret = cgroup_migrate(task, false, to);
e406d1cf
TH
4063 put_task_struct(task);
4064 }
4065 } while (task && !ret);
952aaa12
TH
4066out_err:
4067 cgroup_migrate_finish(&preloaded_csets);
47cfcd09 4068 mutex_unlock(&cgroup_mutex);
e406d1cf 4069 return ret;
8cc99345
TH
4070}
4071
bbcb81d0 4072/*
102a775e 4073 * Stuff for reading the 'tasks'/'procs' files.
bbcb81d0
PM
4074 *
4075 * Reading this file can return large amounts of data if a cgroup has
4076 * *lots* of attached tasks. So it may need several calls to read(),
4077 * but we cannot guarantee that the information we produce is correct
4078 * unless we produce it entirely atomically.
4079 *
bbcb81d0 4080 */
bbcb81d0 4081
24528255
LZ
4082/* which pidlist file are we talking about? */
4083enum cgroup_filetype {
4084 CGROUP_FILE_PROCS,
4085 CGROUP_FILE_TASKS,
4086};
4087
4088/*
4089 * A pidlist is a list of pids that virtually represents the contents of one
4090 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
4091 * a pair (one each for procs, tasks) for each pid namespace that's relevant
4092 * to the cgroup.
4093 */
4094struct cgroup_pidlist {
4095 /*
4096 * used to find which pidlist is wanted. doesn't change as long as
4097 * this particular list stays in the list.
4098 */
4099 struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
4100 /* array of xids */
4101 pid_t *list;
4102 /* how many elements the above list has */
4103 int length;
24528255
LZ
4104 /* each of these stored in a list by its cgroup */
4105 struct list_head links;
4106 /* pointer to the cgroup we belong to, for list removal purposes */
4107 struct cgroup *owner;
b1a21367
TH
4108 /* for delayed destruction */
4109 struct delayed_work destroy_dwork;
24528255
LZ
4110};
4111
d1d9fd33
BB
4112/*
4113 * The following two functions "fix" the issue where there are more pids
4114 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
4115 * TODO: replace with a kernel-wide solution to this problem
4116 */
4117#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2))
4118static void *pidlist_allocate(int count)
4119{
4120 if (PIDLIST_TOO_LARGE(count))
4121 return vmalloc(count * sizeof(pid_t));
4122 else
4123 return kmalloc(count * sizeof(pid_t), GFP_KERNEL);
4124}
b1a21367 4125
d1d9fd33
BB
4126static void pidlist_free(void *p)
4127{
58794514 4128 kvfree(p);
d1d9fd33 4129}
d1d9fd33 4130
b1a21367
TH
4131/*
4132 * Used to destroy all pidlists lingering waiting for destroy timer. None
4133 * should be left afterwards.
4134 */
4135static void cgroup_pidlist_destroy_all(struct cgroup *cgrp)
4136{
4137 struct cgroup_pidlist *l, *tmp_l;
4138
4139 mutex_lock(&cgrp->pidlist_mutex);
4140 list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links)
4141 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0);
4142 mutex_unlock(&cgrp->pidlist_mutex);
4143
4144 flush_workqueue(cgroup_pidlist_destroy_wq);
4145 BUG_ON(!list_empty(&cgrp->pidlists));
4146}
4147
4148static void cgroup_pidlist_destroy_work_fn(struct work_struct *work)
4149{
4150 struct delayed_work *dwork = to_delayed_work(work);
4151 struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist,
4152 destroy_dwork);
4153 struct cgroup_pidlist *tofree = NULL;
4154
4155 mutex_lock(&l->owner->pidlist_mutex);
b1a21367
TH
4156
4157 /*
04502365
TH
4158 * Destroy iff we didn't get queued again. The state won't change
4159 * as destroy_dwork can only be queued while locked.
b1a21367 4160 */
04502365 4161 if (!delayed_work_pending(dwork)) {
b1a21367
TH
4162 list_del(&l->links);
4163 pidlist_free(l->list);
4164 put_pid_ns(l->key.ns);
4165 tofree = l;
4166 }
4167
b1a21367
TH
4168 mutex_unlock(&l->owner->pidlist_mutex);
4169 kfree(tofree);
4170}
4171
bbcb81d0 4172/*
102a775e 4173 * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries
6ee211ad 4174 * Returns the number of unique elements.
bbcb81d0 4175 */
6ee211ad 4176static int pidlist_uniq(pid_t *list, int length)
bbcb81d0 4177{
102a775e 4178 int src, dest = 1;
102a775e
BB
4179
4180 /*
4181 * we presume the 0th element is unique, so i starts at 1. trivial
4182 * edge cases first; no work needs to be done for either
4183 */
4184 if (length == 0 || length == 1)
4185 return length;
4186 /* src and dest walk down the list; dest counts unique elements */
4187 for (src = 1; src < length; src++) {
4188 /* find next unique element */
4189 while (list[src] == list[src-1]) {
4190 src++;
4191 if (src == length)
4192 goto after;
4193 }
4194 /* dest always points to where the next unique element goes */
4195 list[dest] = list[src];
4196 dest++;
4197 }
4198after:
102a775e
BB
4199 return dest;
4200}
4201
afb2bc14
TH
4202/*
4203 * The two pid files - task and cgroup.procs - guaranteed that the result
4204 * is sorted, which forced this whole pidlist fiasco. As pid order is
4205 * different per namespace, each namespace needs differently sorted list,
4206 * making it impossible to use, for example, single rbtree of member tasks
4207 * sorted by task pointer. As pidlists can be fairly large, allocating one
4208 * per open file is dangerous, so cgroup had to implement shared pool of
4209 * pidlists keyed by cgroup and namespace.
4210 *
4211 * All this extra complexity was caused by the original implementation
4212 * committing to an entirely unnecessary property. In the long term, we
aa6ec29b
TH
4213 * want to do away with it. Explicitly scramble sort order if on the
4214 * default hierarchy so that no such expectation exists in the new
4215 * interface.
afb2bc14
TH
4216 *
4217 * Scrambling is done by swapping every two consecutive bits, which is
4218 * non-identity one-to-one mapping which disturbs sort order sufficiently.
4219 */
4220static pid_t pid_fry(pid_t pid)
4221{
4222 unsigned a = pid & 0x55555555;
4223 unsigned b = pid & 0xAAAAAAAA;
4224
4225 return (a << 1) | (b >> 1);
4226}
4227
4228static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid)
4229{
aa6ec29b 4230 if (cgroup_on_dfl(cgrp))
afb2bc14
TH
4231 return pid_fry(pid);
4232 else
4233 return pid;
4234}
4235
102a775e
BB
4236static int cmppid(const void *a, const void *b)
4237{
4238 return *(pid_t *)a - *(pid_t *)b;
4239}
4240
afb2bc14
TH
4241static int fried_cmppid(const void *a, const void *b)
4242{
4243 return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b);
4244}
4245
e6b81710
TH
4246static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp,
4247 enum cgroup_filetype type)
4248{
4249 struct cgroup_pidlist *l;
4250 /* don't need task_nsproxy() if we're looking at ourself */
4251 struct pid_namespace *ns = task_active_pid_ns(current);
4252
4253 lockdep_assert_held(&cgrp->pidlist_mutex);
4254
4255 list_for_each_entry(l, &cgrp->pidlists, links)
4256 if (l->key.type == type && l->key.ns == ns)
4257 return l;
4258 return NULL;
4259}
4260
72a8cb30
BB
4261/*
4262 * find the appropriate pidlist for our purpose (given procs vs tasks)
4263 * returns with the lock on that pidlist already held, and takes care
4264 * of the use count, or returns NULL with no locks held if we're out of
4265 * memory.
4266 */
e6b81710
TH
4267static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
4268 enum cgroup_filetype type)
72a8cb30
BB
4269{
4270 struct cgroup_pidlist *l;
b70cc5fd 4271
e6b81710
TH
4272 lockdep_assert_held(&cgrp->pidlist_mutex);
4273
4274 l = cgroup_pidlist_find(cgrp, type);
4275 if (l)
4276 return l;
4277
72a8cb30 4278 /* entry not found; create a new one */
f4f4be2b 4279 l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL);
e6b81710 4280 if (!l)
72a8cb30 4281 return l;
e6b81710 4282
b1a21367 4283 INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn);
72a8cb30 4284 l->key.type = type;
e6b81710
TH
4285 /* don't need task_nsproxy() if we're looking at ourself */
4286 l->key.ns = get_pid_ns(task_active_pid_ns(current));
72a8cb30
BB
4287 l->owner = cgrp;
4288 list_add(&l->links, &cgrp->pidlists);
72a8cb30
BB
4289 return l;
4290}
4291
102a775e
BB
4292/*
4293 * Load a cgroup's pidarray with either procs' tgids or tasks' pids
4294 */
72a8cb30
BB
4295static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type,
4296 struct cgroup_pidlist **lp)
102a775e
BB
4297{
4298 pid_t *array;
4299 int length;
4300 int pid, n = 0; /* used for populating the array */
72ec7029 4301 struct css_task_iter it;
817929ec 4302 struct task_struct *tsk;
102a775e
BB
4303 struct cgroup_pidlist *l;
4304
4bac00d1
TH
4305 lockdep_assert_held(&cgrp->pidlist_mutex);
4306
102a775e
BB
4307 /*
4308 * If cgroup gets more users after we read count, we won't have
4309 * enough space - tough. This race is indistinguishable to the
4310 * caller from the case that the additional cgroup users didn't
4311 * show up until sometime later on.
4312 */
4313 length = cgroup_task_count(cgrp);
d1d9fd33 4314 array = pidlist_allocate(length);
102a775e
BB
4315 if (!array)
4316 return -ENOMEM;
4317 /* now, populate the array */
9d800df1 4318 css_task_iter_start(&cgrp->self, &it);
72ec7029 4319 while ((tsk = css_task_iter_next(&it))) {
102a775e 4320 if (unlikely(n == length))
817929ec 4321 break;
102a775e 4322 /* get tgid or pid for procs or tasks file respectively */
72a8cb30
BB
4323 if (type == CGROUP_FILE_PROCS)
4324 pid = task_tgid_vnr(tsk);
4325 else
4326 pid = task_pid_vnr(tsk);
102a775e
BB
4327 if (pid > 0) /* make sure to only use valid results */
4328 array[n++] = pid;
817929ec 4329 }
72ec7029 4330 css_task_iter_end(&it);
102a775e
BB
4331 length = n;
4332 /* now sort & (if procs) strip out duplicates */
aa6ec29b 4333 if (cgroup_on_dfl(cgrp))
afb2bc14
TH
4334 sort(array, length, sizeof(pid_t), fried_cmppid, NULL);
4335 else
4336 sort(array, length, sizeof(pid_t), cmppid, NULL);
72a8cb30 4337 if (type == CGROUP_FILE_PROCS)
6ee211ad 4338 length = pidlist_uniq(array, length);
e6b81710 4339
e6b81710 4340 l = cgroup_pidlist_find_create(cgrp, type);
72a8cb30 4341 if (!l) {
d1d9fd33 4342 pidlist_free(array);
72a8cb30 4343 return -ENOMEM;
102a775e 4344 }
e6b81710
TH
4345
4346 /* store array, freeing old if necessary */
d1d9fd33 4347 pidlist_free(l->list);
102a775e
BB
4348 l->list = array;
4349 l->length = length;
72a8cb30 4350 *lp = l;
102a775e 4351 return 0;
bbcb81d0
PM
4352}
4353
846c7bb0 4354/**
a043e3b2 4355 * cgroupstats_build - build and fill cgroupstats
846c7bb0
BS
4356 * @stats: cgroupstats to fill information into
4357 * @dentry: A dentry entry belonging to the cgroup for which stats have
4358 * been requested.
a043e3b2
LZ
4359 *
4360 * Build and fill cgroupstats so that taskstats can export it to user
4361 * space.
846c7bb0
BS
4362 */
4363int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)
4364{
2bd59d48 4365 struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
bd89aabc 4366 struct cgroup *cgrp;
72ec7029 4367 struct css_task_iter it;
846c7bb0 4368 struct task_struct *tsk;
33d283be 4369
2bd59d48
TH
4370 /* it should be kernfs_node belonging to cgroupfs and is a directory */
4371 if (dentry->d_sb->s_type != &cgroup_fs_type || !kn ||
4372 kernfs_type(kn) != KERNFS_DIR)
4373 return -EINVAL;
4374
bad34660
LZ
4375 mutex_lock(&cgroup_mutex);
4376
846c7bb0 4377 /*
2bd59d48 4378 * We aren't being called from kernfs and there's no guarantee on
ec903c0c 4379 * @kn->priv's validity. For this and css_tryget_online_from_dir(),
2bd59d48 4380 * @kn->priv is RCU safe. Let's do the RCU dancing.
846c7bb0 4381 */
2bd59d48
TH
4382 rcu_read_lock();
4383 cgrp = rcu_dereference(kn->priv);
bad34660 4384 if (!cgrp || cgroup_is_dead(cgrp)) {
2bd59d48 4385 rcu_read_unlock();
bad34660 4386 mutex_unlock(&cgroup_mutex);
2bd59d48
TH
4387 return -ENOENT;
4388 }
bad34660 4389 rcu_read_unlock();
846c7bb0 4390
9d800df1 4391 css_task_iter_start(&cgrp->self, &it);
72ec7029 4392 while ((tsk = css_task_iter_next(&it))) {
846c7bb0
BS
4393 switch (tsk->state) {
4394 case TASK_RUNNING:
4395 stats->nr_running++;
4396 break;
4397 case TASK_INTERRUPTIBLE:
4398 stats->nr_sleeping++;
4399 break;
4400 case TASK_UNINTERRUPTIBLE:
4401 stats->nr_uninterruptible++;
4402 break;
4403 case TASK_STOPPED:
4404 stats->nr_stopped++;
4405 break;
4406 default:
4407 if (delayacct_is_task_waiting_on_io(tsk))
4408 stats->nr_io_wait++;
4409 break;
4410 }
4411 }
72ec7029 4412 css_task_iter_end(&it);
846c7bb0 4413
bad34660 4414 mutex_unlock(&cgroup_mutex);
2bd59d48 4415 return 0;
846c7bb0
BS
4416}
4417
8f3ff208 4418
bbcb81d0 4419/*
102a775e 4420 * seq_file methods for the tasks/procs files. The seq_file position is the
cc31edce 4421 * next pid to display; the seq_file iterator is a pointer to the pid
102a775e 4422 * in the cgroup->l->list array.
bbcb81d0 4423 */
cc31edce 4424
102a775e 4425static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos)
bbcb81d0 4426{
cc31edce
PM
4427 /*
4428 * Initially we receive a position value that corresponds to
4429 * one more than the last pid shown (or 0 on the first call or
4430 * after a seek to the start). Use a binary-search to find the
4431 * next pid to display, if any
4432 */
2bd59d48 4433 struct kernfs_open_file *of = s->private;
7da11279 4434 struct cgroup *cgrp = seq_css(s)->cgroup;
4bac00d1 4435 struct cgroup_pidlist *l;
7da11279 4436 enum cgroup_filetype type = seq_cft(s)->private;
cc31edce 4437 int index = 0, pid = *pos;
4bac00d1
TH
4438 int *iter, ret;
4439
4440 mutex_lock(&cgrp->pidlist_mutex);
4441
4442 /*
5d22444f 4443 * !NULL @of->priv indicates that this isn't the first start()
4bac00d1 4444 * after open. If the matching pidlist is around, we can use that.
5d22444f 4445 * Look for it. Note that @of->priv can't be used directly. It
4bac00d1
TH
4446 * could already have been destroyed.
4447 */
5d22444f
TH
4448 if (of->priv)
4449 of->priv = cgroup_pidlist_find(cgrp, type);
4bac00d1
TH
4450
4451 /*
4452 * Either this is the first start() after open or the matching
4453 * pidlist has been destroyed inbetween. Create a new one.
4454 */
5d22444f
TH
4455 if (!of->priv) {
4456 ret = pidlist_array_load(cgrp, type,
4457 (struct cgroup_pidlist **)&of->priv);
4bac00d1
TH
4458 if (ret)
4459 return ERR_PTR(ret);
4460 }
5d22444f 4461 l = of->priv;
cc31edce 4462
cc31edce 4463 if (pid) {
102a775e 4464 int end = l->length;
20777766 4465
cc31edce
PM
4466 while (index < end) {
4467 int mid = (index + end) / 2;
afb2bc14 4468 if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) {
cc31edce
PM
4469 index = mid;
4470 break;
afb2bc14 4471 } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid)
cc31edce
PM
4472 index = mid + 1;
4473 else
4474 end = mid;
4475 }
4476 }
4477 /* If we're off the end of the array, we're done */
102a775e 4478 if (index >= l->length)
cc31edce
PM
4479 return NULL;
4480 /* Update the abstract position to be the actual pid that we found */
102a775e 4481 iter = l->list + index;
afb2bc14 4482 *pos = cgroup_pid_fry(cgrp, *iter);
cc31edce
PM
4483 return iter;
4484}
4485
102a775e 4486static void cgroup_pidlist_stop(struct seq_file *s, void *v)
cc31edce 4487{
2bd59d48 4488 struct kernfs_open_file *of = s->private;
5d22444f 4489 struct cgroup_pidlist *l = of->priv;
62236858 4490
5d22444f
TH
4491 if (l)
4492 mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork,
04502365 4493 CGROUP_PIDLIST_DESTROY_DELAY);
7da11279 4494 mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex);
cc31edce
PM
4495}
4496
102a775e 4497static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos)
cc31edce 4498{
2bd59d48 4499 struct kernfs_open_file *of = s->private;
5d22444f 4500 struct cgroup_pidlist *l = of->priv;
102a775e
BB
4501 pid_t *p = v;
4502 pid_t *end = l->list + l->length;
cc31edce
PM
4503 /*
4504 * Advance to the next pid in the array. If this goes off the
4505 * end, we're done
4506 */
4507 p++;
4508 if (p >= end) {
4509 return NULL;
4510 } else {
7da11279 4511 *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p);
cc31edce
PM
4512 return p;
4513 }
4514}
4515
102a775e 4516static int cgroup_pidlist_show(struct seq_file *s, void *v)
cc31edce 4517{
94ff212d
JP
4518 seq_printf(s, "%d\n", *(int *)v);
4519
4520 return 0;
cc31edce 4521}
bbcb81d0 4522
182446d0
TH
4523static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
4524 struct cftype *cft)
81a6a5cd 4525{
182446d0 4526 return notify_on_release(css->cgroup);
81a6a5cd
PM
4527}
4528
182446d0
TH
4529static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
4530 struct cftype *cft, u64 val)
6379c106 4531{
6379c106 4532 if (val)
182446d0 4533 set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
6379c106 4534 else
182446d0 4535 clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
6379c106
PM
4536 return 0;
4537}
4538
182446d0
TH
4539static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
4540 struct cftype *cft)
97978e6d 4541{
182446d0 4542 return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
97978e6d
DL
4543}
4544
182446d0
TH
4545static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
4546 struct cftype *cft, u64 val)
97978e6d
DL
4547{
4548 if (val)
182446d0 4549 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
97978e6d 4550 else
182446d0 4551 clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
97978e6d
DL
4552 return 0;
4553}
4554
a14c6874
TH
4555/* cgroup core interface files for the default hierarchy */
4556static struct cftype cgroup_dfl_base_files[] = {
81a6a5cd 4557 {
d5c56ced 4558 .name = "cgroup.procs",
6f60eade 4559 .file_offset = offsetof(struct cgroup, procs_file),
6612f05b
TH
4560 .seq_start = cgroup_pidlist_start,
4561 .seq_next = cgroup_pidlist_next,
4562 .seq_stop = cgroup_pidlist_stop,
4563 .seq_show = cgroup_pidlist_show,
5d22444f 4564 .private = CGROUP_FILE_PROCS,
acbef755 4565 .write = cgroup_procs_write,
102a775e 4566 },
f8f22e53
TH
4567 {
4568 .name = "cgroup.controllers",
a14c6874 4569 .flags = CFTYPE_ONLY_ON_ROOT,
f8f22e53
TH
4570 .seq_show = cgroup_root_controllers_show,
4571 },
4572 {
4573 .name = "cgroup.controllers",
a14c6874 4574 .flags = CFTYPE_NOT_ON_ROOT,
f8f22e53
TH
4575 .seq_show = cgroup_controllers_show,
4576 },
4577 {
4578 .name = "cgroup.subtree_control",
f8f22e53 4579 .seq_show = cgroup_subtree_control_show,
451af504 4580 .write = cgroup_subtree_control_write,
f8f22e53 4581 },
842b597e 4582 {
4a07c222 4583 .name = "cgroup.events",
a14c6874 4584 .flags = CFTYPE_NOT_ON_ROOT,
6f60eade 4585 .file_offset = offsetof(struct cgroup, events_file),
4a07c222 4586 .seq_show = cgroup_events_show,
842b597e 4587 },
a14c6874
TH
4588 { } /* terminate */
4589};
d5c56ced 4590
a14c6874
TH
4591/* cgroup core interface files for the legacy hierarchies */
4592static struct cftype cgroup_legacy_base_files[] = {
4593 {
4594 .name = "cgroup.procs",
4595 .seq_start = cgroup_pidlist_start,
4596 .seq_next = cgroup_pidlist_next,
4597 .seq_stop = cgroup_pidlist_stop,
4598 .seq_show = cgroup_pidlist_show,
4599 .private = CGROUP_FILE_PROCS,
4600 .write = cgroup_procs_write,
a14c6874
TH
4601 },
4602 {
4603 .name = "cgroup.clone_children",
4604 .read_u64 = cgroup_clone_children_read,
4605 .write_u64 = cgroup_clone_children_write,
4606 },
4607 {
4608 .name = "cgroup.sane_behavior",
4609 .flags = CFTYPE_ONLY_ON_ROOT,
4610 .seq_show = cgroup_sane_behavior_show,
4611 },
d5c56ced
TH
4612 {
4613 .name = "tasks",
6612f05b
TH
4614 .seq_start = cgroup_pidlist_start,
4615 .seq_next = cgroup_pidlist_next,
4616 .seq_stop = cgroup_pidlist_stop,
4617 .seq_show = cgroup_pidlist_show,
5d22444f 4618 .private = CGROUP_FILE_TASKS,
acbef755 4619 .write = cgroup_tasks_write,
d5c56ced
TH
4620 },
4621 {
4622 .name = "notify_on_release",
d5c56ced
TH
4623 .read_u64 = cgroup_read_notify_on_release,
4624 .write_u64 = cgroup_write_notify_on_release,
4625 },
6e6ff25b
TH
4626 {
4627 .name = "release_agent",
a14c6874 4628 .flags = CFTYPE_ONLY_ON_ROOT,
2da8ca82 4629 .seq_show = cgroup_release_agent_show,
451af504 4630 .write = cgroup_release_agent_write,
5f469907 4631 .max_write_len = PATH_MAX - 1,
6e6ff25b 4632 },
db0416b6 4633 { } /* terminate */
bbcb81d0
PM
4634};
4635
0c21ead1
TH
4636/*
4637 * css destruction is four-stage process.
4638 *
4639 * 1. Destruction starts. Killing of the percpu_ref is initiated.
4640 * Implemented in kill_css().
4641 *
4642 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
ec903c0c
TH
4643 * and thus css_tryget_online() is guaranteed to fail, the css can be
4644 * offlined by invoking offline_css(). After offlining, the base ref is
4645 * put. Implemented in css_killed_work_fn().
0c21ead1
TH
4646 *
4647 * 3. When the percpu_ref reaches zero, the only possible remaining
4648 * accessors are inside RCU read sections. css_release() schedules the
4649 * RCU callback.
4650 *
4651 * 4. After the grace period, the css can be freed. Implemented in
4652 * css_free_work_fn().
4653 *
4654 * It is actually hairier because both step 2 and 4 require process context
4655 * and thus involve punting to css->destroy_work adding two additional
4656 * steps to the already complex sequence.
4657 */
35ef10da 4658static void css_free_work_fn(struct work_struct *work)
48ddbe19
TH
4659{
4660 struct cgroup_subsys_state *css =
35ef10da 4661 container_of(work, struct cgroup_subsys_state, destroy_work);
01e58659 4662 struct cgroup_subsys *ss = css->ss;
0c21ead1 4663 struct cgroup *cgrp = css->cgroup;
48ddbe19 4664
9a1049da
TH
4665 percpu_ref_exit(&css->refcnt);
4666
01e58659 4667 if (ss) {
9d755d33 4668 /* css free path */
8bb5ef79 4669 struct cgroup_subsys_state *parent = css->parent;
01e58659
VD
4670 int id = css->id;
4671
01e58659
VD
4672 ss->css_free(css);
4673 cgroup_idr_remove(&ss->css_idr, id);
9d755d33 4674 cgroup_put(cgrp);
8bb5ef79
TH
4675
4676 if (parent)
4677 css_put(parent);
9d755d33
TH
4678 } else {
4679 /* cgroup free path */
4680 atomic_dec(&cgrp->root->nr_cgrps);
4681 cgroup_pidlist_destroy_all(cgrp);
971ff493 4682 cancel_work_sync(&cgrp->release_agent_work);
9d755d33 4683
d51f39b0 4684 if (cgroup_parent(cgrp)) {
9d755d33
TH
4685 /*
4686 * We get a ref to the parent, and put the ref when
4687 * this cgroup is being freed, so it's guaranteed
4688 * that the parent won't be destroyed before its
4689 * children.
4690 */
d51f39b0 4691 cgroup_put(cgroup_parent(cgrp));
9d755d33
TH
4692 kernfs_put(cgrp->kn);
4693 kfree(cgrp);
4694 } else {
4695 /*
4696 * This is root cgroup's refcnt reaching zero,
4697 * which indicates that the root should be
4698 * released.
4699 */
4700 cgroup_destroy_root(cgrp->root);
4701 }
4702 }
48ddbe19
TH
4703}
4704
0c21ead1 4705static void css_free_rcu_fn(struct rcu_head *rcu_head)
d3daf28d
TH
4706{
4707 struct cgroup_subsys_state *css =
0c21ead1 4708 container_of(rcu_head, struct cgroup_subsys_state, rcu_head);
d3daf28d 4709
35ef10da 4710 INIT_WORK(&css->destroy_work, css_free_work_fn);
e5fca243 4711 queue_work(cgroup_destroy_wq, &css->destroy_work);
48ddbe19
TH
4712}
4713
25e15d83 4714static void css_release_work_fn(struct work_struct *work)
d3daf28d
TH
4715{
4716 struct cgroup_subsys_state *css =
25e15d83 4717 container_of(work, struct cgroup_subsys_state, destroy_work);
15a4c835 4718 struct cgroup_subsys *ss = css->ss;
9d755d33 4719 struct cgroup *cgrp = css->cgroup;
15a4c835 4720
1fed1b2e
TH
4721 mutex_lock(&cgroup_mutex);
4722
de3f0341 4723 css->flags |= CSS_RELEASED;
1fed1b2e
TH
4724 list_del_rcu(&css->sibling);
4725
9d755d33
TH
4726 if (ss) {
4727 /* css release path */
01e58659 4728 cgroup_idr_replace(&ss->css_idr, NULL, css->id);
7d172cc8
TH
4729 if (ss->css_released)
4730 ss->css_released(css);
9d755d33
TH
4731 } else {
4732 /* cgroup release path */
9d755d33
TH
4733 cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
4734 cgrp->id = -1;
a4189487
LZ
4735
4736 /*
4737 * There are two control paths which try to determine
4738 * cgroup from dentry without going through kernfs -
4739 * cgroupstats_build() and css_tryget_online_from_dir().
4740 * Those are supported by RCU protecting clearing of
4741 * cgrp->kn->priv backpointer.
4742 */
6cd0f5bb
TH
4743 if (cgrp->kn)
4744 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv,
4745 NULL);
9d755d33 4746 }
d3daf28d 4747
1fed1b2e
TH
4748 mutex_unlock(&cgroup_mutex);
4749
0c21ead1 4750 call_rcu(&css->rcu_head, css_free_rcu_fn);
d3daf28d
TH
4751}
4752
d3daf28d
TH
4753static void css_release(struct percpu_ref *ref)
4754{
4755 struct cgroup_subsys_state *css =
4756 container_of(ref, struct cgroup_subsys_state, refcnt);
4757
25e15d83
TH
4758 INIT_WORK(&css->destroy_work, css_release_work_fn);
4759 queue_work(cgroup_destroy_wq, &css->destroy_work);
d3daf28d
TH
4760}
4761
ddfcadab
TH
4762static void init_and_link_css(struct cgroup_subsys_state *css,
4763 struct cgroup_subsys *ss, struct cgroup *cgrp)
ddbcc7e8 4764{
0cb51d71
TH
4765 lockdep_assert_held(&cgroup_mutex);
4766
ddfcadab
TH
4767 cgroup_get(cgrp);
4768
d5c419b6 4769 memset(css, 0, sizeof(*css));
bd89aabc 4770 css->cgroup = cgrp;
72c97e54 4771 css->ss = ss;
d5c419b6
TH
4772 INIT_LIST_HEAD(&css->sibling);
4773 INIT_LIST_HEAD(&css->children);
0cb51d71 4774 css->serial_nr = css_serial_nr_next++;
aa226ff4 4775 atomic_set(&css->online_cnt, 0);
0ae78e0b 4776
d51f39b0
TH
4777 if (cgroup_parent(cgrp)) {
4778 css->parent = cgroup_css(cgroup_parent(cgrp), ss);
ddfcadab 4779 css_get(css->parent);
ddfcadab 4780 }
48ddbe19 4781
ca8bdcaf 4782 BUG_ON(cgroup_css(cgrp, ss));
ddbcc7e8
PM
4783}
4784
2a4ac633 4785/* invoke ->css_online() on a new CSS and mark it online if successful */
623f926b 4786static int online_css(struct cgroup_subsys_state *css)
a31f2d3f 4787{
623f926b 4788 struct cgroup_subsys *ss = css->ss;
b1929db4
TH
4789 int ret = 0;
4790
a31f2d3f
TH
4791 lockdep_assert_held(&cgroup_mutex);
4792
92fb9748 4793 if (ss->css_online)
eb95419b 4794 ret = ss->css_online(css);
ae7f164a 4795 if (!ret) {
eb95419b 4796 css->flags |= CSS_ONLINE;
aec25020 4797 rcu_assign_pointer(css->cgroup->subsys[ss->id], css);
aa226ff4
TH
4798
4799 atomic_inc(&css->online_cnt);
4800 if (css->parent)
4801 atomic_inc(&css->parent->online_cnt);
ae7f164a 4802 }
b1929db4 4803 return ret;
a31f2d3f
TH
4804}
4805
2a4ac633 4806/* if the CSS is online, invoke ->css_offline() on it and mark it offline */
623f926b 4807static void offline_css(struct cgroup_subsys_state *css)
a31f2d3f 4808{
623f926b 4809 struct cgroup_subsys *ss = css->ss;
a31f2d3f
TH
4810
4811 lockdep_assert_held(&cgroup_mutex);
4812
4813 if (!(css->flags & CSS_ONLINE))
4814 return;
4815
fa06235b
VD
4816 if (ss->css_reset)
4817 ss->css_reset(css);
4818
d7eeac19 4819 if (ss->css_offline)
eb95419b 4820 ss->css_offline(css);
a31f2d3f 4821
eb95419b 4822 css->flags &= ~CSS_ONLINE;
e3297803 4823 RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL);
f8f22e53
TH
4824
4825 wake_up_all(&css->cgroup->offline_waitq);
a31f2d3f
TH
4826}
4827
c81c925a 4828/**
6cd0f5bb 4829 * css_create - create a cgroup_subsys_state
c81c925a
TH
4830 * @cgrp: the cgroup new css will be associated with
4831 * @ss: the subsys of new css
4832 *
4833 * Create a new css associated with @cgrp - @ss pair. On success, the new
6cd0f5bb
TH
4834 * css is online and installed in @cgrp. This function doesn't create the
4835 * interface files. Returns 0 on success, -errno on failure.
c81c925a 4836 */
6cd0f5bb
TH
4837static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
4838 struct cgroup_subsys *ss)
c81c925a 4839{
d51f39b0 4840 struct cgroup *parent = cgroup_parent(cgrp);
1fed1b2e 4841 struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss);
c81c925a
TH
4842 struct cgroup_subsys_state *css;
4843 int err;
4844
c81c925a
TH
4845 lockdep_assert_held(&cgroup_mutex);
4846
1fed1b2e 4847 css = ss->css_alloc(parent_css);
c81c925a 4848 if (IS_ERR(css))
6cd0f5bb 4849 return css;
c81c925a 4850
ddfcadab 4851 init_and_link_css(css, ss, cgrp);
a2bed820 4852
2aad2a86 4853 err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL);
c81c925a 4854 if (err)
3eb59ec6 4855 goto err_free_css;
c81c925a 4856
cf780b7d 4857 err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL);
15a4c835
TH
4858 if (err < 0)
4859 goto err_free_percpu_ref;
4860 css->id = err;
c81c925a 4861
15a4c835 4862 /* @css is ready to be brought online now, make it visible */
1fed1b2e 4863 list_add_tail_rcu(&css->sibling, &parent_css->children);
15a4c835 4864 cgroup_idr_replace(&ss->css_idr, css, css->id);
c81c925a
TH
4865
4866 err = online_css(css);
4867 if (err)
1fed1b2e 4868 goto err_list_del;
94419627 4869
c81c925a 4870 if (ss->broken_hierarchy && !ss->warned_broken_hierarchy &&
d51f39b0 4871 cgroup_parent(parent)) {
ed3d261b 4872 pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
a2a1f9ea 4873 current->comm, current->pid, ss->name);
c81c925a 4874 if (!strcmp(ss->name, "memory"))
ed3d261b 4875 pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
c81c925a
TH
4876 ss->warned_broken_hierarchy = true;
4877 }
4878
6cd0f5bb 4879 return css;
c81c925a 4880
1fed1b2e
TH
4881err_list_del:
4882 list_del_rcu(&css->sibling);
15a4c835 4883 cgroup_idr_remove(&ss->css_idr, css->id);
3eb59ec6 4884err_free_percpu_ref:
9a1049da 4885 percpu_ref_exit(&css->refcnt);
3eb59ec6 4886err_free_css:
a2bed820 4887 call_rcu(&css->rcu_head, css_free_rcu_fn);
6cd0f5bb 4888 return ERR_PTR(err);
c81c925a
TH
4889}
4890
b3bfd983
TH
4891static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
4892 umode_t mode)
ddbcc7e8 4893{
b11cfb58 4894 struct cgroup *parent, *cgrp, *tcgrp;
a9746d8d 4895 struct cgroup_root *root;
ddbcc7e8 4896 struct cgroup_subsys *ss;
2bd59d48 4897 struct kernfs_node *kn;
b11cfb58 4898 int level, ssid, ret;
ddbcc7e8 4899
71b1fb5c
AC
4900 /* Do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable.
4901 */
4902 if (strchr(name, '\n'))
4903 return -EINVAL;
4904
a9746d8d
TH
4905 parent = cgroup_kn_lock_live(parent_kn);
4906 if (!parent)
4907 return -ENODEV;
4908 root = parent->root;
b11cfb58 4909 level = parent->level + 1;
ddbcc7e8 4910
0a950f65 4911 /* allocate the cgroup and its ID, 0 is reserved for the root */
b11cfb58
TH
4912 cgrp = kzalloc(sizeof(*cgrp) +
4913 sizeof(cgrp->ancestor_ids[0]) * (level + 1), GFP_KERNEL);
ba0f4d76
TH
4914 if (!cgrp) {
4915 ret = -ENOMEM;
4916 goto out_unlock;
0ab02ca8
LZ
4917 }
4918
2aad2a86 4919 ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL);
9d755d33
TH
4920 if (ret)
4921 goto out_free_cgrp;
4922
0ab02ca8
LZ
4923 /*
4924 * Temporarily set the pointer to NULL, so idr_find() won't return
4925 * a half-baked cgroup.
4926 */
cf780b7d 4927 cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL);
0ab02ca8 4928 if (cgrp->id < 0) {
ba0f4d76 4929 ret = -ENOMEM;
9d755d33 4930 goto out_cancel_ref;
976c06bc
TH
4931 }
4932
cc31edce 4933 init_cgroup_housekeeping(cgrp);
ddbcc7e8 4934
9d800df1 4935 cgrp->self.parent = &parent->self;
ba0f4d76 4936 cgrp->root = root;
b11cfb58
TH
4937 cgrp->level = level;
4938
4939 for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp))
4940 cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
ddbcc7e8 4941
b6abdb0e
LZ
4942 if (notify_on_release(parent))
4943 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
4944
2260e7fc
TH
4945 if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags))
4946 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags);
97978e6d 4947
2bd59d48 4948 /* create the directory */
e61734c5 4949 kn = kernfs_create_dir(parent->kn, name, mode, cgrp);
2bd59d48 4950 if (IS_ERR(kn)) {
ba0f4d76
TH
4951 ret = PTR_ERR(kn);
4952 goto out_free_id;
2bd59d48
TH
4953 }
4954 cgrp->kn = kn;
ddbcc7e8 4955
4e139afc 4956 /*
6f30558f
TH
4957 * This extra ref will be put in cgroup_free_fn() and guarantees
4958 * that @cgrp->kn is always accessible.
4e139afc 4959 */
6f30558f 4960 kernfs_get(kn);
ddbcc7e8 4961
0cb51d71 4962 cgrp->self.serial_nr = css_serial_nr_next++;
53fa5261 4963
4e139afc 4964 /* allocation complete, commit to creation */
d5c419b6 4965 list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children);
3c9c825b 4966 atomic_inc(&root->nr_cgrps);
59f5296b 4967 cgroup_get(parent);
415cf07a 4968
0d80255e
TH
4969 /*
4970 * @cgrp is now fully operational. If something fails after this
4971 * point, it'll be released via the normal destruction path.
4972 */
6fa4918d 4973 cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id);
4e96ee8e 4974
ba0f4d76
TH
4975 ret = cgroup_kn_set_ugid(kn);
4976 if (ret)
4977 goto out_destroy;
49957f8e 4978
4df8dc90 4979 ret = css_populate_dir(&cgrp->self, NULL);
ba0f4d76
TH
4980 if (ret)
4981 goto out_destroy;
628f7cd4 4982
9d403e99 4983 /* let's create and online css's */
996cd1fb 4984 do_each_subsys_mask(ss, ssid, parent->subtree_ss_mask) {
6cd0f5bb
TH
4985 struct cgroup_subsys_state *css;
4986
4987 css = css_create(cgrp, ss);
4988 if (IS_ERR(css)) {
4989 ret = PTR_ERR(css);
996cd1fb 4990 goto out_destroy;
6cd0f5bb
TH
4991 }
4992
4993 if (parent->subtree_control & (1 << ssid)) {
4994 ret = css_populate_dir(css, NULL);
4995 if (ret)
4996 goto out_destroy;
4997 }
996cd1fb 4998 } while_each_subsys_mask();
ddbcc7e8 4999
bd53d617
TH
5000 /*
5001 * On the default hierarchy, a child doesn't automatically inherit
667c2491 5002 * subtree_control from the parent. Each is configured manually.
bd53d617 5003 */
667c2491
TH
5004 if (!cgroup_on_dfl(cgrp)) {
5005 cgrp->subtree_control = parent->subtree_control;
8699b776 5006 cgroup_refresh_subtree_ss_mask(cgrp);
667c2491 5007 }
2bd59d48 5008
2bd59d48 5009 kernfs_activate(kn);
ddbcc7e8 5010
ba0f4d76
TH
5011 ret = 0;
5012 goto out_unlock;
ddbcc7e8 5013
ba0f4d76 5014out_free_id:
6fa4918d 5015 cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
9d755d33 5016out_cancel_ref:
9a1049da 5017 percpu_ref_exit(&cgrp->self.refcnt);
ba0f4d76 5018out_free_cgrp:
bd89aabc 5019 kfree(cgrp);
ba0f4d76 5020out_unlock:
a9746d8d 5021 cgroup_kn_unlock(parent_kn);
ba0f4d76 5022 return ret;
4b8b47eb 5023
ba0f4d76 5024out_destroy:
4b8b47eb 5025 cgroup_destroy_locked(cgrp);
ba0f4d76 5026 goto out_unlock;
ddbcc7e8
PM
5027}
5028
223dbc38
TH
5029/*
5030 * This is called when the refcnt of a css is confirmed to be killed.
249f3468
TH
5031 * css_tryget_online() is now guaranteed to fail. Tell the subsystem to
5032 * initate destruction and put the css ref from kill_css().
223dbc38
TH
5033 */
5034static void css_killed_work_fn(struct work_struct *work)
d3daf28d 5035{
223dbc38
TH
5036 struct cgroup_subsys_state *css =
5037 container_of(work, struct cgroup_subsys_state, destroy_work);
d3daf28d 5038
f20104de 5039 mutex_lock(&cgroup_mutex);
09a503ea 5040
aa226ff4
TH
5041 do {
5042 offline_css(css);
5043 css_put(css);
5044 /* @css can't go away while we're holding cgroup_mutex */
5045 css = css->parent;
5046 } while (css && atomic_dec_and_test(&css->online_cnt));
5047
5048 mutex_unlock(&cgroup_mutex);
d3daf28d
TH
5049}
5050
223dbc38
TH
5051/* css kill confirmation processing requires process context, bounce */
5052static void css_killed_ref_fn(struct percpu_ref *ref)
d3daf28d
TH
5053{
5054 struct cgroup_subsys_state *css =
5055 container_of(ref, struct cgroup_subsys_state, refcnt);
5056
aa226ff4
TH
5057 if (atomic_dec_and_test(&css->online_cnt)) {
5058 INIT_WORK(&css->destroy_work, css_killed_work_fn);
5059 queue_work(cgroup_destroy_wq, &css->destroy_work);
5060 }
d3daf28d
TH
5061}
5062
f392e51c
TH
5063/**
5064 * kill_css - destroy a css
5065 * @css: css to destroy
5066 *
5067 * This function initiates destruction of @css by removing cgroup interface
5068 * files and putting its base reference. ->css_offline() will be invoked
ec903c0c
TH
5069 * asynchronously once css_tryget_online() is guaranteed to fail and when
5070 * the reference count reaches zero, @css will be released.
f392e51c
TH
5071 */
5072static void kill_css(struct cgroup_subsys_state *css)
edae0c33 5073{
01f6474c 5074 lockdep_assert_held(&cgroup_mutex);
94419627 5075
2bd59d48
TH
5076 /*
5077 * This must happen before css is disassociated with its cgroup.
5078 * See seq_css() for details.
5079 */
4df8dc90 5080 css_clear_dir(css, NULL);
3c14f8b4 5081
edae0c33
TH
5082 /*
5083 * Killing would put the base ref, but we need to keep it alive
5084 * until after ->css_offline().
5085 */
5086 css_get(css);
5087
5088 /*
5089 * cgroup core guarantees that, by the time ->css_offline() is
5090 * invoked, no new css reference will be given out via
ec903c0c 5091 * css_tryget_online(). We can't simply call percpu_ref_kill() and
edae0c33
TH
5092 * proceed to offlining css's because percpu_ref_kill() doesn't
5093 * guarantee that the ref is seen as killed on all CPUs on return.
5094 *
5095 * Use percpu_ref_kill_and_confirm() to get notifications as each
5096 * css is confirmed to be seen as killed on all CPUs.
5097 */
5098 percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn);
d3daf28d
TH
5099}
5100
5101/**
5102 * cgroup_destroy_locked - the first stage of cgroup destruction
5103 * @cgrp: cgroup to be destroyed
5104 *
5105 * css's make use of percpu refcnts whose killing latency shouldn't be
5106 * exposed to userland and are RCU protected. Also, cgroup core needs to
ec903c0c
TH
5107 * guarantee that css_tryget_online() won't succeed by the time
5108 * ->css_offline() is invoked. To satisfy all the requirements,
5109 * destruction is implemented in the following two steps.
d3daf28d
TH
5110 *
5111 * s1. Verify @cgrp can be destroyed and mark it dying. Remove all
5112 * userland visible parts and start killing the percpu refcnts of
5113 * css's. Set up so that the next stage will be kicked off once all
5114 * the percpu refcnts are confirmed to be killed.
5115 *
5116 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5117 * rest of destruction. Once all cgroup references are gone, the
5118 * cgroup is RCU-freed.
5119 *
5120 * This function implements s1. After this step, @cgrp is gone as far as
5121 * the userland is concerned and a new cgroup with the same name may be
5122 * created. As cgroup doesn't care about the names internally, this
5123 * doesn't cause any problem.
5124 */
42809dd4
TH
5125static int cgroup_destroy_locked(struct cgroup *cgrp)
5126 __releases(&cgroup_mutex) __acquires(&cgroup_mutex)
ddbcc7e8 5127{
2bd59d48 5128 struct cgroup_subsys_state *css;
1c6727af 5129 int ssid;
ddbcc7e8 5130
42809dd4
TH
5131 lockdep_assert_held(&cgroup_mutex);
5132
91486f61
TH
5133 /*
5134 * Only migration can raise populated from zero and we're already
5135 * holding cgroup_mutex.
5136 */
5137 if (cgroup_is_populated(cgrp))
ddbcc7e8 5138 return -EBUSY;
a043e3b2 5139
bb78a92f 5140 /*
d5c419b6
TH
5141 * Make sure there's no live children. We can't test emptiness of
5142 * ->self.children as dead children linger on it while being
5143 * drained; otherwise, "rmdir parent/child parent" may fail.
bb78a92f 5144 */
f3d46500 5145 if (css_has_online_children(&cgrp->self))
bb78a92f
HD
5146 return -EBUSY;
5147
455050d2
TH
5148 /*
5149 * Mark @cgrp dead. This prevents further task migration and child
de3f0341 5150 * creation by disabling cgroup_lock_live_group().
455050d2 5151 */
184faf32 5152 cgrp->self.flags &= ~CSS_ONLINE;
ddbcc7e8 5153
249f3468 5154 /* initiate massacre of all css's */
1c6727af
TH
5155 for_each_css(css, ssid, cgrp)
5156 kill_css(css);
455050d2 5157
455050d2 5158 /*
01f6474c
TH
5159 * Remove @cgrp directory along with the base files. @cgrp has an
5160 * extra ref on its kn.
f20104de 5161 */
01f6474c 5162 kernfs_remove(cgrp->kn);
f20104de 5163
d51f39b0 5164 check_for_release(cgroup_parent(cgrp));
2bd59d48 5165
249f3468 5166 /* put the base reference */
9d755d33 5167 percpu_ref_kill(&cgrp->self.refcnt);
455050d2 5168
ea15f8cc
TH
5169 return 0;
5170};
5171
2bd59d48 5172static int cgroup_rmdir(struct kernfs_node *kn)
42809dd4 5173{
a9746d8d 5174 struct cgroup *cgrp;
2bd59d48 5175 int ret = 0;
42809dd4 5176
a9746d8d
TH
5177 cgrp = cgroup_kn_lock_live(kn);
5178 if (!cgrp)
5179 return 0;
42809dd4 5180
a9746d8d 5181 ret = cgroup_destroy_locked(cgrp);
2bb566cb 5182
a9746d8d 5183 cgroup_kn_unlock(kn);
42809dd4 5184 return ret;
8e3f6541
TH
5185}
5186
2bd59d48
TH
5187static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
5188 .remount_fs = cgroup_remount,
5189 .show_options = cgroup_show_options,
5190 .mkdir = cgroup_mkdir,
5191 .rmdir = cgroup_rmdir,
5192 .rename = cgroup_rename,
5193};
5194
15a4c835 5195static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early)
ddbcc7e8 5196{
ddbcc7e8 5197 struct cgroup_subsys_state *css;
cfe36bde 5198
a5ae9899 5199 pr_debug("Initializing cgroup subsys %s\n", ss->name);
ddbcc7e8 5200
648bb56d
TH
5201 mutex_lock(&cgroup_mutex);
5202
15a4c835 5203 idr_init(&ss->css_idr);
0adb0704 5204 INIT_LIST_HEAD(&ss->cfts);
8e3f6541 5205
3dd06ffa
TH
5206 /* Create the root cgroup state for this subsystem */
5207 ss->root = &cgrp_dfl_root;
5208 css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss));
ddbcc7e8
PM
5209 /* We don't handle early failures gracefully */
5210 BUG_ON(IS_ERR(css));
ddfcadab 5211 init_and_link_css(css, ss, &cgrp_dfl_root.cgrp);
3b514d24
TH
5212
5213 /*
5214 * Root csses are never destroyed and we can't initialize
5215 * percpu_ref during early init. Disable refcnting.
5216 */
5217 css->flags |= CSS_NO_REF;
5218
15a4c835 5219 if (early) {
9395a450 5220 /* allocation can't be done safely during early init */
15a4c835
TH
5221 css->id = 1;
5222 } else {
5223 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL);
5224 BUG_ON(css->id < 0);
5225 }
ddbcc7e8 5226
e8d55fde 5227 /* Update the init_css_set to contain a subsys
817929ec 5228 * pointer to this state - since the subsystem is
e8d55fde 5229 * newly registered, all tasks and hence the
3dd06ffa 5230 * init_css_set is in the subsystem's root cgroup. */
aec25020 5231 init_css_set.subsys[ss->id] = css;
ddbcc7e8 5232
cb4a3167
AS
5233 have_fork_callback |= (bool)ss->fork << ss->id;
5234 have_exit_callback |= (bool)ss->exit << ss->id;
afcf6c8b 5235 have_free_callback |= (bool)ss->free << ss->id;
7e47682e 5236 have_canfork_callback |= (bool)ss->can_fork << ss->id;
ddbcc7e8 5237
e8d55fde
LZ
5238 /* At system boot, before all subsystems have been
5239 * registered, no tasks have been forked, so we don't
5240 * need to invoke fork callbacks here. */
5241 BUG_ON(!list_empty(&init_task.tasks));
5242
ae7f164a 5243 BUG_ON(online_css(css));
a8638030 5244
cf5d5941
BB
5245 mutex_unlock(&cgroup_mutex);
5246}
cf5d5941 5247
ddbcc7e8 5248/**
a043e3b2
LZ
5249 * cgroup_init_early - cgroup initialization at system boot
5250 *
5251 * Initialize cgroups at system boot, and initialize any
5252 * subsystems that request early init.
ddbcc7e8
PM
5253 */
5254int __init cgroup_init_early(void)
5255{
7b9a6ba5 5256 static struct cgroup_sb_opts __initdata opts;
30159ec7 5257 struct cgroup_subsys *ss;
ddbcc7e8 5258 int i;
30159ec7 5259
3dd06ffa 5260 init_cgroup_root(&cgrp_dfl_root, &opts);
3b514d24
TH
5261 cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF;
5262
a4ea1cc9 5263 RCU_INIT_POINTER(init_task.cgroups, &init_css_set);
817929ec 5264
3ed80a62 5265 for_each_subsys(ss, i) {
aec25020 5266 WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id,
63253ad8 5267 "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n",
073219e9 5268 i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free,
aec25020 5269 ss->id, ss->name);
073219e9
TH
5270 WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN,
5271 "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]);
5272
aec25020 5273 ss->id = i;
073219e9 5274 ss->name = cgroup_subsys_name[i];
3e1d2eed
TH
5275 if (!ss->legacy_name)
5276 ss->legacy_name = cgroup_subsys_name[i];
ddbcc7e8
PM
5277
5278 if (ss->early_init)
15a4c835 5279 cgroup_init_subsys(ss, true);
ddbcc7e8
PM
5280 }
5281 return 0;
5282}
5283
6e5c8307 5284static u16 cgroup_disable_mask __initdata;
a3e72739 5285
ddbcc7e8 5286/**
a043e3b2
LZ
5287 * cgroup_init - cgroup initialization
5288 *
5289 * Register cgroup filesystem and /proc file, and initialize
5290 * any subsystems that didn't request early init.
ddbcc7e8
PM
5291 */
5292int __init cgroup_init(void)
5293{
30159ec7 5294 struct cgroup_subsys *ss;
035f4f51 5295 int ssid;
ddbcc7e8 5296
6e5c8307 5297 BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16);
1ed13287 5298 BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem));
a14c6874
TH
5299 BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files));
5300 BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files));
ddbcc7e8 5301
54e7b4eb 5302 mutex_lock(&cgroup_mutex);
54e7b4eb 5303
2378d8b8
TH
5304 /*
5305 * Add init_css_set to the hash table so that dfl_root can link to
5306 * it during init.
5307 */
5308 hash_add(css_set_table, &init_css_set.hlist,
5309 css_set_hash(init_css_set.subsys));
82fe9b0d 5310
3dd06ffa 5311 BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0));
4e96ee8e 5312
54e7b4eb
TH
5313 mutex_unlock(&cgroup_mutex);
5314
172a2c06 5315 for_each_subsys(ss, ssid) {
15a4c835
TH
5316 if (ss->early_init) {
5317 struct cgroup_subsys_state *css =
5318 init_css_set.subsys[ss->id];
5319
5320 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2,
5321 GFP_KERNEL);
5322 BUG_ON(css->id < 0);
5323 } else {
5324 cgroup_init_subsys(ss, false);
5325 }
172a2c06 5326
2d8f243a
TH
5327 list_add_tail(&init_css_set.e_cset_node[ssid],
5328 &cgrp_dfl_root.cgrp.e_csets[ssid]);
172a2c06
TH
5329
5330 /*
c731ae1d
LZ
5331 * Setting dfl_root subsys_mask needs to consider the
5332 * disabled flag and cftype registration needs kmalloc,
5333 * both of which aren't available during early_init.
172a2c06 5334 */
a3e72739
TH
5335 if (cgroup_disable_mask & (1 << ssid)) {
5336 static_branch_disable(cgroup_subsys_enabled_key[ssid]);
5337 printk(KERN_INFO "Disabling %s control group subsystem\n",
5338 ss->name);
a8ddc821 5339 continue;
a3e72739 5340 }
a8ddc821 5341
223ffb29
JW
5342 if (cgroup_ssid_no_v1(ssid))
5343 printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n",
5344 ss->name);
5345
a8ddc821
TH
5346 cgrp_dfl_root.subsys_mask |= 1 << ss->id;
5347
5de4fa13 5348 if (!ss->dfl_cftypes)
a7165264 5349 cgrp_dfl_inhibit_ss_mask |= 1 << ss->id;
5de4fa13 5350
a8ddc821
TH
5351 if (ss->dfl_cftypes == ss->legacy_cftypes) {
5352 WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes));
5353 } else {
5354 WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes));
5355 WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes));
c731ae1d 5356 }
295458e6
VD
5357
5358 if (ss->bind)
5359 ss->bind(init_css_set.subsys[ssid]);
676db4af
GK
5360 }
5361
2378d8b8
TH
5362 /* init_css_set.subsys[] has been updated, re-hash */
5363 hash_del(&init_css_set.hlist);
5364 hash_add(css_set_table, &init_css_set.hlist,
5365 css_set_hash(init_css_set.subsys));
5366
035f4f51
TH
5367 WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup"));
5368 WARN_ON(register_filesystem(&cgroup_fs_type));
67e9c74b 5369 WARN_ON(register_filesystem(&cgroup2_fs_type));
035f4f51 5370 WARN_ON(!proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations));
ddbcc7e8 5371
2bd59d48 5372 return 0;
ddbcc7e8 5373}
b4f48b63 5374
e5fca243
TH
5375static int __init cgroup_wq_init(void)
5376{
5377 /*
5378 * There isn't much point in executing destruction path in
5379 * parallel. Good chunk is serialized with cgroup_mutex anyway.
1a11533f 5380 * Use 1 for @max_active.
e5fca243
TH
5381 *
5382 * We would prefer to do this in cgroup_init() above, but that
5383 * is called before init_workqueues(): so leave this until after.
5384 */
1a11533f 5385 cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
e5fca243 5386 BUG_ON(!cgroup_destroy_wq);
b1a21367
TH
5387
5388 /*
5389 * Used to destroy pidlists and separate to serve as flush domain.
5390 * Cap @max_active to 1 too.
5391 */
5392 cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy",
5393 0, 1);
5394 BUG_ON(!cgroup_pidlist_destroy_wq);
5395
e5fca243
TH
5396 return 0;
5397}
5398core_initcall(cgroup_wq_init);
5399
a424316c
PM
5400/*
5401 * proc_cgroup_show()
5402 * - Print task's cgroup paths into seq_file, one line for each hierarchy
5403 * - Used for /proc/<pid>/cgroup.
a424316c 5404 */
006f4ac4
ZL
5405int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
5406 struct pid *pid, struct task_struct *tsk)
a424316c 5407{
e61734c5 5408 char *buf, *path;
a424316c 5409 int retval;
3dd06ffa 5410 struct cgroup_root *root;
a424316c
PM
5411
5412 retval = -ENOMEM;
e61734c5 5413 buf = kmalloc(PATH_MAX, GFP_KERNEL);
a424316c
PM
5414 if (!buf)
5415 goto out;
5416
a424316c 5417 mutex_lock(&cgroup_mutex);
f0d9a5f1 5418 spin_lock_bh(&css_set_lock);
a424316c 5419
985ed670 5420 for_each_root(root) {
a424316c 5421 struct cgroup_subsys *ss;
bd89aabc 5422 struct cgroup *cgrp;
b85d2040 5423 int ssid, count = 0;
a424316c 5424
a7165264 5425 if (root == &cgrp_dfl_root && !cgrp_dfl_visible)
985ed670
TH
5426 continue;
5427
2c6ab6d2 5428 seq_printf(m, "%d:", root->hierarchy_id);
d98817d4
TH
5429 if (root != &cgrp_dfl_root)
5430 for_each_subsys(ss, ssid)
5431 if (root->subsys_mask & (1 << ssid))
5432 seq_printf(m, "%s%s", count++ ? "," : "",
3e1d2eed 5433 ss->legacy_name);
c6d57f33
PM
5434 if (strlen(root->name))
5435 seq_printf(m, "%sname=%s", count ? "," : "",
5436 root->name);
a424316c 5437 seq_putc(m, ':');
2e91fa7f 5438
7717f7ba 5439 cgrp = task_cgroup_from_root(tsk, root);
2e91fa7f
TH
5440
5441 /*
5442 * On traditional hierarchies, all zombie tasks show up as
5443 * belonging to the root cgroup. On the default hierarchy,
5444 * while a zombie doesn't show up in "cgroup.procs" and
5445 * thus can't be migrated, its /proc/PID/cgroup keeps
5446 * reporting the cgroup it belonged to before exiting. If
5447 * the cgroup is removed before the zombie is reaped,
5448 * " (deleted)" is appended to the cgroup path.
5449 */
5450 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) {
5451 path = cgroup_path(cgrp, buf, PATH_MAX);
5452 if (!path) {
5453 retval = -ENAMETOOLONG;
5454 goto out_unlock;
5455 }
5456 } else {
5457 path = "/";
e61734c5 5458 }
2e91fa7f 5459
e61734c5 5460 seq_puts(m, path);
2e91fa7f
TH
5461
5462 if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp))
5463 seq_puts(m, " (deleted)\n");
5464 else
5465 seq_putc(m, '\n');
a424316c
PM
5466 }
5467
006f4ac4 5468 retval = 0;
a424316c 5469out_unlock:
f0d9a5f1 5470 spin_unlock_bh(&css_set_lock);
a424316c 5471 mutex_unlock(&cgroup_mutex);
a424316c
PM
5472 kfree(buf);
5473out:
5474 return retval;
5475}
5476
a424316c
PM
5477/* Display information about each subsystem and each hierarchy */
5478static int proc_cgroupstats_show(struct seq_file *m, void *v)
5479{
30159ec7 5480 struct cgroup_subsys *ss;
a424316c 5481 int i;
a424316c 5482
8bab8dde 5483 seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n");
aae8aab4
BB
5484 /*
5485 * ideally we don't want subsystems moving around while we do this.
5486 * cgroup_mutex is also necessary to guarantee an atomic snapshot of
5487 * subsys/hierarchy state.
5488 */
a424316c 5489 mutex_lock(&cgroup_mutex);
30159ec7
TH
5490
5491 for_each_subsys(ss, i)
2c6ab6d2 5492 seq_printf(m, "%s\t%d\t%d\t%d\n",
3e1d2eed 5493 ss->legacy_name, ss->root->hierarchy_id,
fc5ed1e9
TH
5494 atomic_read(&ss->root->nr_cgrps),
5495 cgroup_ssid_enabled(i));
30159ec7 5496
a424316c
PM
5497 mutex_unlock(&cgroup_mutex);
5498 return 0;
5499}
5500
5501static int cgroupstats_open(struct inode *inode, struct file *file)
5502{
9dce07f1 5503 return single_open(file, proc_cgroupstats_show, NULL);
a424316c
PM
5504}
5505
828c0950 5506static const struct file_operations proc_cgroupstats_operations = {
a424316c
PM
5507 .open = cgroupstats_open,
5508 .read = seq_read,
5509 .llseek = seq_lseek,
5510 .release = single_release,
5511};
5512
b4f48b63 5513/**
eaf797ab 5514 * cgroup_fork - initialize cgroup related fields during copy_process()
a043e3b2 5515 * @child: pointer to task_struct of forking parent process.
b4f48b63 5516 *
eaf797ab
TH
5517 * A task is associated with the init_css_set until cgroup_post_fork()
5518 * attaches it to the parent's css_set. Empty cg_list indicates that
5519 * @child isn't holding reference to its css_set.
b4f48b63
PM
5520 */
5521void cgroup_fork(struct task_struct *child)
5522{
eaf797ab 5523 RCU_INIT_POINTER(child->cgroups, &init_css_set);
817929ec 5524 INIT_LIST_HEAD(&child->cg_list);
b4f48b63
PM
5525}
5526
7e47682e
AS
5527/**
5528 * cgroup_can_fork - called on a new task before the process is exposed
5529 * @child: the task in question.
5530 *
5531 * This calls the subsystem can_fork() callbacks. If the can_fork() callback
5532 * returns an error, the fork aborts with that error code. This allows for
5533 * a cgroup subsystem to conditionally allow or deny new forks.
5534 */
b53202e6 5535int cgroup_can_fork(struct task_struct *child)
7e47682e
AS
5536{
5537 struct cgroup_subsys *ss;
5538 int i, j, ret;
5539
b4e0eeaf 5540 do_each_subsys_mask(ss, i, have_canfork_callback) {
b53202e6 5541 ret = ss->can_fork(child);
7e47682e
AS
5542 if (ret)
5543 goto out_revert;
b4e0eeaf 5544 } while_each_subsys_mask();
7e47682e
AS
5545
5546 return 0;
5547
5548out_revert:
5549 for_each_subsys(ss, j) {
5550 if (j >= i)
5551 break;
5552 if (ss->cancel_fork)
b53202e6 5553 ss->cancel_fork(child);
7e47682e
AS
5554 }
5555
5556 return ret;
5557}
5558
5559/**
5560 * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
5561 * @child: the task in question
5562 *
5563 * This calls the cancel_fork() callbacks if a fork failed *after*
5564 * cgroup_can_fork() succeded.
5565 */
b53202e6 5566void cgroup_cancel_fork(struct task_struct *child)
7e47682e
AS
5567{
5568 struct cgroup_subsys *ss;
5569 int i;
5570
5571 for_each_subsys(ss, i)
5572 if (ss->cancel_fork)
b53202e6 5573 ss->cancel_fork(child);
7e47682e
AS
5574}
5575
817929ec 5576/**
a043e3b2
LZ
5577 * cgroup_post_fork - called on a new task after adding it to the task list
5578 * @child: the task in question
5579 *
5edee61e
TH
5580 * Adds the task to the list running through its css_set if necessary and
5581 * call the subsystem fork() callbacks. Has to be after the task is
5582 * visible on the task list in case we race with the first call to
0942eeee 5583 * cgroup_task_iter_start() - to guarantee that the new task ends up on its
5edee61e 5584 * list.
a043e3b2 5585 */
b53202e6 5586void cgroup_post_fork(struct task_struct *child)
817929ec 5587{
30159ec7 5588 struct cgroup_subsys *ss;
5edee61e
TH
5589 int i;
5590
3ce3230a 5591 /*
251f8c03 5592 * This may race against cgroup_enable_task_cg_lists(). As that
eaf797ab
TH
5593 * function sets use_task_css_set_links before grabbing
5594 * tasklist_lock and we just went through tasklist_lock to add
5595 * @child, it's guaranteed that either we see the set
5596 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees
5597 * @child during its iteration.
5598 *
5599 * If we won the race, @child is associated with %current's
f0d9a5f1 5600 * css_set. Grabbing css_set_lock guarantees both that the
eaf797ab
TH
5601 * association is stable, and, on completion of the parent's
5602 * migration, @child is visible in the source of migration or
5603 * already in the destination cgroup. This guarantee is necessary
5604 * when implementing operations which need to migrate all tasks of
5605 * a cgroup to another.
5606 *
251f8c03 5607 * Note that if we lose to cgroup_enable_task_cg_lists(), @child
eaf797ab
TH
5608 * will remain in init_css_set. This is safe because all tasks are
5609 * in the init_css_set before cg_links is enabled and there's no
5610 * operation which transfers all tasks out of init_css_set.
3ce3230a 5611 */
817929ec 5612 if (use_task_css_set_links) {
eaf797ab
TH
5613 struct css_set *cset;
5614
f0d9a5f1 5615 spin_lock_bh(&css_set_lock);
0e1d768f 5616 cset = task_css_set(current);
eaf797ab 5617 if (list_empty(&child->cg_list)) {
eaf797ab 5618 get_css_set(cset);
f6d7d049 5619 css_set_move_task(child, NULL, cset, false);
eaf797ab 5620 }
f0d9a5f1 5621 spin_unlock_bh(&css_set_lock);
817929ec 5622 }
5edee61e
TH
5623
5624 /*
5625 * Call ss->fork(). This must happen after @child is linked on
5626 * css_set; otherwise, @child might change state between ->fork()
5627 * and addition to css_set.
5628 */
b4e0eeaf 5629 do_each_subsys_mask(ss, i, have_fork_callback) {
b53202e6 5630 ss->fork(child);
b4e0eeaf 5631 } while_each_subsys_mask();
817929ec 5632}
5edee61e 5633
b4f48b63
PM
5634/**
5635 * cgroup_exit - detach cgroup from exiting task
5636 * @tsk: pointer to task_struct of exiting process
5637 *
5638 * Description: Detach cgroup from @tsk and release it.
5639 *
5640 * Note that cgroups marked notify_on_release force every task in
5641 * them to take the global cgroup_mutex mutex when exiting.
5642 * This could impact scaling on very large systems. Be reluctant to
5643 * use notify_on_release cgroups where very high task exit scaling
5644 * is required on large systems.
5645 *
0e1d768f
TH
5646 * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We
5647 * call cgroup_exit() while the task is still competent to handle
5648 * notify_on_release(), then leave the task attached to the root cgroup in
5649 * each hierarchy for the remainder of its exit. No need to bother with
5650 * init_css_set refcnting. init_css_set never goes away and we can't race
e8604cb4 5651 * with migration path - PF_EXITING is visible to migration path.
b4f48b63 5652 */
1ec41830 5653void cgroup_exit(struct task_struct *tsk)
b4f48b63 5654{
30159ec7 5655 struct cgroup_subsys *ss;
5abb8855 5656 struct css_set *cset;
d41d5a01 5657 int i;
817929ec
PM
5658
5659 /*
0e1d768f 5660 * Unlink from @tsk from its css_set. As migration path can't race
0de0942d 5661 * with us, we can check css_set and cg_list without synchronization.
817929ec 5662 */
0de0942d
TH
5663 cset = task_css_set(tsk);
5664
817929ec 5665 if (!list_empty(&tsk->cg_list)) {
f0d9a5f1 5666 spin_lock_bh(&css_set_lock);
f6d7d049 5667 css_set_move_task(tsk, cset, NULL, false);
f0d9a5f1 5668 spin_unlock_bh(&css_set_lock);
2e91fa7f
TH
5669 } else {
5670 get_css_set(cset);
817929ec
PM
5671 }
5672
cb4a3167 5673 /* see cgroup_post_fork() for details */
b4e0eeaf 5674 do_each_subsys_mask(ss, i, have_exit_callback) {
2e91fa7f 5675 ss->exit(tsk);
b4e0eeaf 5676 } while_each_subsys_mask();
2e91fa7f 5677}
30159ec7 5678
2e91fa7f
TH
5679void cgroup_free(struct task_struct *task)
5680{
5681 struct css_set *cset = task_css_set(task);
afcf6c8b
TH
5682 struct cgroup_subsys *ss;
5683 int ssid;
5684
b4e0eeaf 5685 do_each_subsys_mask(ss, ssid, have_free_callback) {
afcf6c8b 5686 ss->free(task);
b4e0eeaf 5687 } while_each_subsys_mask();
d41d5a01 5688
2e91fa7f 5689 put_css_set(cset);
b4f48b63 5690}
697f4161 5691
bd89aabc 5692static void check_for_release(struct cgroup *cgrp)
81a6a5cd 5693{
27bd4dbb 5694 if (notify_on_release(cgrp) && !cgroup_is_populated(cgrp) &&
971ff493
ZL
5695 !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp))
5696 schedule_work(&cgrp->release_agent_work);
81a6a5cd
PM
5697}
5698
81a6a5cd
PM
5699/*
5700 * Notify userspace when a cgroup is released, by running the
5701 * configured release agent with the name of the cgroup (path
5702 * relative to the root of cgroup file system) as the argument.
5703 *
5704 * Most likely, this user command will try to rmdir this cgroup.
5705 *
5706 * This races with the possibility that some other task will be
5707 * attached to this cgroup before it is removed, or that some other
5708 * user task will 'mkdir' a child cgroup of this cgroup. That's ok.
5709 * The presumed 'rmdir' will fail quietly if this cgroup is no longer
5710 * unused, and this cgroup will be reprieved from its death sentence,
5711 * to continue to serve a useful existence. Next time it's released,
5712 * we will get notified again, if it still has 'notify_on_release' set.
5713 *
5714 * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which
5715 * means only wait until the task is successfully execve()'d. The
5716 * separate release agent task is forked by call_usermodehelper(),
5717 * then control in this thread returns here, without waiting for the
5718 * release agent task. We don't bother to wait because the caller of
5719 * this routine has no use for the exit status of the release agent
5720 * task, so no sense holding our caller up for that.
81a6a5cd 5721 */
81a6a5cd
PM
5722static void cgroup_release_agent(struct work_struct *work)
5723{
971ff493
ZL
5724 struct cgroup *cgrp =
5725 container_of(work, struct cgroup, release_agent_work);
5726 char *pathbuf = NULL, *agentbuf = NULL, *path;
5727 char *argv[3], *envp[3];
5728
81a6a5cd 5729 mutex_lock(&cgroup_mutex);
971ff493
ZL
5730
5731 pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
5732 agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
5733 if (!pathbuf || !agentbuf)
5734 goto out;
5735
5736 path = cgroup_path(cgrp, pathbuf, PATH_MAX);
5737 if (!path)
5738 goto out;
5739
5740 argv[0] = agentbuf;
5741 argv[1] = path;
5742 argv[2] = NULL;
5743
5744 /* minimal command environment */
5745 envp[0] = "HOME=/";
5746 envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
5747 envp[2] = NULL;
5748
81a6a5cd 5749 mutex_unlock(&cgroup_mutex);
971ff493 5750 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
3e2cd91a 5751 goto out_free;
971ff493 5752out:
81a6a5cd 5753 mutex_unlock(&cgroup_mutex);
3e2cd91a 5754out_free:
971ff493
ZL
5755 kfree(agentbuf);
5756 kfree(pathbuf);
81a6a5cd 5757}
8bab8dde
PM
5758
5759static int __init cgroup_disable(char *str)
5760{
30159ec7 5761 struct cgroup_subsys *ss;
8bab8dde 5762 char *token;
30159ec7 5763 int i;
8bab8dde
PM
5764
5765 while ((token = strsep(&str, ",")) != NULL) {
5766 if (!*token)
5767 continue;
be45c900 5768
3ed80a62 5769 for_each_subsys(ss, i) {
3e1d2eed
TH
5770 if (strcmp(token, ss->name) &&
5771 strcmp(token, ss->legacy_name))
5772 continue;
a3e72739 5773 cgroup_disable_mask |= 1 << i;
8bab8dde
PM
5774 }
5775 }
5776 return 1;
5777}
5778__setup("cgroup_disable=", cgroup_disable);
38460b48 5779
223ffb29
JW
5780static int __init cgroup_no_v1(char *str)
5781{
5782 struct cgroup_subsys *ss;
5783 char *token;
5784 int i;
5785
5786 while ((token = strsep(&str, ",")) != NULL) {
5787 if (!*token)
5788 continue;
5789
5790 if (!strcmp(token, "all")) {
6e5c8307 5791 cgroup_no_v1_mask = U16_MAX;
223ffb29
JW
5792 break;
5793 }
5794
5795 for_each_subsys(ss, i) {
5796 if (strcmp(token, ss->name) &&
5797 strcmp(token, ss->legacy_name))
5798 continue;
5799
5800 cgroup_no_v1_mask |= 1 << i;
5801 }
5802 }
5803 return 1;
5804}
5805__setup("cgroup_no_v1=", cgroup_no_v1);
5806
b77d7b60 5807/**
ec903c0c 5808 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
35cf0836
TH
5809 * @dentry: directory dentry of interest
5810 * @ss: subsystem of interest
b77d7b60 5811 *
5a17f543
TH
5812 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
5813 * to get the corresponding css and return it. If such css doesn't exist
5814 * or can't be pinned, an ERR_PTR value is returned.
e5d1367f 5815 */
ec903c0c
TH
5816struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
5817 struct cgroup_subsys *ss)
e5d1367f 5818{
2bd59d48 5819 struct kernfs_node *kn = kernfs_node_from_dentry(dentry);
f17fc25f 5820 struct file_system_type *s_type = dentry->d_sb->s_type;
2bd59d48 5821 struct cgroup_subsys_state *css = NULL;
e5d1367f 5822 struct cgroup *cgrp;
e5d1367f 5823
35cf0836 5824 /* is @dentry a cgroup dir? */
f17fc25f
TH
5825 if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) ||
5826 !kn || kernfs_type(kn) != KERNFS_DIR)
e5d1367f
SE
5827 return ERR_PTR(-EBADF);
5828
5a17f543
TH
5829 rcu_read_lock();
5830
2bd59d48
TH
5831 /*
5832 * This path doesn't originate from kernfs and @kn could already
5833 * have been or be removed at any point. @kn->priv is RCU
a4189487 5834 * protected for this access. See css_release_work_fn() for details.
2bd59d48
TH
5835 */
5836 cgrp = rcu_dereference(kn->priv);
5837 if (cgrp)
5838 css = cgroup_css(cgrp, ss);
5a17f543 5839
ec903c0c 5840 if (!css || !css_tryget_online(css))
5a17f543
TH
5841 css = ERR_PTR(-ENOENT);
5842
5843 rcu_read_unlock();
5844 return css;
e5d1367f 5845}
e5d1367f 5846
1cb650b9
LZ
5847/**
5848 * css_from_id - lookup css by id
5849 * @id: the cgroup id
5850 * @ss: cgroup subsys to be looked into
5851 *
5852 * Returns the css if there's valid one with @id, otherwise returns NULL.
5853 * Should be called under rcu_read_lock().
5854 */
5855struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss)
5856{
6fa4918d 5857 WARN_ON_ONCE(!rcu_read_lock_held());
adbe427b 5858 return id > 0 ? idr_find(&ss->css_idr, id) : NULL;
e5d1367f
SE
5859}
5860
16af4396
TH
5861/**
5862 * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
5863 * @path: path on the default hierarchy
5864 *
5865 * Find the cgroup at @path on the default hierarchy, increment its
5866 * reference count and return it. Returns pointer to the found cgroup on
5867 * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR)
5868 * if @path points to a non-directory.
5869 */
5870struct cgroup *cgroup_get_from_path(const char *path)
5871{
5872 struct kernfs_node *kn;
5873 struct cgroup *cgrp;
5874
5875 mutex_lock(&cgroup_mutex);
5876
5877 kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path);
5878 if (kn) {
5879 if (kernfs_type(kn) == KERNFS_DIR) {
5880 cgrp = kn->priv;
5881 cgroup_get(cgrp);
5882 } else {
5883 cgrp = ERR_PTR(-ENOTDIR);
5884 }
5885 kernfs_put(kn);
5886 } else {
5887 cgrp = ERR_PTR(-ENOENT);
5888 }
5889
5890 mutex_unlock(&cgroup_mutex);
5891 return cgrp;
5892}
5893EXPORT_SYMBOL_GPL(cgroup_get_from_path);
5894
bd1060a1
TH
5895/*
5896 * sock->sk_cgrp_data handling. For more info, see sock_cgroup_data
5897 * definition in cgroup-defs.h.
5898 */
5899#ifdef CONFIG_SOCK_CGROUP_DATA
5900
5901#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)
5902
3fa4cc9c 5903DEFINE_SPINLOCK(cgroup_sk_update_lock);
bd1060a1
TH
5904static bool cgroup_sk_alloc_disabled __read_mostly;
5905
5906void cgroup_sk_alloc_disable(void)
5907{
5908 if (cgroup_sk_alloc_disabled)
5909 return;
5910 pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n");
5911 cgroup_sk_alloc_disabled = true;
5912}
5913
5914#else
5915
5916#define cgroup_sk_alloc_disabled false
5917
5918#endif
5919
5920void cgroup_sk_alloc(struct sock_cgroup_data *skcd)
5921{
5922 if (cgroup_sk_alloc_disabled)
5923 return;
5924
5925 rcu_read_lock();
5926
5927 while (true) {
5928 struct css_set *cset;
5929
5930 cset = task_css_set(current);
5931 if (likely(cgroup_tryget(cset->dfl_cgrp))) {
5932 skcd->val = (unsigned long)cset->dfl_cgrp;
5933 break;
5934 }
5935 cpu_relax();
5936 }
5937
5938 rcu_read_unlock();
5939}
5940
5941void cgroup_sk_free(struct sock_cgroup_data *skcd)
5942{
5943 cgroup_put(sock_cgroup_ptr(skcd));
5944}
5945
5946#endif /* CONFIG_SOCK_CGROUP_DATA */
5947
fe693435 5948#ifdef CONFIG_CGROUP_DEBUG
eb95419b
TH
5949static struct cgroup_subsys_state *
5950debug_css_alloc(struct cgroup_subsys_state *parent_css)
fe693435
PM
5951{
5952 struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
5953
5954 if (!css)
5955 return ERR_PTR(-ENOMEM);
5956
5957 return css;
5958}
5959
eb95419b 5960static void debug_css_free(struct cgroup_subsys_state *css)
fe693435 5961{
eb95419b 5962 kfree(css);
fe693435
PM
5963}
5964
182446d0
TH
5965static u64 debug_taskcount_read(struct cgroup_subsys_state *css,
5966 struct cftype *cft)
fe693435 5967{
182446d0 5968 return cgroup_task_count(css->cgroup);
fe693435
PM
5969}
5970
182446d0
TH
5971static u64 current_css_set_read(struct cgroup_subsys_state *css,
5972 struct cftype *cft)
fe693435
PM
5973{
5974 return (u64)(unsigned long)current->cgroups;
5975}
5976
182446d0 5977static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css,
03c78cbe 5978 struct cftype *cft)
fe693435
PM
5979{
5980 u64 count;
5981
5982 rcu_read_lock();
a8ad805c 5983 count = atomic_read(&task_css_set(current)->refcount);
fe693435
PM
5984 rcu_read_unlock();
5985 return count;
5986}
5987
2da8ca82 5988static int current_css_set_cg_links_read(struct seq_file *seq, void *v)
7717f7ba 5989{
69d0206c 5990 struct cgrp_cset_link *link;
5abb8855 5991 struct css_set *cset;
e61734c5
TH
5992 char *name_buf;
5993
5994 name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL);
5995 if (!name_buf)
5996 return -ENOMEM;
7717f7ba 5997
f0d9a5f1 5998 spin_lock_bh(&css_set_lock);
7717f7ba 5999 rcu_read_lock();
5abb8855 6000 cset = rcu_dereference(current->cgroups);
69d0206c 6001 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {
7717f7ba 6002 struct cgroup *c = link->cgrp;
7717f7ba 6003
a2dd4247 6004 cgroup_name(c, name_buf, NAME_MAX + 1);
2c6ab6d2 6005 seq_printf(seq, "Root %d group %s\n",
a2dd4247 6006 c->root->hierarchy_id, name_buf);
7717f7ba
PM
6007 }
6008 rcu_read_unlock();
f0d9a5f1 6009 spin_unlock_bh(&css_set_lock);
e61734c5 6010 kfree(name_buf);
7717f7ba
PM
6011 return 0;
6012}
6013
6014#define MAX_TASKS_SHOWN_PER_CSS 25
2da8ca82 6015static int cgroup_css_links_read(struct seq_file *seq, void *v)
7717f7ba 6016{
2da8ca82 6017 struct cgroup_subsys_state *css = seq_css(seq);
69d0206c 6018 struct cgrp_cset_link *link;
7717f7ba 6019
f0d9a5f1 6020 spin_lock_bh(&css_set_lock);
182446d0 6021 list_for_each_entry(link, &css->cgroup->cset_links, cset_link) {
69d0206c 6022 struct css_set *cset = link->cset;
7717f7ba
PM
6023 struct task_struct *task;
6024 int count = 0;
c7561128 6025
5abb8855 6026 seq_printf(seq, "css_set %p\n", cset);
c7561128 6027
5abb8855 6028 list_for_each_entry(task, &cset->tasks, cg_list) {
c7561128
TH
6029 if (count++ > MAX_TASKS_SHOWN_PER_CSS)
6030 goto overflow;
6031 seq_printf(seq, " task %d\n", task_pid_vnr(task));
6032 }
6033
6034 list_for_each_entry(task, &cset->mg_tasks, cg_list) {
6035 if (count++ > MAX_TASKS_SHOWN_PER_CSS)
6036 goto overflow;
6037 seq_printf(seq, " task %d\n", task_pid_vnr(task));
7717f7ba 6038 }
c7561128
TH
6039 continue;
6040 overflow:
6041 seq_puts(seq, " ...\n");
7717f7ba 6042 }
f0d9a5f1 6043 spin_unlock_bh(&css_set_lock);
7717f7ba
PM
6044 return 0;
6045}
6046
182446d0 6047static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft)
fe693435 6048{
27bd4dbb 6049 return (!cgroup_is_populated(css->cgroup) &&
a25eb52e 6050 !css_has_online_children(&css->cgroup->self));
fe693435
PM
6051}
6052
6053static struct cftype debug_files[] = {
fe693435
PM
6054 {
6055 .name = "taskcount",
6056 .read_u64 = debug_taskcount_read,
6057 },
6058
6059 {
6060 .name = "current_css_set",
6061 .read_u64 = current_css_set_read,
6062 },
6063
6064 {
6065 .name = "current_css_set_refcount",
6066 .read_u64 = current_css_set_refcount_read,
6067 },
6068
7717f7ba
PM
6069 {
6070 .name = "current_css_set_cg_links",
2da8ca82 6071 .seq_show = current_css_set_cg_links_read,
7717f7ba
PM
6072 },
6073
6074 {
6075 .name = "cgroup_css_links",
2da8ca82 6076 .seq_show = cgroup_css_links_read,
7717f7ba
PM
6077 },
6078
fe693435
PM
6079 {
6080 .name = "releasable",
6081 .read_u64 = releasable_read,
6082 },
fe693435 6083
4baf6e33
TH
6084 { } /* terminate */
6085};
fe693435 6086
073219e9 6087struct cgroup_subsys debug_cgrp_subsys = {
92fb9748
TH
6088 .css_alloc = debug_css_alloc,
6089 .css_free = debug_css_free,
5577964e 6090 .legacy_cftypes = debug_files,
fe693435
PM
6091};
6092#endif /* CONFIG_CGROUP_DEBUG */