kernel/sched/isolation.c

   1 /*
   2  *  Housekeeping management. Manage the targets for routine code that can run on
   3  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   4  *
   5  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   6  *
   7  */
   8
   9 #include <linux/sched/isolation.h>
  10 #include <linux/tick.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/static_key.h>
  14
  15 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
  16 EXPORT_SYMBOL_GPL(housekeeping_overriden);
  17 static cpumask_var_t housekeeping_mask;
  18 static unsigned int housekeeping_flags;
  19
  20 int housekeeping_any_cpu(enum hk_flags flags)
  21 {
  22         if (static_branch_unlikely(&housekeeping_overriden))
  23                 if (housekeeping_flags & flags)
  24                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  25         return smp_processor_id();
  26 }
  27 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  28
  29 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  30 {
  31         if (static_branch_unlikely(&housekeeping_overriden))
  32                 if (housekeeping_flags & flags)
  33                         return housekeeping_mask;
  34         return cpu_possible_mask;
  35 }
  36 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  37
  38 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  39 {
  40         if (static_branch_unlikely(&housekeeping_overriden))
  41                 if (housekeeping_flags & flags)
  42                         set_cpus_allowed_ptr(t, housekeeping_mask);
  43 }
  44 EXPORT_SYMBOL_GPL(housekeeping_affine);
  45
  46 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  47 {
  48         if (static_branch_unlikely(&housekeeping_overriden))
  49                 if (housekeeping_flags & flags)
  50                         return cpumask_test_cpu(cpu, housekeeping_mask);
  51         return true;
  52 }
  53 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  54
  55 void __init housekeeping_init(void)
  56 {
  57         if (!housekeeping_flags)
  58                 return;
  59
  60         static_branch_enable(&housekeeping_overriden);
  61
  62         /* We need at least one CPU to handle housekeeping work */
  63         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  64 }
  65
  66 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  67 {
  68         cpumask_var_t non_housekeeping_mask;
  69         int err;
  70
  71         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  72         err = cpulist_parse(str, non_housekeeping_mask);
  73         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  74                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  75                 free_bootmem_cpumask_var(non_housekeeping_mask);
  76                 return 0;
  77         }
  78
  79         if (!housekeeping_flags) {
  80                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  81                 cpumask_andnot(housekeeping_mask,
  82                                cpu_possible_mask, non_housekeeping_mask);
  83                 if (cpumask_empty(housekeeping_mask))
  84                         cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  85         } else {
  86                 cpumask_var_t tmp;
  87
  88                 alloc_bootmem_cpumask_var(&tmp);
  89                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  90                 if (!cpumask_equal(tmp, housekeeping_mask)) {
  91                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  92                         free_bootmem_cpumask_var(tmp);
  93                         free_bootmem_cpumask_var(non_housekeeping_mask);
  94                         return 0;
  95                 }
  96                 free_bootmem_cpumask_var(tmp);
  97         }
  98
  99         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 100                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 101                         tick_nohz_full_setup(non_housekeeping_mask);
 102                 } else {
 103                         pr_warn("Housekeeping: nohz unsupported."
 104                                 " Build with CONFIG_NO_HZ_FULL\n");
 105                         free_bootmem_cpumask_var(non_housekeeping_mask);
 106                         return 0;
 107                 }
 108         }
 109
 110         housekeeping_flags |= flags;
 111
 112         free_bootmem_cpumask_var(non_housekeeping_mask);
 113
 114         return 1;
 115 }
 116
 117 static int __init housekeeping_nohz_full_setup(char *str)
 118 {
 119         unsigned int flags;
 120
 121         flags = HK_FLAG_TICK | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
 122
 123         return housekeeping_setup(str, flags);
 124 }
 125 __setup("nohz_full=", housekeeping_nohz_full_setup);
 126
 127 static int __init housekeeping_isolcpus_setup(char *str)
 128 {
 129         return housekeeping_setup(str, HK_FLAG_DOMAIN);
 130 }
 131 __setup("isolcpus=", housekeeping_isolcpus_setup);