kernel/sched/isolation.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Housekeeping management. Manage the targets for routine code that can run on
   4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   5  *
   6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   8  *
   9  */
  10
  11 enum hk_flags {
  12         HK_FLAG_TIMER           = BIT(HK_TYPE_TIMER),
  13         HK_FLAG_RCU             = BIT(HK_TYPE_RCU),
  14         HK_FLAG_MISC            = BIT(HK_TYPE_MISC),
  15         HK_FLAG_SCHED           = BIT(HK_TYPE_SCHED),
  16         HK_FLAG_TICK            = BIT(HK_TYPE_TICK),
  17         HK_FLAG_DOMAIN          = BIT(HK_TYPE_DOMAIN),
  18         HK_FLAG_WQ              = BIT(HK_TYPE_WQ),
  19         HK_FLAG_MANAGED_IRQ     = BIT(HK_TYPE_MANAGED_IRQ),
  20         HK_FLAG_KTHREAD         = BIT(HK_TYPE_KTHREAD),
  21 };
  22
  23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  24 EXPORT_SYMBOL_GPL(housekeeping_overridden);
  25
  26 struct housekeeping {
  27         cpumask_var_t cpumasks[HK_TYPE_MAX];
  28         unsigned long flags;
  29 };
  30
  31 static struct housekeeping housekeeping;
  32
  33 bool housekeeping_enabled(enum hk_type type)
  34 {
  35         return !!(housekeeping.flags & BIT(type));
  36 }
  37 EXPORT_SYMBOL_GPL(housekeeping_enabled);
  38
  39 int housekeeping_any_cpu(enum hk_type type)
  40 {
  41         int cpu;
  42
  43         if (static_branch_unlikely(&housekeeping_overridden)) {
  44                 if (housekeeping.flags & BIT(type)) {
  45                         cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
  46                         if (cpu < nr_cpu_ids)
  47                                 return cpu;
  48
  49                         cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
  50                         if (likely(cpu < nr_cpu_ids))
  51                                 return cpu;
  52                         /*
  53                          * Unless we have another problem this can only happen
  54                          * at boot time before start_secondary() brings the 1st
  55                          * housekeeping CPU up.
  56                          */
  57                         WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
  58                                      type != HK_TYPE_TIMER);
  59                 }
  60         }
  61         return smp_processor_id();
  62 }
  63 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  64
  65 const struct cpumask *housekeeping_cpumask(enum hk_type type)
  66 {
  67         if (static_branch_unlikely(&housekeeping_overridden))
  68                 if (housekeeping.flags & BIT(type))
  69                         return housekeeping.cpumasks[type];
  70         return cpu_possible_mask;
  71 }
  72 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  73
  74 void housekeeping_affine(struct task_struct *t, enum hk_type type)
  75 {
  76         if (static_branch_unlikely(&housekeeping_overridden))
  77                 if (housekeeping.flags & BIT(type))
  78                         set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
  79 }
  80 EXPORT_SYMBOL_GPL(housekeeping_affine);
  81
  82 bool housekeeping_test_cpu(int cpu, enum hk_type type)
  83 {
  84         if (static_branch_unlikely(&housekeeping_overridden))
  85                 if (housekeeping.flags & BIT(type))
  86                         return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
  87         return true;
  88 }
  89 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  90
  91 void __init housekeeping_init(void)
  92 {
  93         enum hk_type type;
  94
  95         if (!housekeeping.flags)
  96                 return;
  97
  98         static_branch_enable(&housekeeping_overridden);
  99
 100         if (housekeeping.flags & HK_FLAG_TICK)
 101                 sched_tick_offload_init();
 102
 103         for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
 104                 /* We need at least one CPU to handle housekeeping work */
 105                 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
 106         }
 107 }
 108
 109 static void __init housekeeping_setup_type(enum hk_type type,
 110                                            cpumask_var_t housekeeping_staging)
 111 {
 112
 113         alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
 114         cpumask_copy(housekeeping.cpumasks[type],
 115                      housekeeping_staging);
 116 }
 117
 118 static int __init housekeeping_setup(char *str, unsigned long flags)
 119 {
 120         cpumask_var_t non_housekeeping_mask, housekeeping_staging;
 121         int err = 0;
 122
 123         if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
 124                 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 125                         pr_warn("Housekeeping: nohz unsupported."
 126                                 " Build with CONFIG_NO_HZ_FULL\n");
 127                         return 0;
 128                 }
 129         }
 130
 131         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
 132         if (cpulist_parse(str, non_housekeeping_mask) < 0) {
 133                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
 134                 goto free_non_housekeeping_mask;
 135         }
 136
 137         alloc_bootmem_cpumask_var(&housekeeping_staging);
 138         cpumask_andnot(housekeeping_staging,
 139                        cpu_possible_mask, non_housekeeping_mask);
 140
 141         if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
 142                 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
 143                 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
 144                 if (!housekeeping.flags) {
 145                         pr_warn("Housekeeping: must include one present CPU, "
 146                                 "using boot CPU:%d\n", smp_processor_id());
 147                 }
 148         }
 149
 150         if (!housekeeping.flags) {
 151                 /* First setup call ("nohz_full=" or "isolcpus=") */
 152                 enum hk_type type;
 153
 154                 for_each_set_bit(type, &flags, HK_TYPE_MAX)
 155                         housekeeping_setup_type(type, housekeeping_staging);
 156         } else {
 157                 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
 158                 enum hk_type type;
 159                 unsigned long iter_flags = flags & housekeeping.flags;
 160
 161                 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
 162                         if (!cpumask_equal(housekeeping_staging,
 163                                            housekeeping.cpumasks[type])) {
 164                                 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
 165                                 goto free_housekeeping_staging;
 166                         }
 167                 }
 168
 169                 iter_flags = flags & ~housekeeping.flags;
 170
 171                 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
 172                         housekeeping_setup_type(type, housekeeping_staging);
 173         }
 174
 175         if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
 176                 tick_nohz_full_setup(non_housekeeping_mask);
 177
 178         housekeeping.flags |= flags;
 179         err = 1;
 180
 181 free_housekeeping_staging:
 182         free_bootmem_cpumask_var(housekeeping_staging);
 183 free_non_housekeeping_mask:
 184         free_bootmem_cpumask_var(non_housekeeping_mask);
 185
 186         return err;
 187 }
 188
 189 static int __init housekeeping_nohz_full_setup(char *str)
 190 {
 191         unsigned long flags;
 192
 193         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
 194                 HK_FLAG_MISC | HK_FLAG_KTHREAD;
 195
 196         return housekeeping_setup(str, flags);
 197 }
 198 __setup("nohz_full=", housekeeping_nohz_full_setup);
 199
 200 static int __init housekeeping_isolcpus_setup(char *str)
 201 {
 202         unsigned long flags = 0;
 203         bool illegal = false;
 204         char *par;
 205         int len;
 206
 207         while (isalpha(*str)) {
 208                 if (!strncmp(str, "nohz,", 5)) {
 209                         str += 5;
 210                         flags |= HK_FLAG_TICK;
 211                         continue;
 212                 }
 213
 214                 if (!strncmp(str, "domain,", 7)) {
 215                         str += 7;
 216                         flags |= HK_FLAG_DOMAIN;
 217                         continue;
 218                 }
 219
 220                 if (!strncmp(str, "managed_irq,", 12)) {
 221                         str += 12;
 222                         flags |= HK_FLAG_MANAGED_IRQ;
 223                         continue;
 224                 }
 225
 226                 /*
 227                  * Skip unknown sub-parameter and validate that it is not
 228                  * containing an invalid character.
 229                  */
 230                 for (par = str, len = 0; *str && *str != ','; str++, len++) {
 231                         if (!isalpha(*str) && *str != '_')
 232                                 illegal = true;
 233                 }
 234
 235                 if (illegal) {
 236                         pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
 237                         return 0;
 238                 }
 239
 240                 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
 241                 str++;
 242         }
 243
 244         /* Default behaviour for isolcpus without flags */
 245         if (!flags)
 246                 flags |= HK_FLAG_DOMAIN;
 247
 248         return housekeeping_setup(str, flags);
 249 }
 250 __setup("isolcpus=", housekeeping_isolcpus_setup);