Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
105ab3d8 IM |
2 | #ifndef _LINUX_SCHED_TOPOLOGY_H |
3 | #define _LINUX_SCHED_TOPOLOGY_H | |
4 | ||
ee6a3d19 IM |
5 | #include <linux/topology.h> |
6 | ||
4c822698 IM |
7 | #include <linux/sched/idle.h> |
8 | ||
7e1a9208 JL |
9 | /* |
10 | * Increase resolution of cpu_capacity calculations | |
11 | */ | |
12 | #define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT | |
13 | #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) | |
14 | ||
a60b9eda IM |
15 | /* |
16 | * sched-domains (multiprocessor balancing) declarations: | |
17 | */ | |
18 | #ifdef CONFIG_SMP | |
19 | ||
20 | #define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */ | |
21 | #define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */ | |
22 | #define SD_BALANCE_EXEC 0x0004 /* Balance on exec */ | |
23 | #define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ | |
24 | #define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ | |
25 | #define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ | |
05484e09 MR |
26 | #define SD_ASYM_CPUCAPACITY 0x0040 /* Domain members have different CPU capacities */ |
27 | #define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share CPU capacity */ | |
a60b9eda | 28 | #define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ |
05484e09 | 29 | #define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share CPU pkg resources */ |
a60b9eda IM |
30 | #define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ |
31 | #define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ | |
32 | #define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ | |
33 | #define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ | |
34 | #define SD_NUMA 0x4000 /* cross-node balancing */ | |
35 | ||
a60b9eda IM |
36 | #ifdef CONFIG_SCHED_SMT |
37 | static inline int cpu_smt_flags(void) | |
38 | { | |
39 | return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; | |
40 | } | |
41 | #endif | |
42 | ||
43 | #ifdef CONFIG_SCHED_MC | |
44 | static inline int cpu_core_flags(void) | |
45 | { | |
46 | return SD_SHARE_PKG_RESOURCES; | |
47 | } | |
48 | #endif | |
49 | ||
50 | #ifdef CONFIG_NUMA | |
51 | static inline int cpu_numa_flags(void) | |
52 | { | |
53 | return SD_NUMA; | |
54 | } | |
55 | #endif | |
56 | ||
57 | extern int arch_asym_cpu_priority(int cpu); | |
58 | ||
59 | struct sched_domain_attr { | |
60 | int relax_domain_level; | |
61 | }; | |
62 | ||
63 | #define SD_ATTR_INIT (struct sched_domain_attr) { \ | |
64 | .relax_domain_level = -1, \ | |
65 | } | |
66 | ||
67 | extern int sched_domain_level_max; | |
68 | ||
69 | struct sched_group; | |
70 | ||
71 | struct sched_domain_shared { | |
72 | atomic_t ref; | |
73 | atomic_t nr_busy_cpus; | |
74 | int has_idle_cores; | |
75 | }; | |
76 | ||
77 | struct sched_domain { | |
78 | /* These fields must be setup */ | |
994aeb7a JFG |
79 | struct sched_domain __rcu *parent; /* top domain must be null terminated */ |
80 | struct sched_domain __rcu *child; /* bottom domain must be null terminated */ | |
a60b9eda IM |
81 | struct sched_group *groups; /* the balancing groups of the domain */ |
82 | unsigned long min_interval; /* Minimum balance interval ms */ | |
83 | unsigned long max_interval; /* Maximum balance interval ms */ | |
84 | unsigned int busy_factor; /* less balancing by factor if busy */ | |
85 | unsigned int imbalance_pct; /* No balance until over watermark */ | |
86 | unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */ | |
87 | unsigned int busy_idx; | |
88 | unsigned int idle_idx; | |
89 | unsigned int newidle_idx; | |
90 | unsigned int wake_idx; | |
91 | unsigned int forkexec_idx; | |
a60b9eda IM |
92 | |
93 | int nohz_idle; /* NOHZ IDLE status */ | |
94 | int flags; /* See SD_* */ | |
95 | int level; | |
96 | ||
97 | /* Runtime fields. */ | |
98 | unsigned long last_balance; /* init to jiffies. units in jiffies */ | |
99 | unsigned int balance_interval; /* initialise to 1. units in ms. */ | |
100 | unsigned int nr_balance_failed; /* initialise to 0 */ | |
101 | ||
102 | /* idle_balance() stats */ | |
103 | u64 max_newidle_lb_cost; | |
104 | unsigned long next_decay_max_lb_cost; | |
105 | ||
106 | u64 avg_scan_cost; /* select_idle_sibling */ | |
107 | ||
108 | #ifdef CONFIG_SCHEDSTATS | |
109 | /* load_balance() stats */ | |
110 | unsigned int lb_count[CPU_MAX_IDLE_TYPES]; | |
111 | unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; | |
112 | unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; | |
113 | unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; | |
114 | unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; | |
115 | unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; | |
116 | unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; | |
117 | unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; | |
118 | ||
119 | /* Active load balancing */ | |
120 | unsigned int alb_count; | |
121 | unsigned int alb_failed; | |
122 | unsigned int alb_pushed; | |
123 | ||
124 | /* SD_BALANCE_EXEC stats */ | |
125 | unsigned int sbe_count; | |
126 | unsigned int sbe_balanced; | |
127 | unsigned int sbe_pushed; | |
128 | ||
129 | /* SD_BALANCE_FORK stats */ | |
130 | unsigned int sbf_count; | |
131 | unsigned int sbf_balanced; | |
132 | unsigned int sbf_pushed; | |
133 | ||
134 | /* try_to_wake_up() stats */ | |
135 | unsigned int ttwu_wake_remote; | |
136 | unsigned int ttwu_move_affine; | |
137 | unsigned int ttwu_move_balance; | |
138 | #endif | |
139 | #ifdef CONFIG_SCHED_DEBUG | |
140 | char *name; | |
141 | #endif | |
142 | union { | |
143 | void *private; /* used during construction */ | |
144 | struct rcu_head rcu; /* used during destruction */ | |
145 | }; | |
146 | struct sched_domain_shared *shared; | |
147 | ||
148 | unsigned int span_weight; | |
149 | /* | |
150 | * Span of all CPUs in this domain. | |
151 | * | |
152 | * NOTE: this field is variable length. (Allocated dynamically | |
153 | * by attaching extra space to the end of the structure, | |
154 | * depending on how many CPUs the kernel has booted up with) | |
155 | */ | |
156 | unsigned long span[0]; | |
157 | }; | |
158 | ||
159 | static inline struct cpumask *sched_domain_span(struct sched_domain *sd) | |
160 | { | |
161 | return to_cpumask(sd->span); | |
162 | } | |
163 | ||
164 | extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], | |
165 | struct sched_domain_attr *dattr_new); | |
166 | ||
167 | /* Allocate an array of sched domains, for partition_sched_domains(). */ | |
168 | cpumask_var_t *alloc_sched_domains(unsigned int ndoms); | |
169 | void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); | |
170 | ||
171 | bool cpus_share_cache(int this_cpu, int that_cpu); | |
172 | ||
173 | typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); | |
174 | typedef int (*sched_domain_flags_f)(void); | |
175 | ||
176 | #define SDTL_OVERLAP 0x01 | |
177 | ||
178 | struct sd_data { | |
99687cdb LVO |
179 | struct sched_domain *__percpu *sd; |
180 | struct sched_domain_shared *__percpu *sds; | |
181 | struct sched_group *__percpu *sg; | |
182 | struct sched_group_capacity *__percpu *sgc; | |
a60b9eda IM |
183 | }; |
184 | ||
185 | struct sched_domain_topology_level { | |
186 | sched_domain_mask_f mask; | |
187 | sched_domain_flags_f sd_flags; | |
188 | int flags; | |
189 | int numa_level; | |
190 | struct sd_data data; | |
191 | #ifdef CONFIG_SCHED_DEBUG | |
192 | char *name; | |
193 | #endif | |
194 | }; | |
195 | ||
196 | extern void set_sched_topology(struct sched_domain_topology_level *tl); | |
197 | ||
198 | #ifdef CONFIG_SCHED_DEBUG | |
199 | # define SD_INIT_NAME(type) .name = #type | |
200 | #else | |
201 | # define SD_INIT_NAME(type) | |
202 | #endif | |
203 | ||
5bd0988b QP |
204 | #ifndef arch_scale_cpu_capacity |
205 | static __always_inline | |
206 | unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu) | |
207 | { | |
208 | return SCHED_CAPACITY_SCALE; | |
209 | } | |
210 | #endif | |
211 | ||
a60b9eda IM |
212 | #else /* CONFIG_SMP */ |
213 | ||
214 | struct sched_domain_attr; | |
215 | ||
216 | static inline void | |
217 | partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], | |
218 | struct sched_domain_attr *dattr_new) | |
219 | { | |
220 | } | |
221 | ||
222 | static inline bool cpus_share_cache(int this_cpu, int that_cpu) | |
223 | { | |
224 | return true; | |
225 | } | |
226 | ||
5bd0988b QP |
227 | #ifndef arch_scale_cpu_capacity |
228 | static __always_inline | |
229 | unsigned long arch_scale_cpu_capacity(void __always_unused *sd, int cpu) | |
230 | { | |
231 | return SCHED_CAPACITY_SCALE; | |
232 | } | |
233 | #endif | |
234 | ||
a60b9eda IM |
235 | #endif /* !CONFIG_SMP */ |
236 | ||
ee6a3d19 IM |
237 | static inline int task_node(const struct task_struct *p) |
238 | { | |
239 | return cpu_to_node(task_cpu(p)); | |
240 | } | |
241 | ||
105ab3d8 | 242 | #endif /* _LINUX_SCHED_TOPOLOGY_H */ |