Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9a0ef98e CH |
2 | /* |
3 | * Copyright (C) 2016 Thomas Gleixner. | |
4 | * Copyright (C) 2016-2017 Christoph Hellwig. | |
5 | */ | |
5e385a6e CH |
6 | #include <linux/interrupt.h> |
7 | #include <linux/kernel.h> | |
8 | #include <linux/slab.h> | |
9 | #include <linux/cpu.h> | |
10 | ||
34c3d981 TG |
11 | static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, |
12 | int cpus_per_vec) | |
13 | { | |
14 | const struct cpumask *siblmsk; | |
15 | int cpu, sibl; | |
16 | ||
17 | for ( ; cpus_per_vec > 0; ) { | |
18 | cpu = cpumask_first(nmsk); | |
19 | ||
20 | /* Should not happen, but I'm too lazy to think about it */ | |
21 | if (cpu >= nr_cpu_ids) | |
22 | return; | |
23 | ||
24 | cpumask_clear_cpu(cpu, nmsk); | |
25 | cpumask_set_cpu(cpu, irqmsk); | |
26 | cpus_per_vec--; | |
27 | ||
28 | /* If the cpu has siblings, use them first */ | |
29 | siblmsk = topology_sibling_cpumask(cpu); | |
30 | for (sibl = -1; cpus_per_vec > 0; ) { | |
31 | sibl = cpumask_next(sibl, siblmsk); | |
32 | if (sibl >= nr_cpu_ids) | |
33 | break; | |
34 | if (!cpumask_test_and_clear_cpu(sibl, nmsk)) | |
35 | continue; | |
36 | cpumask_set_cpu(sibl, irqmsk); | |
37 | cpus_per_vec--; | |
38 | } | |
39 | } | |
40 | } | |
41 | ||
9a0ef98e CH |
42 | static cpumask_var_t *alloc_node_to_present_cpumask(void) |
43 | { | |
44 | cpumask_var_t *masks; | |
45 | int node; | |
46 | ||
47 | masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL); | |
48 | if (!masks) | |
49 | return NULL; | |
50 | ||
51 | for (node = 0; node < nr_node_ids; node++) { | |
52 | if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL)) | |
53 | goto out_unwind; | |
54 | } | |
55 | ||
56 | return masks; | |
57 | ||
58 | out_unwind: | |
59 | while (--node >= 0) | |
60 | free_cpumask_var(masks[node]); | |
61 | kfree(masks); | |
62 | return NULL; | |
63 | } | |
64 | ||
65 | static void free_node_to_present_cpumask(cpumask_var_t *masks) | |
66 | { | |
67 | int node; | |
68 | ||
69 | for (node = 0; node < nr_node_ids; node++) | |
70 | free_cpumask_var(masks[node]); | |
71 | kfree(masks); | |
72 | } | |
73 | ||
74 | static void build_node_to_present_cpumask(cpumask_var_t *masks) | |
75 | { | |
76 | int cpu; | |
77 | ||
78 | for_each_present_cpu(cpu) | |
79 | cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]); | |
80 | } | |
81 | ||
82 | static int get_nodes_in_cpumask(cpumask_var_t *node_to_present_cpumask, | |
83 | const struct cpumask *mask, nodemask_t *nodemsk) | |
34c3d981 | 84 | { |
c0af5243 | 85 | int n, nodes = 0; |
34c3d981 TG |
86 | |
87 | /* Calculate the number of nodes in the supplied affinity mask */ | |
9a0ef98e CH |
88 | for_each_node(n) { |
89 | if (cpumask_intersects(mask, node_to_present_cpumask[n])) { | |
34c3d981 TG |
90 | node_set(n, *nodemsk); |
91 | nodes++; | |
92 | } | |
93 | } | |
94 | return nodes; | |
95 | } | |
96 | ||
97 | /** | |
98 | * irq_create_affinity_masks - Create affinity masks for multiqueue spreading | |
67c93c21 CH |
99 | * @nvecs: The total number of vectors |
100 | * @affd: Description of the affinity requirements | |
34c3d981 TG |
101 | * |
102 | * Returns the masks pointer or NULL if allocation failed. | |
103 | */ | |
67c93c21 CH |
104 | struct cpumask * |
105 | irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) | |
34c3d981 | 106 | { |
7bf8222b | 107 | int n, nodes, cpus_per_vec, extra_vecs, curvec; |
67c93c21 | 108 | int affv = nvecs - affd->pre_vectors - affd->post_vectors; |
bfe13077 | 109 | int last_affv = affv + affd->pre_vectors; |
34c3d981 TG |
110 | nodemask_t nodemsk = NODE_MASK_NONE; |
111 | struct cpumask *masks; | |
9a0ef98e | 112 | cpumask_var_t nmsk, *node_to_present_cpumask; |
34c3d981 | 113 | |
6f9a22bc MH |
114 | /* |
115 | * If there aren't any vectors left after applying the pre/post | |
116 | * vectors don't bother with assigning affinity. | |
117 | */ | |
118 | if (!affv) | |
119 | return NULL; | |
120 | ||
34c3d981 TG |
121 | if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) |
122 | return NULL; | |
123 | ||
67c93c21 | 124 | masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); |
34c3d981 TG |
125 | if (!masks) |
126 | goto out; | |
127 | ||
9a0ef98e CH |
128 | node_to_present_cpumask = alloc_node_to_present_cpumask(); |
129 | if (!node_to_present_cpumask) | |
130 | goto out; | |
131 | ||
67c93c21 CH |
132 | /* Fill out vectors at the beginning that don't need affinity */ |
133 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) | |
b6e5d5b9 | 134 | cpumask_copy(masks + curvec, irq_default_affinity); |
67c93c21 | 135 | |
34c3d981 TG |
136 | /* Stabilize the cpumasks */ |
137 | get_online_cpus(); | |
9a0ef98e CH |
138 | build_node_to_present_cpumask(node_to_present_cpumask); |
139 | nodes = get_nodes_in_cpumask(node_to_present_cpumask, cpu_present_mask, | |
140 | &nodemsk); | |
34c3d981 TG |
141 | |
142 | /* | |
c0af5243 | 143 | * If the number of nodes in the mask is greater than or equal the |
34c3d981 TG |
144 | * number of vectors we just spread the vectors across the nodes. |
145 | */ | |
67c93c21 | 146 | if (affv <= nodes) { |
34c3d981 | 147 | for_each_node_mask(n, nodemsk) { |
9a0ef98e CH |
148 | cpumask_copy(masks + curvec, |
149 | node_to_present_cpumask[n]); | |
bfe13077 | 150 | if (++curvec == last_affv) |
34c3d981 TG |
151 | break; |
152 | } | |
67c93c21 | 153 | goto done; |
34c3d981 TG |
154 | } |
155 | ||
34c3d981 | 156 | for_each_node_mask(n, nodemsk) { |
7bf8222b KB |
157 | int ncpus, v, vecs_to_assign, vecs_per_node; |
158 | ||
159 | /* Spread the vectors per node */ | |
b72f8051 | 160 | vecs_per_node = (affv - (curvec - affd->pre_vectors)) / nodes; |
34c3d981 TG |
161 | |
162 | /* Get the cpus on this node which are in the mask */ | |
9a0ef98e | 163 | cpumask_and(nmsk, cpu_present_mask, node_to_present_cpumask[n]); |
34c3d981 TG |
164 | |
165 | /* Calculate the number of cpus per vector */ | |
166 | ncpus = cpumask_weight(nmsk); | |
7bf8222b KB |
167 | vecs_to_assign = min(vecs_per_node, ncpus); |
168 | ||
169 | /* Account for rounding errors */ | |
3412386b | 170 | extra_vecs = ncpus - vecs_to_assign * (ncpus / vecs_to_assign); |
34c3d981 | 171 | |
bfe13077 CH |
172 | for (v = 0; curvec < last_affv && v < vecs_to_assign; |
173 | curvec++, v++) { | |
34c3d981 TG |
174 | cpus_per_vec = ncpus / vecs_to_assign; |
175 | ||
176 | /* Account for extra vectors to compensate rounding errors */ | |
177 | if (extra_vecs) { | |
178 | cpus_per_vec++; | |
7bf8222b | 179 | --extra_vecs; |
34c3d981 TG |
180 | } |
181 | irq_spread_init_one(masks + curvec, nmsk, cpus_per_vec); | |
182 | } | |
183 | ||
bfe13077 | 184 | if (curvec >= last_affv) |
34c3d981 | 185 | break; |
7bf8222b | 186 | --nodes; |
34c3d981 TG |
187 | } |
188 | ||
67c93c21 | 189 | done: |
34c3d981 | 190 | put_online_cpus(); |
67c93c21 CH |
191 | |
192 | /* Fill out vectors at the end that don't need affinity */ | |
193 | for (; curvec < nvecs; curvec++) | |
b6e5d5b9 | 194 | cpumask_copy(masks + curvec, irq_default_affinity); |
9a0ef98e | 195 | free_node_to_present_cpumask(node_to_present_cpumask); |
34c3d981 TG |
196 | out: |
197 | free_cpumask_var(nmsk); | |
198 | return masks; | |
199 | } | |
200 | ||
201 | /** | |
212bd846 | 202 | * irq_calc_affinity_vectors - Calculate the optimal number of vectors |
6f9a22bc | 203 | * @minvec: The minimum number of vectors available |
212bd846 CH |
204 | * @maxvec: The maximum number of vectors available |
205 | * @affd: Description of the affinity requirements | |
34c3d981 | 206 | */ |
6f9a22bc | 207 | int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd) |
34c3d981 | 208 | { |
212bd846 CH |
209 | int resv = affd->pre_vectors + affd->post_vectors; |
210 | int vecs = maxvec - resv; | |
9a0ef98e | 211 | int ret; |
34c3d981 | 212 | |
6f9a22bc MH |
213 | if (resv > minvec) |
214 | return 0; | |
215 | ||
34c3d981 | 216 | get_online_cpus(); |
9a0ef98e | 217 | ret = min_t(int, cpumask_weight(cpu_present_mask), vecs) + resv; |
34c3d981 | 218 | put_online_cpus(); |
9a0ef98e | 219 | return ret; |
34c3d981 | 220 | } |