Commit | Line | Data |
---|---|---|
a1957954 BVA |
1 | #include "os/os.h" |
2 | ||
3 | #include <windows.h> | |
4 | ||
5 | #ifdef CONFIG_WINDOWS_XP | |
6 | int fio_setaffinity(int pid, os_cpu_mask_t cpumask) | |
7 | { | |
8 | HANDLE h; | |
9 | BOOL bSuccess = FALSE; | |
10 | ||
11 | h = OpenThread(THREAD_QUERY_INFORMATION | THREAD_SET_INFORMATION, TRUE, | |
12 | pid); | |
13 | if (h != NULL) { | |
14 | bSuccess = SetThreadAffinityMask(h, cpumask); | |
15 | if (!bSuccess) | |
16 | log_err("fio_setaffinity failed: failed to set thread affinity (pid %d, mask %.16llx)\n", | |
4937100f | 17 | pid, (long long unsigned) cpumask); |
a1957954 BVA |
18 | |
19 | CloseHandle(h); | |
20 | } else { | |
21 | log_err("fio_setaffinity failed: failed to get handle for pid %d\n", | |
22 | pid); | |
23 | } | |
24 | ||
25 | return bSuccess ? 0 : -1; | |
26 | } | |
27 | ||
28 | int fio_getaffinity(int pid, os_cpu_mask_t *mask) | |
29 | { | |
30 | os_cpu_mask_t systemMask; | |
31 | ||
32 | HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, pid); | |
33 | ||
34 | if (h != NULL) { | |
35 | GetProcessAffinityMask(h, mask, &systemMask); | |
36 | CloseHandle(h); | |
37 | } else { | |
38 | log_err("fio_getaffinity failed: failed to get handle for pid %d\n", | |
39 | pid); | |
40 | return -1; | |
41 | } | |
42 | ||
43 | return 0; | |
44 | } | |
45 | ||
46 | void fio_cpu_clear(os_cpu_mask_t *mask, int cpu) | |
47 | { | |
48 | *mask &= ~(1ULL << cpu); | |
49 | } | |
50 | ||
51 | void fio_cpu_set(os_cpu_mask_t *mask, int cpu) | |
52 | { | |
53 | *mask |= 1ULL << cpu; | |
54 | } | |
55 | ||
56 | int fio_cpu_isset(os_cpu_mask_t *mask, int cpu) | |
57 | { | |
58 | return (*mask & (1ULL << cpu)) != 0; | |
59 | } | |
60 | ||
61 | int fio_cpu_count(os_cpu_mask_t *mask) | |
62 | { | |
63 | return hweight64(*mask); | |
64 | } | |
65 | ||
66 | int fio_cpuset_init(os_cpu_mask_t *mask) | |
67 | { | |
68 | *mask = 0; | |
69 | return 0; | |
70 | } | |
71 | ||
72 | int fio_cpuset_exit(os_cpu_mask_t *mask) | |
73 | { | |
74 | return 0; | |
75 | } | |
76 | #else /* CONFIG_WINDOWS_XP */ | |
77 | /* Return all processors regardless of processor group */ | |
78 | unsigned int cpus_online(void) | |
79 | { | |
80 | return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); | |
81 | } | |
82 | ||
83 | static void print_mask(os_cpu_mask_t *cpumask) | |
84 | { | |
85 | for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) | |
f5515176 | 86 | dprint(FD_PROCESS, "cpumask[%d]=%" PRIu64 "\n", i, cpumask->row[i]); |
a1957954 BVA |
87 | } |
88 | ||
89 | /* Return the index of the least significant set CPU in cpumask or -1 if no | |
90 | * CPUs are set */ | |
91 | int first_set_cpu(os_cpu_mask_t *cpumask) | |
92 | { | |
93 | int cpus_offset, mask_first_cpu, row; | |
94 | ||
95 | cpus_offset = 0; | |
96 | row = 0; | |
97 | mask_first_cpu = -1; | |
98 | while (mask_first_cpu < 0 && row < FIO_CPU_MASK_ROWS) { | |
99 | int row_first_cpu; | |
100 | ||
101 | row_first_cpu = __builtin_ffsll(cpumask->row[row]) - 1; | |
f5515176 | 102 | dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%" PRIu64 "\n", |
a1957954 BVA |
103 | row_first_cpu, row, cpumask->row[row]); |
104 | if (row_first_cpu > -1) { | |
105 | mask_first_cpu = cpus_offset + row_first_cpu; | |
106 | dprint(FD_PROCESS, "first set cpu in mask is at index %d\n", | |
107 | mask_first_cpu); | |
108 | } else { | |
109 | cpus_offset += FIO_CPU_MASK_STRIDE; | |
110 | row++; | |
111 | } | |
112 | } | |
113 | ||
114 | return mask_first_cpu; | |
115 | } | |
116 | ||
117 | /* Return the index of the most significant set CPU in cpumask or -1 if no | |
118 | * CPUs are set */ | |
119 | static int last_set_cpu(os_cpu_mask_t *cpumask) | |
120 | { | |
121 | int cpus_offset, mask_last_cpu, row; | |
122 | ||
123 | cpus_offset = (FIO_CPU_MASK_ROWS - 1) * FIO_CPU_MASK_STRIDE; | |
124 | row = FIO_CPU_MASK_ROWS - 1; | |
125 | mask_last_cpu = -1; | |
126 | while (mask_last_cpu < 0 && row >= 0) { | |
127 | int row_last_cpu; | |
128 | ||
129 | if (cpumask->row[row] == 0) | |
130 | row_last_cpu = -1; | |
131 | else { | |
132 | uint64_t tmp = cpumask->row[row]; | |
133 | ||
134 | row_last_cpu = 0; | |
135 | while (tmp >>= 1) | |
136 | row_last_cpu++; | |
137 | } | |
138 | ||
f5515176 | 139 | dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%" PRIu64 "\n", |
a1957954 BVA |
140 | row_last_cpu, row, cpumask->row[row]); |
141 | if (row_last_cpu > -1) { | |
142 | mask_last_cpu = cpus_offset + row_last_cpu; | |
143 | dprint(FD_PROCESS, "last set cpu in mask is at index %d\n", | |
144 | mask_last_cpu); | |
145 | } else { | |
146 | cpus_offset -= FIO_CPU_MASK_STRIDE; | |
147 | row--; | |
148 | } | |
149 | } | |
150 | ||
151 | return mask_last_cpu; | |
152 | } | |
153 | ||
154 | static int mask_to_group_mask(os_cpu_mask_t *cpumask, int *processor_group, uint64_t *affinity_mask) | |
155 | { | |
156 | WORD online_groups, group, group_size; | |
157 | bool found; | |
158 | int cpus_offset, search_cpu, last_cpu, bit_offset, row, end; | |
159 | uint64_t group_cpumask; | |
160 | ||
161 | search_cpu = first_set_cpu(cpumask); | |
162 | if (search_cpu < 0) { | |
163 | log_info("CPU mask doesn't set any CPUs\n"); | |
164 | return 1; | |
165 | } | |
166 | ||
167 | /* Find processor group first set CPU applies to */ | |
168 | online_groups = GetActiveProcessorGroupCount(); | |
169 | group = 0; | |
170 | found = false; | |
171 | cpus_offset = 0; | |
172 | group_size = 0; | |
173 | while (!found && group < online_groups) { | |
174 | group_size = GetActiveProcessorCount(group); | |
175 | dprint(FD_PROCESS, "group=%d group_start=%d group_size=%u search_cpu=%d\n", | |
176 | group, cpus_offset, group_size, search_cpu); | |
177 | if (cpus_offset + group_size > search_cpu) | |
178 | found = true; | |
179 | else { | |
180 | cpus_offset += group_size; | |
181 | group++; | |
182 | } | |
183 | } | |
184 | ||
185 | if (!found) { | |
186 | log_err("CPU mask contains processor beyond last active processor index (%d)\n", | |
187 | cpus_offset - 1); | |
188 | print_mask(cpumask); | |
189 | return 1; | |
190 | } | |
191 | ||
192 | /* Check all the CPUs in the mask apply to ONLY that processor group */ | |
193 | last_cpu = last_set_cpu(cpumask); | |
194 | if (last_cpu > (cpus_offset + group_size - 1)) { | |
195 | log_info("CPU mask cannot bind CPUs (e.g. %d, %d) that are " | |
196 | "in different processor groups\n", search_cpu, | |
197 | last_cpu); | |
198 | print_mask(cpumask); | |
199 | return 1; | |
200 | } | |
201 | ||
202 | /* Extract the current processor group mask from the cpumask */ | |
203 | row = cpus_offset / FIO_CPU_MASK_STRIDE; | |
204 | bit_offset = cpus_offset % FIO_CPU_MASK_STRIDE; | |
205 | group_cpumask = cpumask->row[row] >> bit_offset; | |
206 | end = bit_offset + group_size; | |
207 | if (end > FIO_CPU_MASK_STRIDE && (row + 1 < FIO_CPU_MASK_ROWS)) { | |
208 | /* Some of the next row needs to be part of the mask */ | |
209 | int needed, needed_shift, needed_mask_shift; | |
210 | uint64_t needed_mask; | |
211 | ||
212 | needed = end - FIO_CPU_MASK_STRIDE; | |
213 | needed_shift = FIO_CPU_MASK_STRIDE - bit_offset; | |
214 | needed_mask_shift = FIO_CPU_MASK_STRIDE - needed; | |
215 | needed_mask = (uint64_t)-1 >> needed_mask_shift; | |
f5515176 SW |
216 | dprint(FD_PROCESS, |
217 | "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%" PRIu64 "needed_mask_shift=%d\n", | |
218 | bit_offset, end, needed, needed_shift, needed_mask, | |
219 | needed_mask_shift); | |
a1957954 BVA |
220 | group_cpumask |= (cpumask->row[row + 1] & needed_mask) << needed_shift; |
221 | } | |
222 | group_cpumask &= (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - group_size); | |
223 | ||
224 | /* Return group and mask */ | |
f5515176 SW |
225 | dprint(FD_PROCESS, "Returning group=%d group_mask=%" PRIu64 "\n", |
226 | group, group_cpumask); | |
a1957954 BVA |
227 | *processor_group = group; |
228 | *affinity_mask = group_cpumask; | |
229 | ||
230 | return 0; | |
231 | } | |
232 | ||
233 | int fio_setaffinity(int pid, os_cpu_mask_t cpumask) | |
234 | { | |
235 | HANDLE handle = NULL; | |
236 | int group, ret; | |
237 | uint64_t group_mask = 0; | |
238 | GROUP_AFFINITY new_group_affinity; | |
239 | ||
240 | ret = -1; | |
241 | ||
242 | if (mask_to_group_mask(&cpumask, &group, &group_mask) != 0) | |
243 | goto err; | |
244 | ||
245 | handle = OpenThread(THREAD_QUERY_INFORMATION | THREAD_SET_INFORMATION, | |
246 | TRUE, pid); | |
247 | if (handle == NULL) { | |
248 | log_err("fio_setaffinity: failed to get handle for pid %d\n", pid); | |
249 | goto err; | |
250 | } | |
251 | ||
252 | /* Set group and mask. | |
253 | * Note: if the GROUP_AFFINITY struct's Reserved members are not | |
254 | * initialised to 0 then SetThreadGroupAffinity will fail with | |
255 | * GetLastError() set to ERROR_INVALID_PARAMETER */ | |
256 | new_group_affinity.Mask = (KAFFINITY) group_mask; | |
257 | new_group_affinity.Group = group; | |
258 | new_group_affinity.Reserved[0] = 0; | |
259 | new_group_affinity.Reserved[1] = 0; | |
260 | new_group_affinity.Reserved[2] = 0; | |
261 | if (SetThreadGroupAffinity(handle, &new_group_affinity, NULL) != 0) | |
262 | ret = 0; | |
263 | else { | |
4937100f SW |
264 | log_err("fio_setaffinity: failed to set thread affinity (pid %d, group %d, mask %" PRIx64 ", GetLastError=%lu)\n", |
265 | pid, group, group_mask, GetLastError()); | |
a1957954 BVA |
266 | goto err; |
267 | } | |
268 | ||
269 | err: | |
270 | if (handle) | |
271 | CloseHandle(handle); | |
272 | return ret; | |
273 | } | |
274 | ||
275 | static void cpu_to_row_offset(int cpu, int *row, int *offset) | |
276 | { | |
277 | *row = cpu / FIO_CPU_MASK_STRIDE; | |
278 | *offset = cpu << FIO_CPU_MASK_STRIDE * *row; | |
279 | } | |
280 | ||
281 | int fio_cpuset_init(os_cpu_mask_t *mask) | |
282 | { | |
283 | for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) | |
284 | mask->row[i] = 0; | |
285 | return 0; | |
286 | } | |
287 | ||
288 | /* | |
289 | * fio_getaffinity() should not be called once a fio_setaffinity() call has | |
290 | * been made because fio_setaffinity() may put the process into multiple | |
291 | * processor groups | |
292 | */ | |
293 | int fio_getaffinity(int pid, os_cpu_mask_t *mask) | |
294 | { | |
295 | int ret; | |
296 | int row, offset, end, group, group_size, group_start_cpu; | |
297 | DWORD_PTR process_mask, system_mask; | |
298 | HANDLE handle; | |
299 | PUSHORT current_groups; | |
300 | USHORT group_count; | |
301 | WORD online_groups; | |
302 | ||
303 | ret = -1; | |
304 | current_groups = NULL; | |
305 | handle = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, pid); | |
306 | if (handle == NULL) { | |
307 | log_err("fio_getaffinity: failed to get handle for pid %d\n", | |
308 | pid); | |
309 | goto err; | |
310 | } | |
311 | ||
2f56f0a0 | 312 | group_count = 16; |
a1957954 BVA |
313 | /* |
314 | * GetProcessGroupAffinity() seems to expect more than the natural | |
315 | * alignment for a USHORT from the area pointed to by current_groups so | |
316 | * arrange for maximum alignment by allocating via malloc() | |
317 | */ | |
2f56f0a0 | 318 | current_groups = malloc(group_count * sizeof(USHORT)); |
a1957954 BVA |
319 | if (!current_groups) { |
320 | log_err("fio_getaffinity: malloc failed\n"); | |
321 | goto err; | |
322 | } | |
2f56f0a0 | 323 | if (!GetProcessGroupAffinity(handle, &group_count, current_groups)) { |
4937100f | 324 | log_err("%s: failed to get single group affinity for pid %d (%lu)\n", |
2f56f0a0 BVA |
325 | __func__, pid, GetLastError()); |
326 | goto err; | |
327 | } | |
328 | if (group_count > 1) { | |
329 | log_err("%s: pid %d is associated with %d process groups\n", | |
330 | __func__, pid, group_count); | |
331 | goto err; | |
332 | } | |
333 | if (!GetProcessAffinityMask(handle, &process_mask, &system_mask)) { | |
f5515176 | 334 | log_err("%s: GetProcessAffinityMask() failed for pid %d\n", |
2f56f0a0 | 335 | __func__, pid); |
a1957954 BVA |
336 | goto err; |
337 | } | |
a1957954 BVA |
338 | |
339 | /* Convert group and group relative mask to full CPU mask */ | |
340 | online_groups = GetActiveProcessorGroupCount(); | |
341 | if (online_groups == 0) { | |
342 | log_err("fio_getaffinity: error retrieving total processor groups\n"); | |
343 | goto err; | |
344 | } | |
345 | ||
346 | group = 0; | |
347 | group_start_cpu = 0; | |
348 | group_size = 0; | |
349 | dprint(FD_PROCESS, "current_groups=%d group_count=%d\n", | |
350 | current_groups[0], group_count); | |
351 | while (true) { | |
352 | group_size = GetActiveProcessorCount(group); | |
353 | if (group_size == 0) { | |
354 | log_err("fio_getaffinity: error retrieving size of " | |
355 | "processor group %d\n", group); | |
356 | goto err; | |
357 | } else if (group >= current_groups[0] || group >= online_groups) | |
358 | break; | |
359 | else { | |
360 | group_start_cpu += group_size; | |
361 | group++; | |
362 | } | |
363 | } | |
364 | ||
365 | if (group != current_groups[0]) { | |
366 | log_err("fio_getaffinity: could not find processor group %d\n", | |
367 | current_groups[0]); | |
368 | goto err; | |
369 | } | |
370 | ||
371 | dprint(FD_PROCESS, "group_start_cpu=%d, group size=%u\n", | |
372 | group_start_cpu, group_size); | |
373 | if ((group_start_cpu + group_size) >= FIO_MAX_CPUS) { | |
374 | log_err("fio_getaffinity failed: current CPU affinity (group " | |
375 | "%d, group_start_cpu %d, group_size %d) extends " | |
376 | "beyond mask's highest CPU (%d)\n", group, | |
377 | group_start_cpu, group_size, FIO_MAX_CPUS); | |
378 | goto err; | |
379 | } | |
380 | ||
381 | fio_cpuset_init(mask); | |
382 | cpu_to_row_offset(group_start_cpu, &row, &offset); | |
383 | mask->row[row] = process_mask; | |
384 | mask->row[row] <<= offset; | |
385 | end = offset + group_size; | |
386 | if (end > FIO_CPU_MASK_STRIDE) { | |
387 | int needed; | |
388 | uint64_t needed_mask; | |
389 | ||
390 | needed = FIO_CPU_MASK_STRIDE - end; | |
391 | needed_mask = (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - needed); | |
392 | row++; | |
393 | mask->row[row] = process_mask; | |
394 | mask->row[row] >>= needed; | |
395 | mask->row[row] &= needed_mask; | |
396 | } | |
397 | ret = 0; | |
398 | ||
399 | err: | |
400 | if (handle) | |
401 | CloseHandle(handle); | |
402 | if (current_groups) | |
403 | free(current_groups); | |
404 | ||
405 | return ret; | |
406 | } | |
407 | ||
408 | void fio_cpu_clear(os_cpu_mask_t *mask, int cpu) | |
409 | { | |
410 | int row, offset; | |
411 | cpu_to_row_offset(cpu, &row, &offset); | |
412 | ||
413 | mask->row[row] &= ~(1ULL << offset); | |
414 | } | |
415 | ||
416 | void fio_cpu_set(os_cpu_mask_t *mask, int cpu) | |
417 | { | |
418 | int row, offset; | |
419 | cpu_to_row_offset(cpu, &row, &offset); | |
420 | ||
421 | mask->row[row] |= 1ULL << offset; | |
422 | } | |
423 | ||
424 | int fio_cpu_isset(os_cpu_mask_t *mask, int cpu) | |
425 | { | |
426 | int row, offset; | |
427 | cpu_to_row_offset(cpu, &row, &offset); | |
428 | ||
429 | return (mask->row[row] & (1ULL << offset)) != 0; | |
430 | } | |
431 | ||
432 | int fio_cpu_count(os_cpu_mask_t *mask) | |
433 | { | |
434 | int count = 0; | |
435 | ||
436 | for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) | |
437 | count += hweight64(mask->row[i]); | |
438 | ||
439 | return count; | |
440 | } | |
441 | ||
442 | int fio_cpuset_exit(os_cpu_mask_t *mask) | |
443 | { | |
444 | return 0; | |
445 | } | |
446 | #endif /* CONFIG_WINDOWS_XP */ |