| 1 | #include "os/os.h" |
| 2 | |
| 3 | #include <windows.h> |
| 4 | |
| 5 | #ifdef CONFIG_WINDOWS_XP |
| 6 | int fio_setaffinity(int pid, os_cpu_mask_t cpumask) |
| 7 | { |
| 8 | HANDLE h; |
| 9 | BOOL bSuccess = FALSE; |
| 10 | |
| 11 | h = OpenThread(THREAD_QUERY_INFORMATION | THREAD_SET_INFORMATION, TRUE, |
| 12 | pid); |
| 13 | if (h != NULL) { |
| 14 | bSuccess = SetThreadAffinityMask(h, cpumask); |
| 15 | if (!bSuccess) |
| 16 | log_err("fio_setaffinity failed: failed to set thread affinity (pid %d, mask %.16llx)\n", |
| 17 | pid, cpumask); |
| 18 | |
| 19 | CloseHandle(h); |
| 20 | } else { |
| 21 | log_err("fio_setaffinity failed: failed to get handle for pid %d\n", |
| 22 | pid); |
| 23 | } |
| 24 | |
| 25 | return bSuccess ? 0 : -1; |
| 26 | } |
| 27 | |
| 28 | int fio_getaffinity(int pid, os_cpu_mask_t *mask) |
| 29 | { |
| 30 | os_cpu_mask_t systemMask; |
| 31 | |
| 32 | HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, pid); |
| 33 | |
| 34 | if (h != NULL) { |
| 35 | GetProcessAffinityMask(h, mask, &systemMask); |
| 36 | CloseHandle(h); |
| 37 | } else { |
| 38 | log_err("fio_getaffinity failed: failed to get handle for pid %d\n", |
| 39 | pid); |
| 40 | return -1; |
| 41 | } |
| 42 | |
| 43 | return 0; |
| 44 | } |
| 45 | |
| 46 | void fio_cpu_clear(os_cpu_mask_t *mask, int cpu) |
| 47 | { |
| 48 | *mask &= ~(1ULL << cpu); |
| 49 | } |
| 50 | |
| 51 | void fio_cpu_set(os_cpu_mask_t *mask, int cpu) |
| 52 | { |
| 53 | *mask |= 1ULL << cpu; |
| 54 | } |
| 55 | |
| 56 | int fio_cpu_isset(os_cpu_mask_t *mask, int cpu) |
| 57 | { |
| 58 | return (*mask & (1ULL << cpu)) != 0; |
| 59 | } |
| 60 | |
| 61 | int fio_cpu_count(os_cpu_mask_t *mask) |
| 62 | { |
| 63 | return hweight64(*mask); |
| 64 | } |
| 65 | |
| 66 | int fio_cpuset_init(os_cpu_mask_t *mask) |
| 67 | { |
| 68 | *mask = 0; |
| 69 | return 0; |
| 70 | } |
| 71 | |
| 72 | int fio_cpuset_exit(os_cpu_mask_t *mask) |
| 73 | { |
| 74 | return 0; |
| 75 | } |
| 76 | #else /* CONFIG_WINDOWS_XP */ |
| 77 | /* Return all processors regardless of processor group */ |
| 78 | unsigned int cpus_online(void) |
| 79 | { |
| 80 | return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); |
| 81 | } |
| 82 | |
| 83 | static void print_mask(os_cpu_mask_t *cpumask) |
| 84 | { |
| 85 | for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) |
| 86 | dprint(FD_PROCESS, "cpumask[%d]=%lu\n", i, cpumask->row[i]); |
| 87 | } |
| 88 | |
| 89 | /* Return the index of the least significant set CPU in cpumask or -1 if no |
| 90 | * CPUs are set */ |
| 91 | int first_set_cpu(os_cpu_mask_t *cpumask) |
| 92 | { |
| 93 | int cpus_offset, mask_first_cpu, row; |
| 94 | |
| 95 | cpus_offset = 0; |
| 96 | row = 0; |
| 97 | mask_first_cpu = -1; |
| 98 | while (mask_first_cpu < 0 && row < FIO_CPU_MASK_ROWS) { |
| 99 | int row_first_cpu; |
| 100 | |
| 101 | row_first_cpu = __builtin_ffsll(cpumask->row[row]) - 1; |
| 102 | dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%lu\n", |
| 103 | row_first_cpu, row, cpumask->row[row]); |
| 104 | if (row_first_cpu > -1) { |
| 105 | mask_first_cpu = cpus_offset + row_first_cpu; |
| 106 | dprint(FD_PROCESS, "first set cpu in mask is at index %d\n", |
| 107 | mask_first_cpu); |
| 108 | } else { |
| 109 | cpus_offset += FIO_CPU_MASK_STRIDE; |
| 110 | row++; |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | return mask_first_cpu; |
| 115 | } |
| 116 | |
| 117 | /* Return the index of the most significant set CPU in cpumask or -1 if no |
| 118 | * CPUs are set */ |
| 119 | static int last_set_cpu(os_cpu_mask_t *cpumask) |
| 120 | { |
| 121 | int cpus_offset, mask_last_cpu, row; |
| 122 | |
| 123 | cpus_offset = (FIO_CPU_MASK_ROWS - 1) * FIO_CPU_MASK_STRIDE; |
| 124 | row = FIO_CPU_MASK_ROWS - 1; |
| 125 | mask_last_cpu = -1; |
| 126 | while (mask_last_cpu < 0 && row >= 0) { |
| 127 | int row_last_cpu; |
| 128 | |
| 129 | if (cpumask->row[row] == 0) |
| 130 | row_last_cpu = -1; |
| 131 | else { |
| 132 | uint64_t tmp = cpumask->row[row]; |
| 133 | |
| 134 | row_last_cpu = 0; |
| 135 | while (tmp >>= 1) |
| 136 | row_last_cpu++; |
| 137 | } |
| 138 | |
| 139 | dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%lu\n", |
| 140 | row_last_cpu, row, cpumask->row[row]); |
| 141 | if (row_last_cpu > -1) { |
| 142 | mask_last_cpu = cpus_offset + row_last_cpu; |
| 143 | dprint(FD_PROCESS, "last set cpu in mask is at index %d\n", |
| 144 | mask_last_cpu); |
| 145 | } else { |
| 146 | cpus_offset -= FIO_CPU_MASK_STRIDE; |
| 147 | row--; |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | return mask_last_cpu; |
| 152 | } |
| 153 | |
| 154 | static int mask_to_group_mask(os_cpu_mask_t *cpumask, int *processor_group, uint64_t *affinity_mask) |
| 155 | { |
| 156 | WORD online_groups, group, group_size; |
| 157 | bool found; |
| 158 | int cpus_offset, search_cpu, last_cpu, bit_offset, row, end; |
| 159 | uint64_t group_cpumask; |
| 160 | |
| 161 | search_cpu = first_set_cpu(cpumask); |
| 162 | if (search_cpu < 0) { |
| 163 | log_info("CPU mask doesn't set any CPUs\n"); |
| 164 | return 1; |
| 165 | } |
| 166 | |
| 167 | /* Find processor group first set CPU applies to */ |
| 168 | online_groups = GetActiveProcessorGroupCount(); |
| 169 | group = 0; |
| 170 | found = false; |
| 171 | cpus_offset = 0; |
| 172 | group_size = 0; |
| 173 | while (!found && group < online_groups) { |
| 174 | group_size = GetActiveProcessorCount(group); |
| 175 | dprint(FD_PROCESS, "group=%d group_start=%d group_size=%u search_cpu=%d\n", |
| 176 | group, cpus_offset, group_size, search_cpu); |
| 177 | if (cpus_offset + group_size > search_cpu) |
| 178 | found = true; |
| 179 | else { |
| 180 | cpus_offset += group_size; |
| 181 | group++; |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | if (!found) { |
| 186 | log_err("CPU mask contains processor beyond last active processor index (%d)\n", |
| 187 | cpus_offset - 1); |
| 188 | print_mask(cpumask); |
| 189 | return 1; |
| 190 | } |
| 191 | |
| 192 | /* Check all the CPUs in the mask apply to ONLY that processor group */ |
| 193 | last_cpu = last_set_cpu(cpumask); |
| 194 | if (last_cpu > (cpus_offset + group_size - 1)) { |
| 195 | log_info("CPU mask cannot bind CPUs (e.g. %d, %d) that are " |
| 196 | "in different processor groups\n", search_cpu, |
| 197 | last_cpu); |
| 198 | print_mask(cpumask); |
| 199 | return 1; |
| 200 | } |
| 201 | |
| 202 | /* Extract the current processor group mask from the cpumask */ |
| 203 | row = cpus_offset / FIO_CPU_MASK_STRIDE; |
| 204 | bit_offset = cpus_offset % FIO_CPU_MASK_STRIDE; |
| 205 | group_cpumask = cpumask->row[row] >> bit_offset; |
| 206 | end = bit_offset + group_size; |
| 207 | if (end > FIO_CPU_MASK_STRIDE && (row + 1 < FIO_CPU_MASK_ROWS)) { |
| 208 | /* Some of the next row needs to be part of the mask */ |
| 209 | int needed, needed_shift, needed_mask_shift; |
| 210 | uint64_t needed_mask; |
| 211 | |
| 212 | needed = end - FIO_CPU_MASK_STRIDE; |
| 213 | needed_shift = FIO_CPU_MASK_STRIDE - bit_offset; |
| 214 | needed_mask_shift = FIO_CPU_MASK_STRIDE - needed; |
| 215 | needed_mask = (uint64_t)-1 >> needed_mask_shift; |
| 216 | dprint(FD_PROCESS, "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%ld needed_mask_shift=%d\n", bit_offset, end, needed, needed_shift, needed_mask, needed_mask_shift); |
| 217 | group_cpumask |= (cpumask->row[row + 1] & needed_mask) << needed_shift; |
| 218 | } |
| 219 | group_cpumask &= (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - group_size); |
| 220 | |
| 221 | /* Return group and mask */ |
| 222 | dprint(FD_PROCESS, "Returning group=%d group_mask=%lu\n", group, group_cpumask); |
| 223 | *processor_group = group; |
| 224 | *affinity_mask = group_cpumask; |
| 225 | |
| 226 | return 0; |
| 227 | } |
| 228 | |
| 229 | int fio_setaffinity(int pid, os_cpu_mask_t cpumask) |
| 230 | { |
| 231 | HANDLE handle = NULL; |
| 232 | int group, ret; |
| 233 | uint64_t group_mask = 0; |
| 234 | GROUP_AFFINITY new_group_affinity; |
| 235 | |
| 236 | ret = -1; |
| 237 | |
| 238 | if (mask_to_group_mask(&cpumask, &group, &group_mask) != 0) |
| 239 | goto err; |
| 240 | |
| 241 | handle = OpenThread(THREAD_QUERY_INFORMATION | THREAD_SET_INFORMATION, |
| 242 | TRUE, pid); |
| 243 | if (handle == NULL) { |
| 244 | log_err("fio_setaffinity: failed to get handle for pid %d\n", pid); |
| 245 | goto err; |
| 246 | } |
| 247 | |
| 248 | /* Set group and mask. |
| 249 | * Note: if the GROUP_AFFINITY struct's Reserved members are not |
| 250 | * initialised to 0 then SetThreadGroupAffinity will fail with |
| 251 | * GetLastError() set to ERROR_INVALID_PARAMETER */ |
| 252 | new_group_affinity.Mask = (KAFFINITY) group_mask; |
| 253 | new_group_affinity.Group = group; |
| 254 | new_group_affinity.Reserved[0] = 0; |
| 255 | new_group_affinity.Reserved[1] = 0; |
| 256 | new_group_affinity.Reserved[2] = 0; |
| 257 | if (SetThreadGroupAffinity(handle, &new_group_affinity, NULL) != 0) |
| 258 | ret = 0; |
| 259 | else { |
| 260 | log_err("fio_setaffinity: failed to set thread affinity " |
| 261 | "(pid %d, group %d, mask %" PRIx64 ", " |
| 262 | "GetLastError=%d)\n", pid, group, group_mask, |
| 263 | GetLastError()); |
| 264 | goto err; |
| 265 | } |
| 266 | |
| 267 | err: |
| 268 | if (handle) |
| 269 | CloseHandle(handle); |
| 270 | return ret; |
| 271 | } |
| 272 | |
| 273 | static void cpu_to_row_offset(int cpu, int *row, int *offset) |
| 274 | { |
| 275 | *row = cpu / FIO_CPU_MASK_STRIDE; |
| 276 | *offset = cpu << FIO_CPU_MASK_STRIDE * *row; |
| 277 | } |
| 278 | |
| 279 | int fio_cpuset_init(os_cpu_mask_t *mask) |
| 280 | { |
| 281 | for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) |
| 282 | mask->row[i] = 0; |
| 283 | return 0; |
| 284 | } |
| 285 | |
| 286 | /* |
| 287 | * fio_getaffinity() should not be called once a fio_setaffinity() call has |
| 288 | * been made because fio_setaffinity() may put the process into multiple |
| 289 | * processor groups |
| 290 | */ |
| 291 | int fio_getaffinity(int pid, os_cpu_mask_t *mask) |
| 292 | { |
| 293 | int ret; |
| 294 | int row, offset, end, group, group_size, group_start_cpu; |
| 295 | DWORD_PTR process_mask, system_mask; |
| 296 | HANDLE handle; |
| 297 | PUSHORT current_groups; |
| 298 | USHORT group_count; |
| 299 | WORD online_groups; |
| 300 | |
| 301 | ret = -1; |
| 302 | current_groups = NULL; |
| 303 | handle = OpenProcess(PROCESS_QUERY_INFORMATION, TRUE, pid); |
| 304 | if (handle == NULL) { |
| 305 | log_err("fio_getaffinity: failed to get handle for pid %d\n", |
| 306 | pid); |
| 307 | goto err; |
| 308 | } |
| 309 | |
| 310 | group_count = 1; |
| 311 | /* |
| 312 | * GetProcessGroupAffinity() seems to expect more than the natural |
| 313 | * alignment for a USHORT from the area pointed to by current_groups so |
| 314 | * arrange for maximum alignment by allocating via malloc() |
| 315 | */ |
| 316 | current_groups = malloc(sizeof(USHORT)); |
| 317 | if (!current_groups) { |
| 318 | log_err("fio_getaffinity: malloc failed\n"); |
| 319 | goto err; |
| 320 | } |
| 321 | if (GetProcessGroupAffinity(handle, &group_count, current_groups) == 0) { |
| 322 | /* NB: we also fail here if we are a multi-group process */ |
| 323 | log_err("fio_getaffinity: failed to get single group affinity for pid %d\n", pid); |
| 324 | goto err; |
| 325 | } |
| 326 | GetProcessAffinityMask(handle, &process_mask, &system_mask); |
| 327 | |
| 328 | /* Convert group and group relative mask to full CPU mask */ |
| 329 | online_groups = GetActiveProcessorGroupCount(); |
| 330 | if (online_groups == 0) { |
| 331 | log_err("fio_getaffinity: error retrieving total processor groups\n"); |
| 332 | goto err; |
| 333 | } |
| 334 | |
| 335 | group = 0; |
| 336 | group_start_cpu = 0; |
| 337 | group_size = 0; |
| 338 | dprint(FD_PROCESS, "current_groups=%d group_count=%d\n", |
| 339 | current_groups[0], group_count); |
| 340 | while (true) { |
| 341 | group_size = GetActiveProcessorCount(group); |
| 342 | if (group_size == 0) { |
| 343 | log_err("fio_getaffinity: error retrieving size of " |
| 344 | "processor group %d\n", group); |
| 345 | goto err; |
| 346 | } else if (group >= current_groups[0] || group >= online_groups) |
| 347 | break; |
| 348 | else { |
| 349 | group_start_cpu += group_size; |
| 350 | group++; |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | if (group != current_groups[0]) { |
| 355 | log_err("fio_getaffinity: could not find processor group %d\n", |
| 356 | current_groups[0]); |
| 357 | goto err; |
| 358 | } |
| 359 | |
| 360 | dprint(FD_PROCESS, "group_start_cpu=%d, group size=%u\n", |
| 361 | group_start_cpu, group_size); |
| 362 | if ((group_start_cpu + group_size) >= FIO_MAX_CPUS) { |
| 363 | log_err("fio_getaffinity failed: current CPU affinity (group " |
| 364 | "%d, group_start_cpu %d, group_size %d) extends " |
| 365 | "beyond mask's highest CPU (%d)\n", group, |
| 366 | group_start_cpu, group_size, FIO_MAX_CPUS); |
| 367 | goto err; |
| 368 | } |
| 369 | |
| 370 | fio_cpuset_init(mask); |
| 371 | cpu_to_row_offset(group_start_cpu, &row, &offset); |
| 372 | mask->row[row] = process_mask; |
| 373 | mask->row[row] <<= offset; |
| 374 | end = offset + group_size; |
| 375 | if (end > FIO_CPU_MASK_STRIDE) { |
| 376 | int needed; |
| 377 | uint64_t needed_mask; |
| 378 | |
| 379 | needed = FIO_CPU_MASK_STRIDE - end; |
| 380 | needed_mask = (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - needed); |
| 381 | row++; |
| 382 | mask->row[row] = process_mask; |
| 383 | mask->row[row] >>= needed; |
| 384 | mask->row[row] &= needed_mask; |
| 385 | } |
| 386 | ret = 0; |
| 387 | |
| 388 | err: |
| 389 | if (handle) |
| 390 | CloseHandle(handle); |
| 391 | if (current_groups) |
| 392 | free(current_groups); |
| 393 | |
| 394 | return ret; |
| 395 | } |
| 396 | |
| 397 | void fio_cpu_clear(os_cpu_mask_t *mask, int cpu) |
| 398 | { |
| 399 | int row, offset; |
| 400 | cpu_to_row_offset(cpu, &row, &offset); |
| 401 | |
| 402 | mask->row[row] &= ~(1ULL << offset); |
| 403 | } |
| 404 | |
| 405 | void fio_cpu_set(os_cpu_mask_t *mask, int cpu) |
| 406 | { |
| 407 | int row, offset; |
| 408 | cpu_to_row_offset(cpu, &row, &offset); |
| 409 | |
| 410 | mask->row[row] |= 1ULL << offset; |
| 411 | } |
| 412 | |
| 413 | int fio_cpu_isset(os_cpu_mask_t *mask, int cpu) |
| 414 | { |
| 415 | int row, offset; |
| 416 | cpu_to_row_offset(cpu, &row, &offset); |
| 417 | |
| 418 | return (mask->row[row] & (1ULL << offset)) != 0; |
| 419 | } |
| 420 | |
| 421 | int fio_cpu_count(os_cpu_mask_t *mask) |
| 422 | { |
| 423 | int count = 0; |
| 424 | |
| 425 | for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) |
| 426 | count += hweight64(mask->row[i]); |
| 427 | |
| 428 | return count; |
| 429 | } |
| 430 | |
| 431 | int fio_cpuset_exit(os_cpu_mask_t *mask) |
| 432 | { |
| 433 | return 0; |
| 434 | } |
| 435 | #endif /* CONFIG_WINDOWS_XP */ |