projects / linux-block.git / blame
| Commit | Line | Data |
|---|---|---|
| 267ed5d8 AK |
1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* Manage affinity to optimize IPIs inside the kernel perf API. */ | |
| 3 | #define _GNU_SOURCE 1 | |
| 4 | #include <sched.h> | |
| 5 | #include <stdlib.h> | |
| 6 | #include <linux/bitmap.h> | |
| 7 | #include <linux/zalloc.h> | |
| 8 | #include "perf.h" | |
| 9 | #include "cpumap.h" | |
| 10 | #include "affinity.h" | |
| 11 | ||
| 12 | static int get_cpu_set_size(void) | |
| 13 | { | |
| 6d18804b | 14 | int sz = cpu__max_cpu().cpu + 8 - 1; |
| 267ed5d8 AK |
15 | /* |
| 16 | * sched_getaffinity doesn't like masks smaller than the kernel. | |
| 17 | * Hopefully that's big enough. | |
| 18 | */ | |
| 19 | if (sz < 4096) | |
| 20 | sz = 4096; | |
| 21 | return sz / 8; | |
| 22 | } | |
| 23 | ||
| 24 | int affinity__setup(struct affinity *a) | |
| 25 | { | |
| 26 | int cpu_set_size = get_cpu_set_size(); | |
| 27 | ||
| 7fc5b571 | 28 | a->orig_cpus = bitmap_zalloc(cpu_set_size * 8); |
| 267ed5d8 AK |
29 | if (!a->orig_cpus) |
| 30 | return -1; | |
| 31 | sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); | |
| 7fc5b571 | 32 | a->sched_cpus = bitmap_zalloc(cpu_set_size * 8); |
| 267ed5d8 AK |
33 | if (!a->sched_cpus) { |
| 34 | zfree(&a->orig_cpus); | |
| 35 | return -1; | |
| 36 | } | |
| 37 | bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size); | |
| 38 | a->changed = false; | |
| 39 | return 0; | |
| 40 | } | |
| 41 | ||
| 42 | /* | |
| 43 | * perf_event_open does an IPI internally to the target CPU. | |
| 44 | * It is more efficient to change perf's affinity to the target | |
| 45 | * CPU and then set up all events on that CPU, so we amortize | |
| 46 | * CPU communication. | |
| 47 | */ | |
| 48 | void affinity__set(struct affinity *a, int cpu) | |
| 49 | { | |
| 50 | int cpu_set_size = get_cpu_set_size(); | |
| 51 | ||
| 72cd652b AR |
52 | /* |
| 53 | * Return: | |
| 54 | * - if cpu is -1 | |
| 55 | * - restrict out of bound access to sched_cpus | |
| 56 | */ | |
| 57 | if (cpu == -1 || ((cpu >= (cpu_set_size * 8)))) | |
| 267ed5d8 | 58 | return; |
| 72cd652b | 59 | |
| 267ed5d8 | 60 | a->changed = true; |
| 75d7ba32 | 61 | __set_bit(cpu, a->sched_cpus); |
| 267ed5d8 AK |
62 | /* |
| 63 | * We ignore errors because affinity is just an optimization. | |
| 64 | * This could happen for example with isolated CPUs or cpusets. | |
| 65 | * In this case the IPIs inside the kernel's perf API still work. | |
| 66 | */ | |
| 67 | sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus); | |
| 75d7ba32 | 68 | __clear_bit(cpu, a->sched_cpus); |
| 267ed5d8 AK |
69 | } |
| 70 | ||
| 1855b796 | 71 | static void __affinity__cleanup(struct affinity *a) |
| 267ed5d8 AK |
72 | { |
| 73 | int cpu_set_size = get_cpu_set_size(); | |
| 74 | ||
| 75 | if (a->changed) | |
| 76 | sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); | |
| 77 | zfree(&a->sched_cpus); | |
| 78 | zfree(&a->orig_cpus); | |
| 79 | } | |
| 1855b796 ACM |
80 | |
| 81 | void affinity__cleanup(struct affinity *a) | |
| 82 | { | |
| 83 | if (a != NULL) | |
| 84 | __affinity__cleanup(a); | |
| 85 | } |