Commit | Line | Data |
---|---|---|
75ccbef6 BT |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright(c) 2019 Intel Corporation. */ | |
3 | ||
4 | #include <linux/hash.h> | |
5 | #include <linux/bpf.h> | |
6 | #include <linux/filter.h> | |
7 | ||
8 | /* The BPF dispatcher is a multiway branch code generator. The | |
9 | * dispatcher is a mechanism to avoid the performance penalty of an | |
10 | * indirect call, which is expensive when retpolines are enabled. A | |
11 | * dispatch client registers a BPF program into the dispatcher, and if | |
12 | * there is available room in the dispatcher a direct call to the BPF | |
13 | * program will be generated. All calls to the BPF programs called via | |
14 | * the dispatcher will then be a direct call, instead of an | |
15 | * indirect. The dispatcher hijacks a trampoline function it via the | |
16 | * __fentry__ of the trampoline. The trampoline function has the | |
17 | * following signature: | |
18 | * | |
19 | * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi, | |
20 | * unsigned int (*bpf_func)(const void *, | |
21 | * const struct bpf_insn *)); | |
22 | */ | |
23 | ||
24 | static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog( | |
25 | struct bpf_dispatcher *d, struct bpf_prog *prog) | |
26 | { | |
27 | int i; | |
28 | ||
29 | for (i = 0; i < BPF_DISPATCHER_MAX; i++) { | |
30 | if (prog == d->progs[i].prog) | |
31 | return &d->progs[i]; | |
32 | } | |
33 | return NULL; | |
34 | } | |
35 | ||
36 | static struct bpf_dispatcher_prog *bpf_dispatcher_find_free( | |
37 | struct bpf_dispatcher *d) | |
38 | { | |
39 | return bpf_dispatcher_find_prog(d, NULL); | |
40 | } | |
41 | ||
42 | static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d, | |
43 | struct bpf_prog *prog) | |
44 | { | |
45 | struct bpf_dispatcher_prog *entry; | |
46 | ||
47 | if (!prog) | |
48 | return false; | |
49 | ||
50 | entry = bpf_dispatcher_find_prog(d, prog); | |
51 | if (entry) { | |
52 | refcount_inc(&entry->users); | |
53 | return false; | |
54 | } | |
55 | ||
56 | entry = bpf_dispatcher_find_free(d); | |
57 | if (!entry) | |
58 | return false; | |
59 | ||
60 | bpf_prog_inc(prog); | |
61 | entry->prog = prog; | |
62 | refcount_set(&entry->users, 1); | |
63 | d->num_progs++; | |
64 | return true; | |
65 | } | |
66 | ||
67 | static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d, | |
68 | struct bpf_prog *prog) | |
69 | { | |
70 | struct bpf_dispatcher_prog *entry; | |
71 | ||
72 | if (!prog) | |
73 | return false; | |
74 | ||
75 | entry = bpf_dispatcher_find_prog(d, prog); | |
76 | if (!entry) | |
77 | return false; | |
78 | ||
79 | if (refcount_dec_and_test(&entry->users)) { | |
80 | entry->prog = NULL; | |
81 | bpf_prog_put(prog); | |
82 | d->num_progs--; | |
83 | return true; | |
84 | } | |
85 | return false; | |
86 | } | |
87 | ||
88 | int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs) | |
89 | { | |
90 | return -ENOTSUPP; | |
91 | } | |
92 | ||
93 | static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image) | |
94 | { | |
95 | s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; | |
96 | int i; | |
97 | ||
98 | for (i = 0; i < BPF_DISPATCHER_MAX; i++) { | |
99 | if (d->progs[i].prog) | |
100 | *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func; | |
101 | } | |
102 | return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs); | |
103 | } | |
104 | ||
105 | static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) | |
106 | { | |
107 | void *old, *new; | |
108 | u32 noff; | |
109 | int err; | |
110 | ||
111 | if (!prev_num_progs) { | |
112 | old = NULL; | |
113 | noff = 0; | |
114 | } else { | |
115 | old = d->image + d->image_off; | |
7ac88eba | 116 | noff = d->image_off ^ (PAGE_SIZE / 2); |
75ccbef6 BT |
117 | } |
118 | ||
119 | new = d->num_progs ? d->image + noff : NULL; | |
120 | if (new) { | |
121 | if (bpf_dispatcher_prepare(d, new)) | |
122 | return; | |
123 | } | |
124 | ||
125 | err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new); | |
126 | if (err || !new) | |
127 | return; | |
128 | ||
129 | d->image_off = noff; | |
130 | } | |
131 | ||
132 | void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, | |
133 | struct bpf_prog *to) | |
134 | { | |
135 | bool changed = false; | |
136 | int prev_num_progs; | |
137 | ||
138 | if (from == to) | |
139 | return; | |
140 | ||
141 | mutex_lock(&d->mutex); | |
142 | if (!d->image) { | |
7ac88eba | 143 | d->image = bpf_jit_alloc_exec_page(); |
75ccbef6 BT |
144 | if (!d->image) |
145 | goto out; | |
517b75e4 | 146 | bpf_image_ksym_add(d->image, &d->ksym); |
75ccbef6 BT |
147 | } |
148 | ||
149 | prev_num_progs = d->num_progs; | |
150 | changed |= bpf_dispatcher_remove_prog(d, from); | |
151 | changed |= bpf_dispatcher_add_prog(d, to); | |
152 | ||
153 | if (!changed) | |
154 | goto out; | |
155 | ||
156 | bpf_dispatcher_update(d, prev_num_progs); | |
157 | out: | |
158 | mutex_unlock(&d->mutex); | |
159 | } |