Commit | Line | Data |
---|---|---|
5dc61552 TY |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * BPF JIT compiler for LoongArch | |
4 | * | |
5 | * Copyright (C) 2022 Loongson Technology Corporation Limited | |
6 | */ | |
7 | #include "bpf_jit.h" | |
8 | ||
9 | #define REG_TCC LOONGARCH_GPR_A6 | |
10 | #define TCC_SAVED LOONGARCH_GPR_S5 | |
11 | ||
12 | #define SAVE_RA BIT(0) | |
13 | #define SAVE_TCC BIT(1) | |
14 | ||
15 | static const int regmap[] = { | |
16 | /* return value from in-kernel function, and exit value for eBPF program */ | |
17 | [BPF_REG_0] = LOONGARCH_GPR_A5, | |
18 | /* arguments from eBPF program to in-kernel function */ | |
19 | [BPF_REG_1] = LOONGARCH_GPR_A0, | |
20 | [BPF_REG_2] = LOONGARCH_GPR_A1, | |
21 | [BPF_REG_3] = LOONGARCH_GPR_A2, | |
22 | [BPF_REG_4] = LOONGARCH_GPR_A3, | |
23 | [BPF_REG_5] = LOONGARCH_GPR_A4, | |
24 | /* callee saved registers that in-kernel function will preserve */ | |
25 | [BPF_REG_6] = LOONGARCH_GPR_S0, | |
26 | [BPF_REG_7] = LOONGARCH_GPR_S1, | |
27 | [BPF_REG_8] = LOONGARCH_GPR_S2, | |
28 | [BPF_REG_9] = LOONGARCH_GPR_S3, | |
29 | /* read-only frame pointer to access stack */ | |
30 | [BPF_REG_FP] = LOONGARCH_GPR_S4, | |
31 | /* temporary register for blinding constants */ | |
32 | [BPF_REG_AX] = LOONGARCH_GPR_T0, | |
33 | }; | |
34 | ||
35 | static void mark_call(struct jit_ctx *ctx) | |
36 | { | |
37 | ctx->flags |= SAVE_RA; | |
38 | } | |
39 | ||
40 | static void mark_tail_call(struct jit_ctx *ctx) | |
41 | { | |
42 | ctx->flags |= SAVE_TCC; | |
43 | } | |
44 | ||
45 | static bool seen_call(struct jit_ctx *ctx) | |
46 | { | |
47 | return (ctx->flags & SAVE_RA); | |
48 | } | |
49 | ||
50 | static bool seen_tail_call(struct jit_ctx *ctx) | |
51 | { | |
52 | return (ctx->flags & SAVE_TCC); | |
53 | } | |
54 | ||
55 | static u8 tail_call_reg(struct jit_ctx *ctx) | |
56 | { | |
57 | if (seen_call(ctx)) | |
58 | return TCC_SAVED; | |
59 | ||
60 | return REG_TCC; | |
61 | } | |
62 | ||
63 | /* | |
64 | * eBPF prog stack layout: | |
65 | * | |
66 | * high | |
67 | * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP | |
68 | * | $ra | | |
69 | * +-------------------------+ | |
70 | * | $fp | | |
71 | * +-------------------------+ | |
72 | * | $s0 | | |
73 | * +-------------------------+ | |
74 | * | $s1 | | |
75 | * +-------------------------+ | |
76 | * | $s2 | | |
77 | * +-------------------------+ | |
78 | * | $s3 | | |
79 | * +-------------------------+ | |
80 | * | $s4 | | |
81 | * +-------------------------+ | |
82 | * | $s5 | | |
83 | * +-------------------------+ <--BPF_REG_FP | |
84 | * | prog->aux->stack_depth | | |
85 | * | (optional) | | |
86 | * current $sp -------------> +-------------------------+ | |
87 | * low | |
88 | */ | |
89 | static void build_prologue(struct jit_ctx *ctx) | |
90 | { | |
91 | int stack_adjust = 0, store_offset, bpf_stack_adjust; | |
92 | ||
93 | bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); | |
94 | ||
95 | /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */ | |
96 | stack_adjust += sizeof(long) * 8; | |
97 | ||
98 | stack_adjust = round_up(stack_adjust, 16); | |
99 | stack_adjust += bpf_stack_adjust; | |
100 | ||
101 | /* | |
102 | * First instruction initializes the tail call count (TCC). | |
103 | * On tail call we skip this instruction, and the TCC is | |
104 | * passed in REG_TCC from the caller. | |
105 | */ | |
106 | emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); | |
107 | ||
108 | emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); | |
109 | ||
110 | store_offset = stack_adjust - sizeof(long); | |
111 | emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); | |
112 | ||
113 | store_offset -= sizeof(long); | |
114 | emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); | |
115 | ||
116 | store_offset -= sizeof(long); | |
117 | emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); | |
118 | ||
119 | store_offset -= sizeof(long); | |
120 | emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); | |
121 | ||
122 | store_offset -= sizeof(long); | |
123 | emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); | |
124 | ||
125 | store_offset -= sizeof(long); | |
126 | emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); | |
127 | ||
128 | store_offset -= sizeof(long); | |
129 | emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); | |
130 | ||
131 | store_offset -= sizeof(long); | |
132 | emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); | |
133 | ||
134 | emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); | |
135 | ||
136 | if (bpf_stack_adjust) | |
137 | emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); | |
138 | ||
139 | /* | |
140 | * Program contains calls and tail calls, so REG_TCC need | |
141 | * to be saved across calls. | |
142 | */ | |
143 | if (seen_tail_call(ctx) && seen_call(ctx)) | |
144 | move_reg(ctx, TCC_SAVED, REG_TCC); | |
145 | ||
146 | ctx->stack_size = stack_adjust; | |
147 | } | |
148 | ||
149 | static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) | |
150 | { | |
151 | int stack_adjust = ctx->stack_size; | |
152 | int load_offset; | |
153 | ||
154 | load_offset = stack_adjust - sizeof(long); | |
155 | emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); | |
156 | ||
157 | load_offset -= sizeof(long); | |
158 | emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); | |
159 | ||
160 | load_offset -= sizeof(long); | |
161 | emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); | |
162 | ||
163 | load_offset -= sizeof(long); | |
164 | emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); | |
165 | ||
166 | load_offset -= sizeof(long); | |
167 | emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); | |
168 | ||
169 | load_offset -= sizeof(long); | |
170 | emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); | |
171 | ||
172 | load_offset -= sizeof(long); | |
173 | emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); | |
174 | ||
175 | load_offset -= sizeof(long); | |
176 | emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); | |
177 | ||
178 | emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); | |
179 | ||
180 | if (!is_tail_call) { | |
181 | /* Set return value */ | |
182 | move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]); | |
183 | /* Return to the caller */ | |
184 | emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0); | |
185 | } else { | |
186 | /* | |
187 | * Call the next bpf prog and skip the first instruction | |
188 | * of TCC initialization. | |
189 | */ | |
190 | emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1); | |
191 | } | |
192 | } | |
193 | ||
194 | static void build_epilogue(struct jit_ctx *ctx) | |
195 | { | |
196 | __build_epilogue(ctx, false); | |
197 | } | |
198 | ||
199 | bool bpf_jit_supports_kfunc_call(void) | |
200 | { | |
201 | return true; | |
202 | } | |
203 | ||
21c5ae5c HC |
204 | bool bpf_jit_supports_far_kfunc_call(void) |
205 | { | |
206 | return true; | |
207 | } | |
208 | ||
5dc61552 TY |
209 | /* initialized on the first pass of build_body() */ |
210 | static int out_offset = -1; | |
211 | static int emit_bpf_tail_call(struct jit_ctx *ctx) | |
212 | { | |
213 | int off; | |
214 | u8 tcc = tail_call_reg(ctx); | |
215 | u8 a1 = LOONGARCH_GPR_A1; | |
216 | u8 a2 = LOONGARCH_GPR_A2; | |
217 | u8 t1 = LOONGARCH_GPR_T1; | |
218 | u8 t2 = LOONGARCH_GPR_T2; | |
219 | u8 t3 = LOONGARCH_GPR_T3; | |
220 | const int idx0 = ctx->idx; | |
221 | ||
222 | #define cur_offset (ctx->idx - idx0) | |
223 | #define jmp_offset (out_offset - (cur_offset)) | |
224 | ||
225 | /* | |
226 | * a0: &ctx | |
227 | * a1: &array | |
228 | * a2: index | |
229 | * | |
230 | * if (index >= array->map.max_entries) | |
231 | * goto out; | |
232 | */ | |
233 | off = offsetof(struct bpf_array, map.max_entries); | |
234 | emit_insn(ctx, ldwu, t1, a1, off); | |
235 | /* bgeu $a2, $t1, jmp_offset */ | |
236 | if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) | |
237 | goto toofar; | |
238 | ||
239 | /* | |
240 | * if (--TCC < 0) | |
241 | * goto out; | |
242 | */ | |
243 | emit_insn(ctx, addid, REG_TCC, tcc, -1); | |
244 | if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0) | |
245 | goto toofar; | |
246 | ||
247 | /* | |
248 | * prog = array->ptrs[index]; | |
249 | * if (!prog) | |
250 | * goto out; | |
251 | */ | |
252 | emit_insn(ctx, alsld, t2, a2, a1, 2); | |
253 | off = offsetof(struct bpf_array, ptrs); | |
254 | emit_insn(ctx, ldd, t2, t2, off); | |
255 | /* beq $t2, $zero, jmp_offset */ | |
256 | if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) | |
257 | goto toofar; | |
258 | ||
259 | /* goto *(prog->bpf_func + 4); */ | |
260 | off = offsetof(struct bpf_prog, bpf_func); | |
261 | emit_insn(ctx, ldd, t3, t2, off); | |
262 | __build_epilogue(ctx, true); | |
263 | ||
264 | /* out: */ | |
265 | if (out_offset == -1) | |
266 | out_offset = cur_offset; | |
267 | if (cur_offset != out_offset) { | |
268 | pr_err_once("tail_call out_offset = %d, expected %d!\n", | |
269 | cur_offset, out_offset); | |
270 | return -1; | |
271 | } | |
272 | ||
273 | return 0; | |
274 | ||
275 | toofar: | |
276 | pr_info_once("tail_call: jump too far\n"); | |
277 | return -1; | |
278 | #undef cur_offset | |
279 | #undef jmp_offset | |
280 | } | |
281 | ||
282 | static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) | |
283 | { | |
284 | const u8 t1 = LOONGARCH_GPR_T1; | |
285 | const u8 t2 = LOONGARCH_GPR_T2; | |
286 | const u8 t3 = LOONGARCH_GPR_T3; | |
bbfddb90 | 287 | const u8 r0 = regmap[BPF_REG_0]; |
5dc61552 TY |
288 | const u8 src = regmap[insn->src_reg]; |
289 | const u8 dst = regmap[insn->dst_reg]; | |
290 | const s16 off = insn->off; | |
291 | const s32 imm = insn->imm; | |
292 | const bool isdw = BPF_SIZE(insn->code) == BPF_DW; | |
293 | ||
294 | move_imm(ctx, t1, off, false); | |
295 | emit_insn(ctx, addd, t1, dst, t1); | |
296 | move_reg(ctx, t3, src); | |
297 | ||
298 | switch (imm) { | |
299 | /* lock *(size *)(dst + off) <op>= src */ | |
300 | case BPF_ADD: | |
301 | if (isdw) | |
302 | emit_insn(ctx, amaddd, t2, t1, src); | |
303 | else | |
304 | emit_insn(ctx, amaddw, t2, t1, src); | |
305 | break; | |
306 | case BPF_AND: | |
307 | if (isdw) | |
308 | emit_insn(ctx, amandd, t2, t1, src); | |
309 | else | |
310 | emit_insn(ctx, amandw, t2, t1, src); | |
311 | break; | |
312 | case BPF_OR: | |
313 | if (isdw) | |
314 | emit_insn(ctx, amord, t2, t1, src); | |
315 | else | |
316 | emit_insn(ctx, amorw, t2, t1, src); | |
317 | break; | |
318 | case BPF_XOR: | |
319 | if (isdw) | |
320 | emit_insn(ctx, amxord, t2, t1, src); | |
321 | else | |
322 | emit_insn(ctx, amxorw, t2, t1, src); | |
323 | break; | |
324 | /* src = atomic_fetch_<op>(dst + off, src) */ | |
325 | case BPF_ADD | BPF_FETCH: | |
326 | if (isdw) { | |
327 | emit_insn(ctx, amaddd, src, t1, t3); | |
328 | } else { | |
329 | emit_insn(ctx, amaddw, src, t1, t3); | |
330 | emit_zext_32(ctx, src, true); | |
331 | } | |
332 | break; | |
333 | case BPF_AND | BPF_FETCH: | |
334 | if (isdw) { | |
335 | emit_insn(ctx, amandd, src, t1, t3); | |
336 | } else { | |
337 | emit_insn(ctx, amandw, src, t1, t3); | |
338 | emit_zext_32(ctx, src, true); | |
339 | } | |
340 | break; | |
341 | case BPF_OR | BPF_FETCH: | |
342 | if (isdw) { | |
343 | emit_insn(ctx, amord, src, t1, t3); | |
344 | } else { | |
345 | emit_insn(ctx, amorw, src, t1, t3); | |
346 | emit_zext_32(ctx, src, true); | |
347 | } | |
348 | break; | |
349 | case BPF_XOR | BPF_FETCH: | |
350 | if (isdw) { | |
351 | emit_insn(ctx, amxord, src, t1, t3); | |
352 | } else { | |
353 | emit_insn(ctx, amxorw, src, t1, t3); | |
354 | emit_zext_32(ctx, src, true); | |
355 | } | |
356 | break; | |
357 | /* src = atomic_xchg(dst + off, src); */ | |
358 | case BPF_XCHG: | |
359 | if (isdw) { | |
360 | emit_insn(ctx, amswapd, src, t1, t3); | |
361 | } else { | |
362 | emit_insn(ctx, amswapw, src, t1, t3); | |
363 | emit_zext_32(ctx, src, true); | |
364 | } | |
365 | break; | |
366 | /* r0 = atomic_cmpxchg(dst + off, r0, src); */ | |
367 | case BPF_CMPXCHG: | |
5dc61552 TY |
368 | move_reg(ctx, t2, r0); |
369 | if (isdw) { | |
370 | emit_insn(ctx, lld, r0, t1, 0); | |
371 | emit_insn(ctx, bne, t2, r0, 4); | |
372 | move_reg(ctx, t3, src); | |
373 | emit_insn(ctx, scd, t3, t1, 0); | |
374 | emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); | |
375 | } else { | |
376 | emit_insn(ctx, llw, r0, t1, 0); | |
377 | emit_zext_32(ctx, t2, true); | |
378 | emit_zext_32(ctx, r0, true); | |
379 | emit_insn(ctx, bne, t2, r0, 4); | |
380 | move_reg(ctx, t3, src); | |
381 | emit_insn(ctx, scw, t3, t1, 0); | |
382 | emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); | |
383 | emit_zext_32(ctx, r0, true); | |
384 | } | |
385 | break; | |
386 | } | |
387 | } | |
388 | ||
389 | static bool is_signed_bpf_cond(u8 cond) | |
390 | { | |
391 | return cond == BPF_JSGT || cond == BPF_JSLT || | |
392 | cond == BPF_JSGE || cond == BPF_JSLE; | |
393 | } | |
394 | ||
dbcd7f5f YT |
395 | #define BPF_FIXUP_REG_MASK GENMASK(31, 27) |
396 | #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) | |
397 | ||
398 | bool ex_handler_bpf(const struct exception_table_entry *ex, | |
399 | struct pt_regs *regs) | |
400 | { | |
401 | int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); | |
402 | off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); | |
403 | ||
404 | regs->regs[dst_reg] = 0; | |
405 | regs->csr_era = (unsigned long)&ex->fixup - offset; | |
406 | ||
407 | return true; | |
408 | } | |
409 | ||
410 | /* For accesses to BTF pointers, add an entry to the exception table */ | |
411 | static int add_exception_handler(const struct bpf_insn *insn, | |
412 | struct jit_ctx *ctx, | |
413 | int dst_reg) | |
414 | { | |
415 | unsigned long pc; | |
416 | off_t offset; | |
417 | struct exception_table_entry *ex; | |
418 | ||
7111afe8 HC |
419 | if (!ctx->image || !ctx->prog->aux->extable) |
420 | return 0; | |
421 | ||
422 | if (BPF_MODE(insn->code) != BPF_PROBE_MEM && | |
423 | BPF_MODE(insn->code) != BPF_PROBE_MEMSX) | |
dbcd7f5f YT |
424 | return 0; |
425 | ||
426 | if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) | |
427 | return -EINVAL; | |
428 | ||
429 | ex = &ctx->prog->aux->extable[ctx->num_exentries]; | |
430 | pc = (unsigned long)&ctx->image[ctx->idx - 1]; | |
431 | ||
432 | offset = pc - (long)&ex->insn; | |
433 | if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) | |
434 | return -ERANGE; | |
435 | ||
436 | ex->insn = offset; | |
437 | ||
438 | /* | |
439 | * Since the extable follows the program, the fixup offset is always | |
440 | * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value | |
441 | * to keep things simple, and put the destination register in the upper | |
442 | * bits. We don't need to worry about buildtime or runtime sort | |
443 | * modifying the upper bits because the table is already sorted, and | |
444 | * isn't part of the main exception table. | |
445 | */ | |
446 | offset = (long)&ex->fixup - (pc + LOONGARCH_INSN_SIZE); | |
447 | if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) | |
448 | return -ERANGE; | |
449 | ||
450 | ex->type = EX_TYPE_BPF; | |
451 | ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); | |
452 | ||
453 | ctx->num_exentries++; | |
454 | ||
455 | return 0; | |
456 | } | |
457 | ||
5dc61552 TY |
458 | static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) |
459 | { | |
bbfddb90 HC |
460 | u8 tm = -1; |
461 | u64 func_addr; | |
7111afe8 | 462 | bool func_addr_fixed, sign_extend; |
bbfddb90 HC |
463 | int i = insn - ctx->prog->insnsi; |
464 | int ret, jmp_offset; | |
5dc61552 TY |
465 | const u8 code = insn->code; |
466 | const u8 cond = BPF_OP(code); | |
467 | const u8 t1 = LOONGARCH_GPR_T1; | |
468 | const u8 t2 = LOONGARCH_GPR_T2; | |
469 | const u8 src = regmap[insn->src_reg]; | |
470 | const u8 dst = regmap[insn->dst_reg]; | |
471 | const s16 off = insn->off; | |
472 | const s32 imm = insn->imm; | |
bbfddb90 | 473 | const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; |
5dc61552 TY |
474 | |
475 | switch (code) { | |
476 | /* dst = src */ | |
477 | case BPF_ALU | BPF_MOV | BPF_X: | |
478 | case BPF_ALU64 | BPF_MOV | BPF_X: | |
f48012f1 HC |
479 | switch (off) { |
480 | case 0: | |
481 | move_reg(ctx, dst, src); | |
482 | emit_zext_32(ctx, dst, is32); | |
483 | break; | |
484 | case 8: | |
485 | move_reg(ctx, t1, src); | |
486 | emit_insn(ctx, extwb, dst, t1); | |
772cbe94 | 487 | emit_zext_32(ctx, dst, is32); |
f48012f1 HC |
488 | break; |
489 | case 16: | |
490 | move_reg(ctx, t1, src); | |
491 | emit_insn(ctx, extwh, dst, t1); | |
772cbe94 | 492 | emit_zext_32(ctx, dst, is32); |
f48012f1 HC |
493 | break; |
494 | case 32: | |
495 | emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); | |
496 | break; | |
497 | } | |
5dc61552 TY |
498 | break; |
499 | ||
500 | /* dst = imm */ | |
501 | case BPF_ALU | BPF_MOV | BPF_K: | |
502 | case BPF_ALU64 | BPF_MOV | BPF_K: | |
503 | move_imm(ctx, dst, imm, is32); | |
504 | break; | |
505 | ||
506 | /* dst = dst + src */ | |
507 | case BPF_ALU | BPF_ADD | BPF_X: | |
508 | case BPF_ALU64 | BPF_ADD | BPF_X: | |
509 | emit_insn(ctx, addd, dst, dst, src); | |
510 | emit_zext_32(ctx, dst, is32); | |
511 | break; | |
512 | ||
513 | /* dst = dst + imm */ | |
514 | case BPF_ALU | BPF_ADD | BPF_K: | |
515 | case BPF_ALU64 | BPF_ADD | BPF_K: | |
516 | if (is_signed_imm12(imm)) { | |
517 | emit_insn(ctx, addid, dst, dst, imm); | |
518 | } else { | |
519 | move_imm(ctx, t1, imm, is32); | |
520 | emit_insn(ctx, addd, dst, dst, t1); | |
521 | } | |
522 | emit_zext_32(ctx, dst, is32); | |
523 | break; | |
524 | ||
525 | /* dst = dst - src */ | |
526 | case BPF_ALU | BPF_SUB | BPF_X: | |
527 | case BPF_ALU64 | BPF_SUB | BPF_X: | |
528 | emit_insn(ctx, subd, dst, dst, src); | |
529 | emit_zext_32(ctx, dst, is32); | |
530 | break; | |
531 | ||
532 | /* dst = dst - imm */ | |
533 | case BPF_ALU | BPF_SUB | BPF_K: | |
534 | case BPF_ALU64 | BPF_SUB | BPF_K: | |
535 | if (is_signed_imm12(-imm)) { | |
536 | emit_insn(ctx, addid, dst, dst, -imm); | |
537 | } else { | |
538 | move_imm(ctx, t1, imm, is32); | |
539 | emit_insn(ctx, subd, dst, dst, t1); | |
540 | } | |
541 | emit_zext_32(ctx, dst, is32); | |
542 | break; | |
543 | ||
544 | /* dst = dst * src */ | |
545 | case BPF_ALU | BPF_MUL | BPF_X: | |
546 | case BPF_ALU64 | BPF_MUL | BPF_X: | |
547 | emit_insn(ctx, muld, dst, dst, src); | |
548 | emit_zext_32(ctx, dst, is32); | |
549 | break; | |
550 | ||
551 | /* dst = dst * imm */ | |
552 | case BPF_ALU | BPF_MUL | BPF_K: | |
553 | case BPF_ALU64 | BPF_MUL | BPF_K: | |
554 | move_imm(ctx, t1, imm, is32); | |
555 | emit_insn(ctx, muld, dst, dst, t1); | |
556 | emit_zext_32(ctx, dst, is32); | |
557 | break; | |
558 | ||
559 | /* dst = dst / src */ | |
560 | case BPF_ALU | BPF_DIV | BPF_X: | |
561 | case BPF_ALU64 | BPF_DIV | BPF_X: | |
2425c9e0 HC |
562 | if (!off) { |
563 | emit_zext_32(ctx, dst, is32); | |
564 | move_reg(ctx, t1, src); | |
565 | emit_zext_32(ctx, t1, is32); | |
566 | emit_insn(ctx, divdu, dst, dst, t1); | |
567 | emit_zext_32(ctx, dst, is32); | |
568 | } else { | |
569 | emit_sext_32(ctx, dst, is32); | |
570 | move_reg(ctx, t1, src); | |
571 | emit_sext_32(ctx, t1, is32); | |
572 | emit_insn(ctx, divd, dst, dst, t1); | |
573 | emit_sext_32(ctx, dst, is32); | |
574 | } | |
5dc61552 TY |
575 | break; |
576 | ||
577 | /* dst = dst / imm */ | |
578 | case BPF_ALU | BPF_DIV | BPF_K: | |
579 | case BPF_ALU64 | BPF_DIV | BPF_K: | |
2425c9e0 HC |
580 | if (!off) { |
581 | move_imm(ctx, t1, imm, is32); | |
582 | emit_zext_32(ctx, dst, is32); | |
583 | emit_insn(ctx, divdu, dst, dst, t1); | |
584 | emit_zext_32(ctx, dst, is32); | |
585 | } else { | |
586 | move_imm(ctx, t1, imm, false); | |
587 | emit_sext_32(ctx, t1, is32); | |
588 | emit_sext_32(ctx, dst, is32); | |
589 | emit_insn(ctx, divd, dst, dst, t1); | |
590 | emit_sext_32(ctx, dst, is32); | |
591 | } | |
5dc61552 TY |
592 | break; |
593 | ||
594 | /* dst = dst % src */ | |
595 | case BPF_ALU | BPF_MOD | BPF_X: | |
596 | case BPF_ALU64 | BPF_MOD | BPF_X: | |
7b6b13d3 HC |
597 | if (!off) { |
598 | emit_zext_32(ctx, dst, is32); | |
599 | move_reg(ctx, t1, src); | |
600 | emit_zext_32(ctx, t1, is32); | |
601 | emit_insn(ctx, moddu, dst, dst, t1); | |
602 | emit_zext_32(ctx, dst, is32); | |
603 | } else { | |
604 | emit_sext_32(ctx, dst, is32); | |
605 | move_reg(ctx, t1, src); | |
606 | emit_sext_32(ctx, t1, is32); | |
607 | emit_insn(ctx, modd, dst, dst, t1); | |
608 | emit_sext_32(ctx, dst, is32); | |
609 | } | |
5dc61552 TY |
610 | break; |
611 | ||
612 | /* dst = dst % imm */ | |
613 | case BPF_ALU | BPF_MOD | BPF_K: | |
614 | case BPF_ALU64 | BPF_MOD | BPF_K: | |
7b6b13d3 HC |
615 | if (!off) { |
616 | move_imm(ctx, t1, imm, is32); | |
617 | emit_zext_32(ctx, dst, is32); | |
618 | emit_insn(ctx, moddu, dst, dst, t1); | |
619 | emit_zext_32(ctx, dst, is32); | |
620 | } else { | |
621 | move_imm(ctx, t1, imm, false); | |
622 | emit_sext_32(ctx, t1, is32); | |
623 | emit_sext_32(ctx, dst, is32); | |
624 | emit_insn(ctx, modd, dst, dst, t1); | |
625 | emit_sext_32(ctx, dst, is32); | |
626 | } | |
5dc61552 TY |
627 | break; |
628 | ||
629 | /* dst = -dst */ | |
630 | case BPF_ALU | BPF_NEG: | |
631 | case BPF_ALU64 | BPF_NEG: | |
632 | move_imm(ctx, t1, imm, is32); | |
633 | emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); | |
634 | emit_zext_32(ctx, dst, is32); | |
635 | break; | |
636 | ||
637 | /* dst = dst & src */ | |
638 | case BPF_ALU | BPF_AND | BPF_X: | |
639 | case BPF_ALU64 | BPF_AND | BPF_X: | |
640 | emit_insn(ctx, and, dst, dst, src); | |
641 | emit_zext_32(ctx, dst, is32); | |
642 | break; | |
643 | ||
644 | /* dst = dst & imm */ | |
645 | case BPF_ALU | BPF_AND | BPF_K: | |
646 | case BPF_ALU64 | BPF_AND | BPF_K: | |
647 | if (is_unsigned_imm12(imm)) { | |
648 | emit_insn(ctx, andi, dst, dst, imm); | |
649 | } else { | |
650 | move_imm(ctx, t1, imm, is32); | |
651 | emit_insn(ctx, and, dst, dst, t1); | |
652 | } | |
653 | emit_zext_32(ctx, dst, is32); | |
654 | break; | |
655 | ||
656 | /* dst = dst | src */ | |
657 | case BPF_ALU | BPF_OR | BPF_X: | |
658 | case BPF_ALU64 | BPF_OR | BPF_X: | |
659 | emit_insn(ctx, or, dst, dst, src); | |
660 | emit_zext_32(ctx, dst, is32); | |
661 | break; | |
662 | ||
663 | /* dst = dst | imm */ | |
664 | case BPF_ALU | BPF_OR | BPF_K: | |
665 | case BPF_ALU64 | BPF_OR | BPF_K: | |
666 | if (is_unsigned_imm12(imm)) { | |
667 | emit_insn(ctx, ori, dst, dst, imm); | |
668 | } else { | |
669 | move_imm(ctx, t1, imm, is32); | |
670 | emit_insn(ctx, or, dst, dst, t1); | |
671 | } | |
672 | emit_zext_32(ctx, dst, is32); | |
673 | break; | |
674 | ||
675 | /* dst = dst ^ src */ | |
676 | case BPF_ALU | BPF_XOR | BPF_X: | |
677 | case BPF_ALU64 | BPF_XOR | BPF_X: | |
678 | emit_insn(ctx, xor, dst, dst, src); | |
679 | emit_zext_32(ctx, dst, is32); | |
680 | break; | |
681 | ||
682 | /* dst = dst ^ imm */ | |
683 | case BPF_ALU | BPF_XOR | BPF_K: | |
684 | case BPF_ALU64 | BPF_XOR | BPF_K: | |
685 | if (is_unsigned_imm12(imm)) { | |
686 | emit_insn(ctx, xori, dst, dst, imm); | |
687 | } else { | |
688 | move_imm(ctx, t1, imm, is32); | |
689 | emit_insn(ctx, xor, dst, dst, t1); | |
690 | } | |
691 | emit_zext_32(ctx, dst, is32); | |
692 | break; | |
693 | ||
694 | /* dst = dst << src (logical) */ | |
695 | case BPF_ALU | BPF_LSH | BPF_X: | |
696 | emit_insn(ctx, sllw, dst, dst, src); | |
697 | emit_zext_32(ctx, dst, is32); | |
698 | break; | |
699 | ||
700 | case BPF_ALU64 | BPF_LSH | BPF_X: | |
701 | emit_insn(ctx, slld, dst, dst, src); | |
702 | break; | |
703 | ||
704 | /* dst = dst << imm (logical) */ | |
705 | case BPF_ALU | BPF_LSH | BPF_K: | |
706 | emit_insn(ctx, slliw, dst, dst, imm); | |
707 | emit_zext_32(ctx, dst, is32); | |
708 | break; | |
709 | ||
710 | case BPF_ALU64 | BPF_LSH | BPF_K: | |
711 | emit_insn(ctx, sllid, dst, dst, imm); | |
712 | break; | |
713 | ||
714 | /* dst = dst >> src (logical) */ | |
715 | case BPF_ALU | BPF_RSH | BPF_X: | |
716 | emit_insn(ctx, srlw, dst, dst, src); | |
717 | emit_zext_32(ctx, dst, is32); | |
718 | break; | |
719 | ||
720 | case BPF_ALU64 | BPF_RSH | BPF_X: | |
721 | emit_insn(ctx, srld, dst, dst, src); | |
722 | break; | |
723 | ||
724 | /* dst = dst >> imm (logical) */ | |
725 | case BPF_ALU | BPF_RSH | BPF_K: | |
726 | emit_insn(ctx, srliw, dst, dst, imm); | |
727 | emit_zext_32(ctx, dst, is32); | |
728 | break; | |
729 | ||
730 | case BPF_ALU64 | BPF_RSH | BPF_K: | |
731 | emit_insn(ctx, srlid, dst, dst, imm); | |
732 | break; | |
733 | ||
734 | /* dst = dst >> src (arithmetic) */ | |
735 | case BPF_ALU | BPF_ARSH | BPF_X: | |
736 | emit_insn(ctx, sraw, dst, dst, src); | |
737 | emit_zext_32(ctx, dst, is32); | |
738 | break; | |
739 | ||
740 | case BPF_ALU64 | BPF_ARSH | BPF_X: | |
741 | emit_insn(ctx, srad, dst, dst, src); | |
742 | break; | |
743 | ||
744 | /* dst = dst >> imm (arithmetic) */ | |
745 | case BPF_ALU | BPF_ARSH | BPF_K: | |
746 | emit_insn(ctx, sraiw, dst, dst, imm); | |
747 | emit_zext_32(ctx, dst, is32); | |
748 | break; | |
749 | ||
750 | case BPF_ALU64 | BPF_ARSH | BPF_K: | |
751 | emit_insn(ctx, sraid, dst, dst, imm); | |
752 | break; | |
753 | ||
754 | /* dst = BSWAP##imm(dst) */ | |
755 | case BPF_ALU | BPF_END | BPF_FROM_LE: | |
756 | switch (imm) { | |
757 | case 16: | |
758 | /* zero-extend 16 bits into 64 bits */ | |
759 | emit_insn(ctx, bstrpickd, dst, dst, 15, 0); | |
760 | break; | |
761 | case 32: | |
762 | /* zero-extend 32 bits into 64 bits */ | |
763 | emit_zext_32(ctx, dst, is32); | |
764 | break; | |
765 | case 64: | |
766 | /* do nothing */ | |
767 | break; | |
768 | } | |
769 | break; | |
770 | ||
771 | case BPF_ALU | BPF_END | BPF_FROM_BE: | |
4ebf9216 | 772 | case BPF_ALU64 | BPF_END | BPF_FROM_LE: |
5dc61552 TY |
773 | switch (imm) { |
774 | case 16: | |
775 | emit_insn(ctx, revb2h, dst, dst); | |
776 | /* zero-extend 16 bits into 64 bits */ | |
777 | emit_insn(ctx, bstrpickd, dst, dst, 15, 0); | |
778 | break; | |
779 | case 32: | |
780 | emit_insn(ctx, revb2w, dst, dst); | |
e2f7b3d8 TY |
781 | /* clear the upper 32 bits */ |
782 | emit_zext_32(ctx, dst, true); | |
5dc61552 TY |
783 | break; |
784 | case 64: | |
785 | emit_insn(ctx, revbd, dst, dst); | |
786 | break; | |
787 | } | |
788 | break; | |
789 | ||
790 | /* PC += off if dst cond src */ | |
791 | case BPF_JMP | BPF_JEQ | BPF_X: | |
792 | case BPF_JMP | BPF_JNE | BPF_X: | |
793 | case BPF_JMP | BPF_JGT | BPF_X: | |
794 | case BPF_JMP | BPF_JGE | BPF_X: | |
795 | case BPF_JMP | BPF_JLT | BPF_X: | |
796 | case BPF_JMP | BPF_JLE | BPF_X: | |
797 | case BPF_JMP | BPF_JSGT | BPF_X: | |
798 | case BPF_JMP | BPF_JSGE | BPF_X: | |
799 | case BPF_JMP | BPF_JSLT | BPF_X: | |
800 | case BPF_JMP | BPF_JSLE | BPF_X: | |
801 | case BPF_JMP32 | BPF_JEQ | BPF_X: | |
802 | case BPF_JMP32 | BPF_JNE | BPF_X: | |
803 | case BPF_JMP32 | BPF_JGT | BPF_X: | |
804 | case BPF_JMP32 | BPF_JGE | BPF_X: | |
805 | case BPF_JMP32 | BPF_JLT | BPF_X: | |
806 | case BPF_JMP32 | BPF_JLE | BPF_X: | |
807 | case BPF_JMP32 | BPF_JSGT | BPF_X: | |
808 | case BPF_JMP32 | BPF_JSGE | BPF_X: | |
809 | case BPF_JMP32 | BPF_JSLT | BPF_X: | |
810 | case BPF_JMP32 | BPF_JSLE | BPF_X: | |
811 | jmp_offset = bpf2la_offset(i, off, ctx); | |
812 | move_reg(ctx, t1, dst); | |
813 | move_reg(ctx, t2, src); | |
814 | if (is_signed_bpf_cond(BPF_OP(code))) { | |
815 | emit_sext_32(ctx, t1, is32); | |
816 | emit_sext_32(ctx, t2, is32); | |
817 | } else { | |
818 | emit_zext_32(ctx, t1, is32); | |
819 | emit_zext_32(ctx, t2, is32); | |
820 | } | |
821 | if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) | |
822 | goto toofar; | |
823 | break; | |
824 | ||
825 | /* PC += off if dst cond imm */ | |
826 | case BPF_JMP | BPF_JEQ | BPF_K: | |
827 | case BPF_JMP | BPF_JNE | BPF_K: | |
828 | case BPF_JMP | BPF_JGT | BPF_K: | |
829 | case BPF_JMP | BPF_JGE | BPF_K: | |
830 | case BPF_JMP | BPF_JLT | BPF_K: | |
831 | case BPF_JMP | BPF_JLE | BPF_K: | |
832 | case BPF_JMP | BPF_JSGT | BPF_K: | |
833 | case BPF_JMP | BPF_JSGE | BPF_K: | |
834 | case BPF_JMP | BPF_JSLT | BPF_K: | |
835 | case BPF_JMP | BPF_JSLE | BPF_K: | |
836 | case BPF_JMP32 | BPF_JEQ | BPF_K: | |
837 | case BPF_JMP32 | BPF_JNE | BPF_K: | |
838 | case BPF_JMP32 | BPF_JGT | BPF_K: | |
839 | case BPF_JMP32 | BPF_JGE | BPF_K: | |
840 | case BPF_JMP32 | BPF_JLT | BPF_K: | |
841 | case BPF_JMP32 | BPF_JLE | BPF_K: | |
842 | case BPF_JMP32 | BPF_JSGT | BPF_K: | |
843 | case BPF_JMP32 | BPF_JSGE | BPF_K: | |
844 | case BPF_JMP32 | BPF_JSLT | BPF_K: | |
845 | case BPF_JMP32 | BPF_JSLE | BPF_K: | |
5dc61552 TY |
846 | jmp_offset = bpf2la_offset(i, off, ctx); |
847 | if (imm) { | |
848 | move_imm(ctx, t1, imm, false); | |
bbfddb90 | 849 | tm = t1; |
5dc61552 TY |
850 | } else { |
851 | /* If imm is 0, simply use zero register. */ | |
bbfddb90 | 852 | tm = LOONGARCH_GPR_ZERO; |
5dc61552 TY |
853 | } |
854 | move_reg(ctx, t2, dst); | |
855 | if (is_signed_bpf_cond(BPF_OP(code))) { | |
bbfddb90 | 856 | emit_sext_32(ctx, tm, is32); |
5dc61552 TY |
857 | emit_sext_32(ctx, t2, is32); |
858 | } else { | |
bbfddb90 | 859 | emit_zext_32(ctx, tm, is32); |
5dc61552 TY |
860 | emit_zext_32(ctx, t2, is32); |
861 | } | |
bbfddb90 | 862 | if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) |
5dc61552 TY |
863 | goto toofar; |
864 | break; | |
865 | ||
866 | /* PC += off if dst & src */ | |
867 | case BPF_JMP | BPF_JSET | BPF_X: | |
868 | case BPF_JMP32 | BPF_JSET | BPF_X: | |
869 | jmp_offset = bpf2la_offset(i, off, ctx); | |
870 | emit_insn(ctx, and, t1, dst, src); | |
871 | emit_zext_32(ctx, t1, is32); | |
872 | if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) | |
873 | goto toofar; | |
874 | break; | |
875 | ||
876 | /* PC += off if dst & imm */ | |
877 | case BPF_JMP | BPF_JSET | BPF_K: | |
878 | case BPF_JMP32 | BPF_JSET | BPF_K: | |
879 | jmp_offset = bpf2la_offset(i, off, ctx); | |
880 | move_imm(ctx, t1, imm, is32); | |
881 | emit_insn(ctx, and, t1, dst, t1); | |
882 | emit_zext_32(ctx, t1, is32); | |
883 | if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) | |
884 | goto toofar; | |
885 | break; | |
886 | ||
887 | /* PC += off */ | |
888 | case BPF_JMP | BPF_JA: | |
9ddd2b8d HC |
889 | case BPF_JMP32 | BPF_JA: |
890 | if (BPF_CLASS(code) == BPF_JMP) | |
891 | jmp_offset = bpf2la_offset(i, off, ctx); | |
892 | else | |
893 | jmp_offset = bpf2la_offset(i, imm, ctx); | |
5dc61552 TY |
894 | if (emit_uncond_jmp(ctx, jmp_offset) < 0) |
895 | goto toofar; | |
896 | break; | |
897 | ||
898 | /* function call */ | |
899 | case BPF_JMP | BPF_CALL: | |
5dc61552 TY |
900 | mark_call(ctx); |
901 | ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, | |
902 | &func_addr, &func_addr_fixed); | |
903 | if (ret < 0) | |
904 | return ret; | |
905 | ||
64f50f65 | 906 | move_addr(ctx, t1, func_addr); |
5dc61552 TY |
907 | emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0); |
908 | move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); | |
909 | break; | |
910 | ||
911 | /* tail call */ | |
912 | case BPF_JMP | BPF_TAIL_CALL: | |
913 | mark_tail_call(ctx); | |
914 | if (emit_bpf_tail_call(ctx) < 0) | |
915 | return -EINVAL; | |
916 | break; | |
917 | ||
918 | /* function return */ | |
919 | case BPF_JMP | BPF_EXIT: | |
5dc61552 TY |
920 | if (i == ctx->prog->len - 1) |
921 | break; | |
922 | ||
923 | jmp_offset = epilogue_offset(ctx); | |
924 | if (emit_uncond_jmp(ctx, jmp_offset) < 0) | |
925 | goto toofar; | |
926 | break; | |
927 | ||
928 | /* dst = imm64 */ | |
929 | case BPF_LD | BPF_IMM | BPF_DW: | |
36a87385 HC |
930 | { |
931 | const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; | |
932 | ||
5dc61552 TY |
933 | move_imm(ctx, dst, imm64, is32); |
934 | return 1; | |
36a87385 | 935 | } |
5dc61552 TY |
936 | |
937 | /* dst = *(size *)(src + off) */ | |
938 | case BPF_LDX | BPF_MEM | BPF_B: | |
939 | case BPF_LDX | BPF_MEM | BPF_H: | |
940 | case BPF_LDX | BPF_MEM | BPF_W: | |
941 | case BPF_LDX | BPF_MEM | BPF_DW: | |
dbcd7f5f YT |
942 | case BPF_LDX | BPF_PROBE_MEM | BPF_DW: |
943 | case BPF_LDX | BPF_PROBE_MEM | BPF_W: | |
944 | case BPF_LDX | BPF_PROBE_MEM | BPF_H: | |
945 | case BPF_LDX | BPF_PROBE_MEM | BPF_B: | |
7111afe8 HC |
946 | /* dst_reg = (s64)*(signed size *)(src_reg + off) */ |
947 | case BPF_LDX | BPF_MEMSX | BPF_B: | |
948 | case BPF_LDX | BPF_MEMSX | BPF_H: | |
949 | case BPF_LDX | BPF_MEMSX | BPF_W: | |
950 | case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: | |
951 | case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: | |
952 | case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: | |
953 | sign_extend = BPF_MODE(insn->code) == BPF_MEMSX || | |
954 | BPF_MODE(insn->code) == BPF_PROBE_MEMSX; | |
5dc61552 TY |
955 | switch (BPF_SIZE(code)) { |
956 | case BPF_B: | |
957 | if (is_signed_imm12(off)) { | |
7111afe8 HC |
958 | if (sign_extend) |
959 | emit_insn(ctx, ldb, dst, src, off); | |
960 | else | |
961 | emit_insn(ctx, ldbu, dst, src, off); | |
5dc61552 TY |
962 | } else { |
963 | move_imm(ctx, t1, off, is32); | |
7111afe8 HC |
964 | if (sign_extend) |
965 | emit_insn(ctx, ldxb, dst, src, t1); | |
966 | else | |
967 | emit_insn(ctx, ldxbu, dst, src, t1); | |
5dc61552 TY |
968 | } |
969 | break; | |
970 | case BPF_H: | |
971 | if (is_signed_imm12(off)) { | |
7111afe8 HC |
972 | if (sign_extend) |
973 | emit_insn(ctx, ldh, dst, src, off); | |
974 | else | |
975 | emit_insn(ctx, ldhu, dst, src, off); | |
5dc61552 TY |
976 | } else { |
977 | move_imm(ctx, t1, off, is32); | |
7111afe8 HC |
978 | if (sign_extend) |
979 | emit_insn(ctx, ldxh, dst, src, t1); | |
980 | else | |
981 | emit_insn(ctx, ldxhu, dst, src, t1); | |
5dc61552 TY |
982 | } |
983 | break; | |
984 | case BPF_W: | |
985 | if (is_signed_imm12(off)) { | |
7111afe8 HC |
986 | if (sign_extend) |
987 | emit_insn(ctx, ldw, dst, src, off); | |
988 | else | |
989 | emit_insn(ctx, ldwu, dst, src, off); | |
5dc61552 TY |
990 | } else { |
991 | move_imm(ctx, t1, off, is32); | |
7111afe8 HC |
992 | if (sign_extend) |
993 | emit_insn(ctx, ldxw, dst, src, t1); | |
994 | else | |
995 | emit_insn(ctx, ldxwu, dst, src, t1); | |
5dc61552 TY |
996 | } |
997 | break; | |
998 | case BPF_DW: | |
fe575755 HC |
999 | move_imm(ctx, t1, off, is32); |
1000 | emit_insn(ctx, ldxd, dst, src, t1); | |
5dc61552 TY |
1001 | break; |
1002 | } | |
dbcd7f5f YT |
1003 | |
1004 | ret = add_exception_handler(insn, ctx, dst); | |
1005 | if (ret) | |
1006 | return ret; | |
5dc61552 TY |
1007 | break; |
1008 | ||
1009 | /* *(size *)(dst + off) = imm */ | |
1010 | case BPF_ST | BPF_MEM | BPF_B: | |
1011 | case BPF_ST | BPF_MEM | BPF_H: | |
1012 | case BPF_ST | BPF_MEM | BPF_W: | |
1013 | case BPF_ST | BPF_MEM | BPF_DW: | |
1014 | switch (BPF_SIZE(code)) { | |
1015 | case BPF_B: | |
1016 | move_imm(ctx, t1, imm, is32); | |
1017 | if (is_signed_imm12(off)) { | |
1018 | emit_insn(ctx, stb, t1, dst, off); | |
1019 | } else { | |
1020 | move_imm(ctx, t2, off, is32); | |
1021 | emit_insn(ctx, stxb, t1, dst, t2); | |
1022 | } | |
1023 | break; | |
1024 | case BPF_H: | |
1025 | move_imm(ctx, t1, imm, is32); | |
1026 | if (is_signed_imm12(off)) { | |
1027 | emit_insn(ctx, sth, t1, dst, off); | |
1028 | } else { | |
1029 | move_imm(ctx, t2, off, is32); | |
1030 | emit_insn(ctx, stxh, t1, dst, t2); | |
1031 | } | |
1032 | break; | |
1033 | case BPF_W: | |
1034 | move_imm(ctx, t1, imm, is32); | |
1035 | if (is_signed_imm12(off)) { | |
1036 | emit_insn(ctx, stw, t1, dst, off); | |
1037 | } else if (is_signed_imm14(off)) { | |
1038 | emit_insn(ctx, stptrw, t1, dst, off); | |
1039 | } else { | |
1040 | move_imm(ctx, t2, off, is32); | |
1041 | emit_insn(ctx, stxw, t1, dst, t2); | |
1042 | } | |
1043 | break; | |
1044 | case BPF_DW: | |
1045 | move_imm(ctx, t1, imm, is32); | |
1046 | if (is_signed_imm12(off)) { | |
1047 | emit_insn(ctx, std, t1, dst, off); | |
1048 | } else if (is_signed_imm14(off)) { | |
1049 | emit_insn(ctx, stptrd, t1, dst, off); | |
1050 | } else { | |
1051 | move_imm(ctx, t2, off, is32); | |
1052 | emit_insn(ctx, stxd, t1, dst, t2); | |
1053 | } | |
1054 | break; | |
1055 | } | |
1056 | break; | |
1057 | ||
1058 | /* *(size *)(dst + off) = src */ | |
1059 | case BPF_STX | BPF_MEM | BPF_B: | |
1060 | case BPF_STX | BPF_MEM | BPF_H: | |
1061 | case BPF_STX | BPF_MEM | BPF_W: | |
1062 | case BPF_STX | BPF_MEM | BPF_DW: | |
1063 | switch (BPF_SIZE(code)) { | |
1064 | case BPF_B: | |
1065 | if (is_signed_imm12(off)) { | |
1066 | emit_insn(ctx, stb, src, dst, off); | |
1067 | } else { | |
1068 | move_imm(ctx, t1, off, is32); | |
1069 | emit_insn(ctx, stxb, src, dst, t1); | |
1070 | } | |
1071 | break; | |
1072 | case BPF_H: | |
1073 | if (is_signed_imm12(off)) { | |
1074 | emit_insn(ctx, sth, src, dst, off); | |
1075 | } else { | |
1076 | move_imm(ctx, t1, off, is32); | |
1077 | emit_insn(ctx, stxh, src, dst, t1); | |
1078 | } | |
1079 | break; | |
1080 | case BPF_W: | |
1081 | if (is_signed_imm12(off)) { | |
1082 | emit_insn(ctx, stw, src, dst, off); | |
1083 | } else if (is_signed_imm14(off)) { | |
1084 | emit_insn(ctx, stptrw, src, dst, off); | |
1085 | } else { | |
1086 | move_imm(ctx, t1, off, is32); | |
1087 | emit_insn(ctx, stxw, src, dst, t1); | |
1088 | } | |
1089 | break; | |
1090 | case BPF_DW: | |
1091 | if (is_signed_imm12(off)) { | |
1092 | emit_insn(ctx, std, src, dst, off); | |
1093 | } else if (is_signed_imm14(off)) { | |
1094 | emit_insn(ctx, stptrd, src, dst, off); | |
1095 | } else { | |
1096 | move_imm(ctx, t1, off, is32); | |
1097 | emit_insn(ctx, stxd, src, dst, t1); | |
1098 | } | |
1099 | break; | |
1100 | } | |
1101 | break; | |
1102 | ||
1103 | case BPF_STX | BPF_ATOMIC | BPF_W: | |
1104 | case BPF_STX | BPF_ATOMIC | BPF_DW: | |
1105 | emit_atomic(insn, ctx); | |
1106 | break; | |
1107 | ||
a6f6a95f GG |
1108 | /* Speculation barrier */ |
1109 | case BPF_ST | BPF_NOSPEC: | |
1110 | break; | |
1111 | ||
5dc61552 TY |
1112 | default: |
1113 | pr_err("bpf_jit: unknown opcode %02x\n", code); | |
1114 | return -EINVAL; | |
1115 | } | |
1116 | ||
1117 | return 0; | |
1118 | ||
1119 | toofar: | |
1120 | pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); | |
1121 | return -E2BIG; | |
1122 | } | |
1123 | ||
1124 | static int build_body(struct jit_ctx *ctx, bool extra_pass) | |
1125 | { | |
1126 | int i; | |
1127 | const struct bpf_prog *prog = ctx->prog; | |
1128 | ||
1129 | for (i = 0; i < prog->len; i++) { | |
1130 | const struct bpf_insn *insn = &prog->insnsi[i]; | |
1131 | int ret; | |
1132 | ||
1133 | if (ctx->image == NULL) | |
1134 | ctx->offset[i] = ctx->idx; | |
1135 | ||
1136 | ret = build_insn(insn, ctx, extra_pass); | |
1137 | if (ret > 0) { | |
1138 | i++; | |
1139 | if (ctx->image == NULL) | |
1140 | ctx->offset[i] = ctx->idx; | |
1141 | continue; | |
1142 | } | |
1143 | if (ret) | |
1144 | return ret; | |
1145 | } | |
1146 | ||
1147 | if (ctx->image == NULL) | |
1148 | ctx->offset[i] = ctx->idx; | |
1149 | ||
1150 | return 0; | |
1151 | } | |
1152 | ||
1153 | /* Fill space with break instructions */ | |
1154 | static void jit_fill_hole(void *area, unsigned int size) | |
1155 | { | |
1156 | u32 *ptr; | |
1157 | ||
1158 | /* We are guaranteed to have aligned memory */ | |
1159 | for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) | |
1160 | *ptr++ = INSN_BREAK; | |
1161 | } | |
1162 | ||
1163 | static int validate_code(struct jit_ctx *ctx) | |
1164 | { | |
1165 | int i; | |
1166 | union loongarch_instruction insn; | |
1167 | ||
1168 | for (i = 0; i < ctx->idx; i++) { | |
1169 | insn = ctx->image[i]; | |
1170 | /* Check INSN_BREAK */ | |
1171 | if (insn.word == INSN_BREAK) | |
1172 | return -1; | |
1173 | } | |
1174 | ||
dbcd7f5f YT |
1175 | if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) |
1176 | return -1; | |
1177 | ||
5dc61552 TY |
1178 | return 0; |
1179 | } | |
1180 | ||
1181 | struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |
1182 | { | |
1183 | bool tmp_blinded = false, extra_pass = false; | |
1184 | u8 *image_ptr; | |
dbcd7f5f | 1185 | int image_size, prog_size, extable_size; |
5dc61552 TY |
1186 | struct jit_ctx ctx; |
1187 | struct jit_data *jit_data; | |
1188 | struct bpf_binary_header *header; | |
1189 | struct bpf_prog *tmp, *orig_prog = prog; | |
1190 | ||
1191 | /* | |
1192 | * If BPF JIT was not enabled then we must fall back to | |
1193 | * the interpreter. | |
1194 | */ | |
1195 | if (!prog->jit_requested) | |
1196 | return orig_prog; | |
1197 | ||
1198 | tmp = bpf_jit_blind_constants(prog); | |
1199 | /* | |
1200 | * If blinding was requested and we failed during blinding, | |
1201 | * we must fall back to the interpreter. Otherwise, we save | |
1202 | * the new JITed code. | |
1203 | */ | |
1204 | if (IS_ERR(tmp)) | |
1205 | return orig_prog; | |
1206 | ||
1207 | if (tmp != prog) { | |
1208 | tmp_blinded = true; | |
1209 | prog = tmp; | |
1210 | } | |
1211 | ||
1212 | jit_data = prog->aux->jit_data; | |
1213 | if (!jit_data) { | |
1214 | jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); | |
1215 | if (!jit_data) { | |
1216 | prog = orig_prog; | |
1217 | goto out; | |
1218 | } | |
1219 | prog->aux->jit_data = jit_data; | |
1220 | } | |
1221 | if (jit_data->ctx.offset) { | |
1222 | ctx = jit_data->ctx; | |
1223 | image_ptr = jit_data->image; | |
1224 | header = jit_data->header; | |
1225 | extra_pass = true; | |
dbcd7f5f | 1226 | prog_size = sizeof(u32) * ctx.idx; |
5dc61552 TY |
1227 | goto skip_init_ctx; |
1228 | } | |
1229 | ||
1230 | memset(&ctx, 0, sizeof(ctx)); | |
1231 | ctx.prog = prog; | |
1232 | ||
1233 | ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); | |
1234 | if (ctx.offset == NULL) { | |
1235 | prog = orig_prog; | |
1236 | goto out_offset; | |
1237 | } | |
1238 | ||
1239 | /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ | |
1240 | build_prologue(&ctx); | |
1241 | if (build_body(&ctx, extra_pass)) { | |
1242 | prog = orig_prog; | |
1243 | goto out_offset; | |
1244 | } | |
1245 | ctx.epilogue_offset = ctx.idx; | |
1246 | build_epilogue(&ctx); | |
1247 | ||
dbcd7f5f YT |
1248 | extable_size = prog->aux->num_exentries * sizeof(struct exception_table_entry); |
1249 | ||
5dc61552 TY |
1250 | /* Now we know the actual image size. |
1251 | * As each LoongArch instruction is of length 32bit, | |
1252 | * we are translating number of JITed intructions into | |
1253 | * the size required to store these JITed code. | |
1254 | */ | |
dbcd7f5f YT |
1255 | prog_size = sizeof(u32) * ctx.idx; |
1256 | image_size = prog_size + extable_size; | |
5dc61552 TY |
1257 | /* Now we know the size of the structure to make */ |
1258 | header = bpf_jit_binary_alloc(image_size, &image_ptr, | |
1259 | sizeof(u32), jit_fill_hole); | |
1260 | if (header == NULL) { | |
1261 | prog = orig_prog; | |
1262 | goto out_offset; | |
1263 | } | |
1264 | ||
1265 | /* 2. Now, the actual pass to generate final JIT code */ | |
1266 | ctx.image = (union loongarch_instruction *)image_ptr; | |
dbcd7f5f YT |
1267 | if (extable_size) |
1268 | prog->aux->extable = (void *)image_ptr + prog_size; | |
5dc61552 TY |
1269 | |
1270 | skip_init_ctx: | |
1271 | ctx.idx = 0; | |
dbcd7f5f | 1272 | ctx.num_exentries = 0; |
5dc61552 TY |
1273 | |
1274 | build_prologue(&ctx); | |
1275 | if (build_body(&ctx, extra_pass)) { | |
1276 | bpf_jit_binary_free(header); | |
1277 | prog = orig_prog; | |
1278 | goto out_offset; | |
1279 | } | |
1280 | build_epilogue(&ctx); | |
1281 | ||
1282 | /* 3. Extra pass to validate JITed code */ | |
1283 | if (validate_code(&ctx)) { | |
1284 | bpf_jit_binary_free(header); | |
1285 | prog = orig_prog; | |
1286 | goto out_offset; | |
1287 | } | |
1288 | ||
1289 | /* And we're done */ | |
1290 | if (bpf_jit_enable > 1) | |
dbcd7f5f | 1291 | bpf_jit_dump(prog->len, prog_size, 2, ctx.image); |
5dc61552 TY |
1292 | |
1293 | /* Update the icache */ | |
1294 | flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); | |
1295 | ||
1296 | if (!prog->is_func || extra_pass) { | |
e60adf51 CL |
1297 | int err; |
1298 | ||
5dc61552 TY |
1299 | if (extra_pass && ctx.idx != jit_data->ctx.idx) { |
1300 | pr_err_once("multi-func JIT bug %d != %d\n", | |
1301 | ctx.idx, jit_data->ctx.idx); | |
e60adf51 CL |
1302 | goto out_free; |
1303 | } | |
1304 | err = bpf_jit_binary_lock_ro(header); | |
1305 | if (err) { | |
1306 | pr_err_once("bpf_jit_binary_lock_ro() returned %d\n", | |
1307 | err); | |
1308 | goto out_free; | |
5dc61552 | 1309 | } |
5dc61552 TY |
1310 | } else { |
1311 | jit_data->ctx = ctx; | |
1312 | jit_data->image = image_ptr; | |
1313 | jit_data->header = header; | |
1314 | } | |
1315 | prog->jited = 1; | |
dbcd7f5f | 1316 | prog->jited_len = prog_size; |
5dc61552 TY |
1317 | prog->bpf_func = (void *)ctx.image; |
1318 | ||
1319 | if (!prog->is_func || extra_pass) { | |
1320 | int i; | |
1321 | ||
1322 | /* offset[prog->len] is the size of program */ | |
1323 | for (i = 0; i <= prog->len; i++) | |
1324 | ctx.offset[i] *= LOONGARCH_INSN_SIZE; | |
1325 | bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); | |
1326 | ||
1327 | out_offset: | |
1328 | kvfree(ctx.offset); | |
1329 | kfree(jit_data); | |
1330 | prog->aux->jit_data = NULL; | |
1331 | } | |
1332 | ||
1333 | out: | |
1334 | if (tmp_blinded) | |
1335 | bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); | |
1336 | ||
1337 | out_offset = -1; | |
1338 | ||
1339 | return prog; | |
e60adf51 CL |
1340 | |
1341 | out_free: | |
1342 | bpf_jit_binary_free(header); | |
1343 | prog->bpf_func = NULL; | |
1344 | prog->jited = 0; | |
1345 | prog->jited_len = 0; | |
1346 | goto out_offset; | |
5dc61552 | 1347 | } |
bb035ef0 HC |
1348 | |
1349 | /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ | |
1350 | bool bpf_jit_supports_subprog_tailcalls(void) | |
1351 | { | |
1352 | return true; | |
1353 | } |