Commit | Line | Data |
---|---|---|
19e5eb15 HC |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/mm.h> | |
3 | #include <linux/module.h> | |
4 | #include <asm/alternative.h> | |
5 | #include <asm/cacheflush.h> | |
6 | #include <asm/inst.h> | |
7 | #include <asm/sections.h> | |
8 | ||
9 | int __read_mostly alternatives_patched; | |
10 | ||
11 | EXPORT_SYMBOL_GPL(alternatives_patched); | |
12 | ||
13 | #define MAX_PATCH_SIZE (((u8)(-1)) / LOONGARCH_INSN_SIZE) | |
14 | ||
15 | static int __initdata_or_module debug_alternative; | |
16 | ||
17 | static int __init debug_alt(char *str) | |
18 | { | |
19 | debug_alternative = 1; | |
20 | return 1; | |
21 | } | |
22 | __setup("debug-alternative", debug_alt); | |
23 | ||
24 | #define DPRINTK(fmt, args...) \ | |
25 | do { \ | |
26 | if (debug_alternative) \ | |
27 | printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args); \ | |
28 | } while (0) | |
29 | ||
30 | #define DUMP_WORDS(buf, count, fmt, args...) \ | |
31 | do { \ | |
32 | if (unlikely(debug_alternative)) { \ | |
33 | int _j; \ | |
34 | union loongarch_instruction *_buf = buf; \ | |
35 | \ | |
36 | if (!(count)) \ | |
37 | break; \ | |
38 | \ | |
39 | printk(KERN_DEBUG fmt, ##args); \ | |
40 | for (_j = 0; _j < count - 1; _j++) \ | |
41 | printk(KERN_CONT "<%08x> ", _buf[_j].word); \ | |
42 | printk(KERN_CONT "<%08x>\n", _buf[_j].word); \ | |
43 | } \ | |
44 | } while (0) | |
45 | ||
46 | /* Use this to add nops to a buffer, then text_poke the whole buffer. */ | |
47 | static void __init_or_module add_nops(union loongarch_instruction *insn, int count) | |
48 | { | |
49 | while (count--) { | |
50 | insn->word = INSN_NOP; | |
51 | insn++; | |
52 | } | |
53 | } | |
54 | ||
55 | /* Is the jump addr in local .altinstructions */ | |
56 | static inline bool in_alt_jump(unsigned long jump, void *start, void *end) | |
57 | { | |
58 | return jump >= (unsigned long)start && jump < (unsigned long)end; | |
59 | } | |
60 | ||
61 | static void __init_or_module recompute_jump(union loongarch_instruction *buf, | |
62 | union loongarch_instruction *dest, union loongarch_instruction *src, | |
63 | void *start, void *end) | |
64 | { | |
65 | unsigned int si, si_l, si_h; | |
66 | unsigned long cur_pc, jump_addr, pc; | |
67 | long offset; | |
68 | ||
69 | cur_pc = (unsigned long)src; | |
70 | pc = (unsigned long)dest; | |
71 | ||
72 | si_l = src->reg0i26_format.immediate_l; | |
73 | si_h = src->reg0i26_format.immediate_h; | |
74 | switch (src->reg0i26_format.opcode) { | |
75 | case b_op: | |
76 | case bl_op: | |
2959fce7 | 77 | jump_addr = cur_pc + sign_extend64((si_h << 16 | si_l) << 2, 27); |
19e5eb15 HC |
78 | if (in_alt_jump(jump_addr, start, end)) |
79 | return; | |
80 | offset = jump_addr - pc; | |
81 | BUG_ON(offset < -SZ_128M || offset >= SZ_128M); | |
82 | offset >>= 2; | |
83 | buf->reg0i26_format.immediate_h = offset >> 16; | |
84 | buf->reg0i26_format.immediate_l = offset; | |
85 | return; | |
86 | } | |
87 | ||
88 | si_l = src->reg1i21_format.immediate_l; | |
89 | si_h = src->reg1i21_format.immediate_h; | |
90 | switch (src->reg1i21_format.opcode) { | |
91 | case bceqz_op: /* bceqz_op = bcnez_op */ | |
92 | BUG_ON(buf->reg1i21_format.rj & BIT(4)); | |
93 | fallthrough; | |
94 | case beqz_op: | |
95 | case bnez_op: | |
2959fce7 | 96 | jump_addr = cur_pc + sign_extend64((si_h << 16 | si_l) << 2, 22); |
19e5eb15 HC |
97 | if (in_alt_jump(jump_addr, start, end)) |
98 | return; | |
99 | offset = jump_addr - pc; | |
100 | BUG_ON(offset < -SZ_4M || offset >= SZ_4M); | |
101 | offset >>= 2; | |
102 | buf->reg1i21_format.immediate_h = offset >> 16; | |
103 | buf->reg1i21_format.immediate_l = offset; | |
104 | return; | |
105 | } | |
106 | ||
107 | si = src->reg2i16_format.immediate; | |
108 | switch (src->reg2i16_format.opcode) { | |
109 | case beq_op: | |
110 | case bne_op: | |
111 | case blt_op: | |
112 | case bge_op: | |
113 | case bltu_op: | |
114 | case bgeu_op: | |
2959fce7 | 115 | jump_addr = cur_pc + sign_extend64(si << 2, 17); |
19e5eb15 HC |
116 | if (in_alt_jump(jump_addr, start, end)) |
117 | return; | |
118 | offset = jump_addr - pc; | |
119 | BUG_ON(offset < -SZ_128K || offset >= SZ_128K); | |
120 | offset >>= 2; | |
121 | buf->reg2i16_format.immediate = offset; | |
122 | return; | |
123 | } | |
124 | } | |
125 | ||
126 | static int __init_or_module copy_alt_insns(union loongarch_instruction *buf, | |
127 | union loongarch_instruction *dest, union loongarch_instruction *src, int nr) | |
128 | { | |
129 | int i; | |
130 | ||
131 | for (i = 0; i < nr; i++) { | |
132 | buf[i].word = src[i].word; | |
133 | ||
134 | if (is_pc_ins(&src[i])) { | |
135 | pr_err("Not support pcrel instruction at present!"); | |
136 | return -EINVAL; | |
137 | } | |
138 | ||
139 | if (is_branch_ins(&src[i]) && | |
140 | src[i].reg2i16_format.opcode != jirl_op) { | |
141 | recompute_jump(&buf[i], &dest[i], &src[i], src, src + nr); | |
142 | } | |
143 | } | |
144 | ||
145 | return 0; | |
146 | } | |
147 | ||
148 | /* | |
149 | * text_poke_early - Update instructions on a live kernel at boot time | |
150 | * | |
151 | * When you use this code to patch more than one byte of an instruction | |
152 | * you need to make sure that other CPUs cannot execute this code in parallel. | |
153 | * Also no thread must be currently preempted in the middle of these | |
154 | * instructions. And on the local CPU you need to be protected again NMI or MCE | |
155 | * handlers seeing an inconsistent instruction while you patch. | |
156 | */ | |
157 | static void *__init_or_module text_poke_early(union loongarch_instruction *insn, | |
158 | union loongarch_instruction *buf, unsigned int nr) | |
159 | { | |
160 | int i; | |
161 | unsigned long flags; | |
162 | ||
163 | local_irq_save(flags); | |
164 | ||
165 | for (i = 0; i < nr; i++) | |
166 | insn[i].word = buf[i].word; | |
167 | ||
168 | local_irq_restore(flags); | |
169 | ||
170 | wbflush(); | |
171 | flush_icache_range((unsigned long)insn, (unsigned long)(insn + nr)); | |
172 | ||
173 | return insn; | |
174 | } | |
175 | ||
176 | /* | |
177 | * Replace instructions with better alternatives for this CPU type. This runs | |
178 | * before SMP is initialized to avoid SMP problems with self modifying code. | |
179 | * This implies that asymmetric systems where APs have less capabilities than | |
180 | * the boot processor are not handled. Tough. Make sure you disable such | |
181 | * features by hand. | |
182 | */ | |
183 | void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end) | |
184 | { | |
185 | struct alt_instr *a; | |
186 | unsigned int nr_instr, nr_repl, nr_insnbuf; | |
187 | union loongarch_instruction *instr, *replacement; | |
188 | union loongarch_instruction insnbuf[MAX_PATCH_SIZE]; | |
189 | ||
190 | DPRINTK("alt table %px, -> %px", start, end); | |
191 | /* | |
192 | * The scan order should be from start to end. A later scanned | |
193 | * alternative code can overwrite previously scanned alternative code. | |
194 | * Some kernel functions (e.g. memcpy, memset, etc) use this order to | |
195 | * patch code. | |
196 | * | |
197 | * So be careful if you want to change the scan order to any other | |
198 | * order. | |
199 | */ | |
200 | for (a = start; a < end; a++) { | |
201 | nr_insnbuf = 0; | |
202 | ||
203 | instr = (void *)&a->instr_offset + a->instr_offset; | |
204 | replacement = (void *)&a->replace_offset + a->replace_offset; | |
205 | ||
206 | BUG_ON(a->instrlen > sizeof(insnbuf)); | |
207 | BUG_ON(a->instrlen & 0x3); | |
208 | BUG_ON(a->replacementlen & 0x3); | |
209 | ||
210 | nr_instr = a->instrlen / LOONGARCH_INSN_SIZE; | |
211 | nr_repl = a->replacementlen / LOONGARCH_INSN_SIZE; | |
212 | ||
213 | if (!cpu_has(a->feature)) { | |
214 | DPRINTK("feat not exist: %d, old: (%px len: %d), repl: (%px, len: %d)", | |
215 | a->feature, instr, a->instrlen, | |
216 | replacement, a->replacementlen); | |
217 | ||
218 | continue; | |
219 | } | |
220 | ||
221 | DPRINTK("feat: %d, old: (%px len: %d), repl: (%px, len: %d)", | |
222 | a->feature, instr, a->instrlen, | |
223 | replacement, a->replacementlen); | |
224 | ||
225 | DUMP_WORDS(instr, nr_instr, "%px: old_insn: ", instr); | |
226 | DUMP_WORDS(replacement, nr_repl, "%px: rpl_insn: ", replacement); | |
227 | ||
228 | copy_alt_insns(insnbuf, instr, replacement, nr_repl); | |
229 | nr_insnbuf = nr_repl; | |
230 | ||
231 | if (nr_instr > nr_repl) { | |
232 | add_nops(insnbuf + nr_repl, nr_instr - nr_repl); | |
233 | nr_insnbuf += nr_instr - nr_repl; | |
234 | } | |
235 | DUMP_WORDS(insnbuf, nr_insnbuf, "%px: final_insn: ", instr); | |
236 | ||
237 | text_poke_early(instr, insnbuf, nr_insnbuf); | |
238 | } | |
239 | } | |
240 | ||
241 | void __init alternative_instructions(void) | |
242 | { | |
243 | apply_alternatives(__alt_instructions, __alt_instructions_end); | |
244 | ||
245 | alternatives_patched = 1; | |
246 | } |