Commit | Line | Data |
---|---|---|
50acfb2b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
08f051ed AW |
2 | /* |
3 | * Copyright (C) 2017 SiFive | |
08f051ed AW |
4 | */ |
5 | ||
2960f371 | 6 | #include <linux/acpi.h> |
5c20a3a9 | 7 | #include <linux/of.h> |
6b9391b5 | 8 | #include <linux/prctl.h> |
2960f371 | 9 | #include <asm/acpi.h> |
08f051ed AW |
10 | #include <asm/cacheflush.h> |
11 | ||
58de7754 GG |
12 | #ifdef CONFIG_SMP |
13 | ||
14 | #include <asm/sbi.h> | |
15 | ||
8bf90f32 CH |
16 | static void ipi_remote_fence_i(void *info) |
17 | { | |
18 | return local_flush_icache_all(); | |
19 | } | |
20 | ||
58de7754 GG |
21 | void flush_icache_all(void) |
22 | { | |
bb8958d5 AG |
23 | local_flush_icache_all(); |
24 | ||
9546f004 SH |
25 | if (num_online_cpus() < 2) |
26 | return; | |
27 | else if (riscv_use_sbi_for_rfence()) | |
8bf90f32 CH |
28 | sbi_remote_fence_i(NULL); |
29 | else | |
30 | on_each_cpu(ipi_remote_fence_i, NULL, 1); | |
58de7754 | 31 | } |
1833e327 | 32 | EXPORT_SYMBOL(flush_icache_all); |
58de7754 GG |
33 | |
34 | /* | |
35 | * Performs an icache flush for the given MM context. RISC-V has no direct | |
36 | * mechanism for instruction cache shoot downs, so instead we send an IPI that | |
37 | * informs the remote harts they need to flush their local instruction caches. | |
38 | * To avoid pathologically slow behavior in a common case (a bunch of | |
39 | * single-hart processes on a many-hart machine, ie 'make -j') we avoid the | |
40 | * IPIs for harts that are not currently executing a MM context and instead | |
41 | * schedule a deferred local instruction cache flush to be performed before | |
42 | * execution resumes on each hart. | |
43 | */ | |
44 | void flush_icache_mm(struct mm_struct *mm, bool local) | |
45 | { | |
46 | unsigned int cpu; | |
8bf90f32 | 47 | cpumask_t others, *mask; |
58de7754 GG |
48 | |
49 | preempt_disable(); | |
50 | ||
51 | /* Mark every hart's icache as needing a flush for this MM. */ | |
52 | mask = &mm->context.icache_stale_mask; | |
53 | cpumask_setall(mask); | |
54 | /* Flush this hart's I$ now, and mark it as flushed. */ | |
55 | cpu = smp_processor_id(); | |
56 | cpumask_clear_cpu(cpu, mask); | |
57 | local_flush_icache_all(); | |
58 | ||
59 | /* | |
60 | * Flush the I$ of other harts concurrently executing, and mark them as | |
61 | * flushed. | |
62 | */ | |
63 | cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); | |
64 | local |= cpumask_empty(&others); | |
8bf90f32 | 65 | if (mm == current->active_mm && local) { |
58de7754 GG |
66 | /* |
67 | * It's assumed that at least one strongly ordered operation is | |
68 | * performed on this hart between setting a hart's cpumask bit | |
69 | * and scheduling this MM context on that hart. Sending an SBI | |
70 | * remote message will do this, but in the case where no | |
71 | * messages are sent we still need to order this hart's writes | |
72 | * with flush_icache_deferred(). | |
73 | */ | |
74 | smp_mb(); | |
dc892fb4 | 75 | } else if (riscv_use_sbi_for_rfence()) { |
26fb751c | 76 | sbi_remote_fence_i(&others); |
8bf90f32 CH |
77 | } else { |
78 | on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); | |
58de7754 GG |
79 | } |
80 | ||
81 | preempt_enable(); | |
82 | } | |
83 | ||
84 | #endif /* CONFIG_SMP */ | |
85 | ||
6bd33e1e | 86 | #ifdef CONFIG_MMU |
01261e24 | 87 | void flush_icache_pte(struct mm_struct *mm, pte_t pte) |
08f051ed | 88 | { |
864609c6 | 89 | struct folio *folio = page_folio(pte_page(pte)); |
08f051ed | 90 | |
864609c6 | 91 | if (!test_bit(PG_dcache_clean, &folio->flags)) { |
01261e24 | 92 | flush_icache_mm(mm, false); |
864609c6 | 93 | set_bit(PG_dcache_clean, &folio->flags); |
950b879b | 94 | } |
08f051ed | 95 | } |
6bd33e1e | 96 | #endif /* CONFIG_MMU */ |
5c20a3a9 AJ |
97 | |
98 | unsigned int riscv_cbom_block_size; | |
99 | EXPORT_SYMBOL_GPL(riscv_cbom_block_size); | |
100 | ||
7ea5a736 AJ |
101 | unsigned int riscv_cboz_block_size; |
102 | EXPORT_SYMBOL_GPL(riscv_cboz_block_size); | |
103 | ||
3b472f86 JZ |
104 | static void __init cbo_get_block_size(struct device_node *node, |
105 | const char *name, u32 *block_size, | |
106 | unsigned long *first_hartid) | |
5c20a3a9 | 107 | { |
8b05e7d0 AJ |
108 | unsigned long hartid; |
109 | u32 val; | |
110 | ||
111 | if (riscv_of_processor_hartid(node, &hartid)) | |
112 | return; | |
113 | ||
114 | if (of_property_read_u32(node, name, &val)) | |
115 | return; | |
116 | ||
117 | if (!*block_size) { | |
118 | *block_size = val; | |
119 | *first_hartid = hartid; | |
120 | } else if (*block_size != val) { | |
121 | pr_warn("%s mismatched between harts %lu and %lu\n", | |
122 | name, *first_hartid, hartid); | |
123 | } | |
124 | } | |
125 | ||
3b472f86 | 126 | void __init riscv_init_cbo_blocksizes(void) |
5c20a3a9 | 127 | { |
7ea5a736 AJ |
128 | unsigned long cbom_hartid, cboz_hartid; |
129 | u32 cbom_block_size = 0, cboz_block_size = 0; | |
5c20a3a9 | 130 | struct device_node *node; |
2960f371 S |
131 | struct acpi_table_header *rhct; |
132 | acpi_status status; | |
133 | ||
134 | if (acpi_disabled) { | |
135 | for_each_of_cpu_node(node) { | |
136 | /* set block-size for cbom and/or cboz extension if available */ | |
137 | cbo_get_block_size(node, "riscv,cbom-block-size", | |
138 | &cbom_block_size, &cbom_hartid); | |
139 | cbo_get_block_size(node, "riscv,cboz-block-size", | |
140 | &cboz_block_size, &cboz_hartid); | |
141 | } | |
142 | } else { | |
143 | status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); | |
144 | if (ACPI_FAILURE(status)) | |
145 | return; | |
5c20a3a9 | 146 | |
2960f371 S |
147 | acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL); |
148 | acpi_put_table((struct acpi_table_header *)rhct); | |
5c20a3a9 AJ |
149 | } |
150 | ||
7ea5a736 AJ |
151 | if (cbom_block_size) |
152 | riscv_cbom_block_size = cbom_block_size; | |
153 | ||
154 | if (cboz_block_size) | |
155 | riscv_cboz_block_size = cboz_block_size; | |
5c20a3a9 | 156 | } |
6b9391b5 CJ |
157 | |
158 | #ifdef CONFIG_SMP | |
159 | static void set_icache_stale_mask(void) | |
160 | { | |
7c1e5b96 | 161 | int cpu = get_cpu(); |
6b9391b5 CJ |
162 | cpumask_t *mask; |
163 | bool stale_cpu; | |
164 | ||
165 | /* | |
166 | * Mark every other hart's icache as needing a flush for | |
167 | * this MM. Maintain the previous value of the current | |
168 | * cpu to handle the case when this function is called | |
169 | * concurrently on different harts. | |
170 | */ | |
171 | mask = ¤t->mm->context.icache_stale_mask; | |
7c1e5b96 | 172 | stale_cpu = cpumask_test_cpu(cpu, mask); |
6b9391b5 CJ |
173 | |
174 | cpumask_setall(mask); | |
7c1e5b96 CJ |
175 | cpumask_assign_cpu(cpu, mask, stale_cpu); |
176 | put_cpu(); | |
6b9391b5 CJ |
177 | } |
178 | #endif | |
179 | ||
180 | /** | |
181 | * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in | |
182 | * userspace. | |
183 | * @ctx: Set the type of icache flushing instructions permitted/prohibited in | |
184 | * userspace. Supported values described below. | |
185 | * | |
186 | * Supported values for ctx: | |
187 | * | |
188 | * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space. | |
189 | * | |
190 | * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in | |
191 | * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``. | |
192 | * Therefore, caution must be taken; use this flag only when you can guarantee | |
193 | * that no thread in the process will emit fence.i from this point onward. | |
194 | * | |
195 | * @scope: Set scope of where icache flushing instructions are allowed to be | |
196 | * emitted. Supported values described below. | |
197 | * | |
198 | * Supported values for scope: | |
199 | * | |
200 | * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process | |
201 | * is coherent with instruction storage upon | |
202 | * migration. | |
203 | * | |
204 | * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is | |
205 | * coherent with instruction storage upon | |
206 | * migration. | |
207 | * | |
208 | * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are | |
209 | * permitted to emit icache flushing instructions. Whenever any thread in the | |
210 | * process is migrated, the corresponding hart's icache will be guaranteed to be | |
211 | * consistent with instruction storage. This does not enforce any guarantees | |
212 | * outside of migration. If a thread modifies an instruction that another thread | |
213 | * may attempt to execute, the other thread must still emit an icache flushing | |
214 | * instruction before attempting to execute the potentially modified | |
215 | * instruction. This must be performed by the user-space program. | |
216 | * | |
217 | * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the | |
218 | * thread calling this function is permitted to emit icache flushing | |
219 | * instructions. When the thread is migrated, the corresponding hart's icache | |
220 | * will be guaranteed to be consistent with instruction storage. | |
221 | * | |
222 | * On kernels configured without SMP, this function is a nop as migrations | |
223 | * across harts will not occur. | |
224 | */ | |
225 | int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope) | |
226 | { | |
227 | #ifdef CONFIG_SMP | |
228 | switch (ctx) { | |
229 | case PR_RISCV_CTX_SW_FENCEI_ON: | |
230 | switch (scope) { | |
231 | case PR_RISCV_SCOPE_PER_PROCESS: | |
232 | current->mm->context.force_icache_flush = true; | |
233 | break; | |
234 | case PR_RISCV_SCOPE_PER_THREAD: | |
235 | current->thread.force_icache_flush = true; | |
236 | break; | |
237 | default: | |
238 | return -EINVAL; | |
239 | } | |
240 | break; | |
241 | case PR_RISCV_CTX_SW_FENCEI_OFF: | |
242 | switch (scope) { | |
243 | case PR_RISCV_SCOPE_PER_PROCESS: | |
6b9391b5 | 244 | set_icache_stale_mask(); |
7c1e5b96 | 245 | current->mm->context.force_icache_flush = false; |
6b9391b5 CJ |
246 | break; |
247 | case PR_RISCV_SCOPE_PER_THREAD: | |
6b9391b5 | 248 | set_icache_stale_mask(); |
7c1e5b96 | 249 | current->thread.force_icache_flush = false; |
6b9391b5 CJ |
250 | break; |
251 | default: | |
252 | return -EINVAL; | |
253 | } | |
254 | break; | |
255 | default: | |
256 | return -EINVAL; | |
257 | } | |
258 | return 0; | |
259 | #else | |
260 | switch (ctx) { | |
261 | case PR_RISCV_CTX_SW_FENCEI_ON: | |
262 | case PR_RISCV_CTX_SW_FENCEI_OFF: | |
263 | return 0; | |
264 | default: | |
265 | return -EINVAL; | |
266 | } | |
267 | #endif | |
268 | } |