Commit | Line | Data |
---|---|---|
56e1a4cc JG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright 2024 Google LLC. | |
4 | */ | |
5 | #include <kunit/test.h> | |
6 | #include <linux/io-pgtable.h> | |
7 | ||
8 | #include "arm-smmu-v3.h" | |
9 | ||
10 | struct arm_smmu_test_writer { | |
11 | struct arm_smmu_entry_writer writer; | |
12 | struct kunit *test; | |
13 | const __le64 *init_entry; | |
14 | const __le64 *target_entry; | |
15 | __le64 *entry; | |
16 | ||
17 | bool invalid_entry_written; | |
18 | unsigned int num_syncs; | |
19 | }; | |
20 | ||
21 | #define NUM_ENTRY_QWORDS 8 | |
22 | #define NUM_EXPECTED_SYNCS(x) x | |
23 | ||
24 | static struct arm_smmu_ste bypass_ste; | |
25 | static struct arm_smmu_ste abort_ste; | |
26 | static struct arm_smmu_device smmu = { | |
27 | .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR | |
28 | }; | |
29 | static struct mm_struct sva_mm = { | |
30 | .pgd = (void *)0xdaedbeefdeadbeefULL, | |
31 | }; | |
32 | ||
33 | static bool arm_smmu_entry_differs_in_used_bits(const __le64 *entry, | |
34 | const __le64 *used_bits, | |
35 | const __le64 *target, | |
36 | unsigned int length) | |
37 | { | |
38 | bool differs = false; | |
39 | unsigned int i; | |
40 | ||
41 | for (i = 0; i < length; i++) { | |
42 | if ((entry[i] & used_bits[i]) != target[i]) | |
43 | differs = true; | |
44 | } | |
45 | return differs; | |
46 | } | |
47 | ||
48 | static void | |
49 | arm_smmu_test_writer_record_syncs(struct arm_smmu_entry_writer *writer) | |
50 | { | |
51 | struct arm_smmu_test_writer *test_writer = | |
52 | container_of(writer, struct arm_smmu_test_writer, writer); | |
53 | __le64 *entry_used_bits; | |
54 | ||
55 | entry_used_bits = kunit_kzalloc( | |
56 | test_writer->test, sizeof(*entry_used_bits) * NUM_ENTRY_QWORDS, | |
57 | GFP_KERNEL); | |
58 | KUNIT_ASSERT_NOT_NULL(test_writer->test, entry_used_bits); | |
59 | ||
60 | pr_debug("STE value is now set to: "); | |
61 | print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, | |
62 | test_writer->entry, | |
63 | NUM_ENTRY_QWORDS * sizeof(*test_writer->entry), | |
64 | false); | |
65 | ||
66 | test_writer->num_syncs += 1; | |
67 | if (!test_writer->entry[0]) { | |
68 | test_writer->invalid_entry_written = true; | |
69 | } else { | |
70 | /* | |
71 | * At any stage in a hitless transition, the entry must be | |
72 | * equivalent to either the initial entry or the target entry | |
73 | * when only considering the bits used by the current | |
74 | * configuration. | |
75 | */ | |
76 | writer->ops->get_used(test_writer->entry, entry_used_bits); | |
77 | KUNIT_EXPECT_FALSE( | |
78 | test_writer->test, | |
79 | arm_smmu_entry_differs_in_used_bits( | |
80 | test_writer->entry, entry_used_bits, | |
81 | test_writer->init_entry, NUM_ENTRY_QWORDS) && | |
82 | arm_smmu_entry_differs_in_used_bits( | |
83 | test_writer->entry, entry_used_bits, | |
84 | test_writer->target_entry, | |
85 | NUM_ENTRY_QWORDS)); | |
86 | } | |
87 | } | |
88 | ||
89 | static void | |
90 | arm_smmu_v3_test_debug_print_used_bits(struct arm_smmu_entry_writer *writer, | |
91 | const __le64 *ste) | |
92 | { | |
93 | __le64 used_bits[NUM_ENTRY_QWORDS] = {}; | |
94 | ||
95 | arm_smmu_get_ste_used(ste, used_bits); | |
96 | pr_debug("STE used bits: "); | |
97 | print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, used_bits, | |
98 | sizeof(used_bits), false); | |
99 | } | |
100 | ||
101 | static const struct arm_smmu_entry_writer_ops test_ste_ops = { | |
102 | .sync = arm_smmu_test_writer_record_syncs, | |
103 | .get_used = arm_smmu_get_ste_used, | |
104 | }; | |
105 | ||
106 | static const struct arm_smmu_entry_writer_ops test_cd_ops = { | |
107 | .sync = arm_smmu_test_writer_record_syncs, | |
108 | .get_used = arm_smmu_get_cd_used, | |
109 | }; | |
110 | ||
111 | static void arm_smmu_v3_test_ste_expect_transition( | |
112 | struct kunit *test, const struct arm_smmu_ste *cur, | |
113 | const struct arm_smmu_ste *target, unsigned int num_syncs_expected, | |
114 | bool hitless) | |
115 | { | |
116 | struct arm_smmu_ste cur_copy = *cur; | |
117 | struct arm_smmu_test_writer test_writer = { | |
118 | .writer = { | |
119 | .ops = &test_ste_ops, | |
120 | }, | |
121 | .test = test, | |
122 | .init_entry = cur->data, | |
123 | .target_entry = target->data, | |
124 | .entry = cur_copy.data, | |
125 | .num_syncs = 0, | |
126 | .invalid_entry_written = false, | |
127 | ||
128 | }; | |
129 | ||
130 | pr_debug("STE initial value: "); | |
131 | print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, | |
132 | sizeof(cur_copy), false); | |
133 | arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); | |
134 | pr_debug("STE target value: "); | |
135 | print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, | |
136 | sizeof(cur_copy), false); | |
137 | arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, | |
138 | target->data); | |
139 | ||
140 | arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); | |
141 | ||
142 | KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); | |
143 | KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); | |
144 | KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); | |
145 | } | |
146 | ||
147 | static void arm_smmu_v3_test_ste_expect_hitless_transition( | |
148 | struct kunit *test, const struct arm_smmu_ste *cur, | |
149 | const struct arm_smmu_ste *target, unsigned int num_syncs_expected) | |
150 | { | |
151 | arm_smmu_v3_test_ste_expect_transition(test, cur, target, | |
152 | num_syncs_expected, true); | |
153 | } | |
154 | ||
155 | static const dma_addr_t fake_cdtab_dma_addr = 0xF0F0F0F0F0F0; | |
156 | ||
157 | static void arm_smmu_test_make_cdtable_ste(struct arm_smmu_ste *ste, | |
158 | const dma_addr_t dma_addr) | |
159 | { | |
160 | struct arm_smmu_master master = { | |
161 | .cd_table.cdtab_dma = dma_addr, | |
162 | .cd_table.s1cdmax = 0xFF, | |
163 | .cd_table.s1fmt = STRTAB_STE_0_S1FMT_64K_L2, | |
164 | .smmu = &smmu, | |
165 | }; | |
166 | ||
167 | arm_smmu_make_cdtable_ste(ste, &master); | |
168 | } | |
169 | ||
170 | static void arm_smmu_v3_write_ste_test_bypass_to_abort(struct kunit *test) | |
171 | { | |
172 | /* | |
173 | * Bypass STEs has used bits in the first two Qwords, while abort STEs | |
174 | * only have used bits in the first QWord. Transitioning from bypass to | |
175 | * abort requires two syncs: the first to set the first qword and make | |
176 | * the STE into an abort, the second to clean up the second qword. | |
177 | */ | |
178 | arm_smmu_v3_test_ste_expect_hitless_transition( | |
179 | test, &bypass_ste, &abort_ste, NUM_EXPECTED_SYNCS(2)); | |
180 | } | |
181 | ||
182 | static void arm_smmu_v3_write_ste_test_abort_to_bypass(struct kunit *test) | |
183 | { | |
184 | /* | |
185 | * Transitioning from abort to bypass also requires two syncs: the first | |
186 | * to set the second qword data required by the bypass STE, and the | |
187 | * second to set the first qword and switch to bypass. | |
188 | */ | |
189 | arm_smmu_v3_test_ste_expect_hitless_transition( | |
190 | test, &abort_ste, &bypass_ste, NUM_EXPECTED_SYNCS(2)); | |
191 | } | |
192 | ||
193 | static void arm_smmu_v3_write_ste_test_cdtable_to_abort(struct kunit *test) | |
194 | { | |
195 | struct arm_smmu_ste ste; | |
196 | ||
197 | arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); | |
198 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, | |
199 | NUM_EXPECTED_SYNCS(2)); | |
200 | } | |
201 | ||
202 | static void arm_smmu_v3_write_ste_test_abort_to_cdtable(struct kunit *test) | |
203 | { | |
204 | struct arm_smmu_ste ste; | |
205 | ||
206 | arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); | |
207 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, | |
208 | NUM_EXPECTED_SYNCS(2)); | |
209 | } | |
210 | ||
211 | static void arm_smmu_v3_write_ste_test_cdtable_to_bypass(struct kunit *test) | |
212 | { | |
213 | struct arm_smmu_ste ste; | |
214 | ||
215 | arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); | |
216 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, | |
217 | NUM_EXPECTED_SYNCS(3)); | |
218 | } | |
219 | ||
220 | static void arm_smmu_v3_write_ste_test_bypass_to_cdtable(struct kunit *test) | |
221 | { | |
222 | struct arm_smmu_ste ste; | |
223 | ||
224 | arm_smmu_test_make_cdtable_ste(&ste, fake_cdtab_dma_addr); | |
225 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, | |
226 | NUM_EXPECTED_SYNCS(3)); | |
227 | } | |
228 | ||
229 | static void arm_smmu_test_make_s2_ste(struct arm_smmu_ste *ste, | |
230 | bool ats_enabled) | |
231 | { | |
232 | struct arm_smmu_master master = { | |
233 | .smmu = &smmu, | |
234 | .ats_enabled = ats_enabled, | |
235 | }; | |
236 | struct io_pgtable io_pgtable = {}; | |
237 | struct arm_smmu_domain smmu_domain = { | |
238 | .pgtbl_ops = &io_pgtable.ops, | |
239 | }; | |
240 | ||
241 | io_pgtable.cfg.arm_lpae_s2_cfg.vttbr = 0xdaedbeefdeadbeefULL; | |
242 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.ps = 1; | |
243 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tg = 2; | |
244 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sh = 3; | |
245 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.orgn = 1; | |
246 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.irgn = 2; | |
247 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.sl = 3; | |
248 | io_pgtable.cfg.arm_lpae_s2_cfg.vtcr.tsz = 4; | |
249 | ||
250 | arm_smmu_make_s2_domain_ste(ste, &master, &smmu_domain); | |
251 | } | |
252 | ||
253 | static void arm_smmu_v3_write_ste_test_s2_to_abort(struct kunit *test) | |
254 | { | |
255 | struct arm_smmu_ste ste; | |
256 | ||
257 | arm_smmu_test_make_s2_ste(&ste, true); | |
258 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &abort_ste, | |
259 | NUM_EXPECTED_SYNCS(2)); | |
260 | } | |
261 | ||
262 | static void arm_smmu_v3_write_ste_test_abort_to_s2(struct kunit *test) | |
263 | { | |
264 | struct arm_smmu_ste ste; | |
265 | ||
266 | arm_smmu_test_make_s2_ste(&ste, true); | |
267 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &abort_ste, &ste, | |
268 | NUM_EXPECTED_SYNCS(2)); | |
269 | } | |
270 | ||
271 | static void arm_smmu_v3_write_ste_test_s2_to_bypass(struct kunit *test) | |
272 | { | |
273 | struct arm_smmu_ste ste; | |
274 | ||
275 | arm_smmu_test_make_s2_ste(&ste, true); | |
276 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &ste, &bypass_ste, | |
277 | NUM_EXPECTED_SYNCS(2)); | |
278 | } | |
279 | ||
280 | static void arm_smmu_v3_write_ste_test_bypass_to_s2(struct kunit *test) | |
281 | { | |
282 | struct arm_smmu_ste ste; | |
283 | ||
284 | arm_smmu_test_make_s2_ste(&ste, true); | |
285 | arm_smmu_v3_test_ste_expect_hitless_transition(test, &bypass_ste, &ste, | |
286 | NUM_EXPECTED_SYNCS(2)); | |
287 | } | |
288 | ||
289 | static void arm_smmu_v3_test_cd_expect_transition( | |
290 | struct kunit *test, const struct arm_smmu_cd *cur, | |
291 | const struct arm_smmu_cd *target, unsigned int num_syncs_expected, | |
292 | bool hitless) | |
293 | { | |
294 | struct arm_smmu_cd cur_copy = *cur; | |
295 | struct arm_smmu_test_writer test_writer = { | |
296 | .writer = { | |
297 | .ops = &test_cd_ops, | |
298 | }, | |
299 | .test = test, | |
300 | .init_entry = cur->data, | |
301 | .target_entry = target->data, | |
302 | .entry = cur_copy.data, | |
303 | .num_syncs = 0, | |
304 | .invalid_entry_written = false, | |
305 | ||
306 | }; | |
307 | ||
308 | pr_debug("CD initial value: "); | |
309 | print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, cur_copy.data, | |
310 | sizeof(cur_copy), false); | |
311 | arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, cur->data); | |
312 | pr_debug("CD target value: "); | |
313 | print_hex_dump_debug(" ", DUMP_PREFIX_NONE, 16, 8, target->data, | |
314 | sizeof(cur_copy), false); | |
315 | arm_smmu_v3_test_debug_print_used_bits(&test_writer.writer, | |
316 | target->data); | |
317 | ||
318 | arm_smmu_write_entry(&test_writer.writer, cur_copy.data, target->data); | |
319 | ||
320 | KUNIT_EXPECT_EQ(test, test_writer.invalid_entry_written, !hitless); | |
321 | KUNIT_EXPECT_EQ(test, test_writer.num_syncs, num_syncs_expected); | |
322 | KUNIT_EXPECT_MEMEQ(test, target->data, cur_copy.data, sizeof(cur_copy)); | |
323 | } | |
324 | ||
325 | static void arm_smmu_v3_test_cd_expect_non_hitless_transition( | |
326 | struct kunit *test, const struct arm_smmu_cd *cur, | |
327 | const struct arm_smmu_cd *target, unsigned int num_syncs_expected) | |
328 | { | |
329 | arm_smmu_v3_test_cd_expect_transition(test, cur, target, | |
330 | num_syncs_expected, false); | |
331 | } | |
332 | ||
333 | static void arm_smmu_v3_test_cd_expect_hitless_transition( | |
334 | struct kunit *test, const struct arm_smmu_cd *cur, | |
335 | const struct arm_smmu_cd *target, unsigned int num_syncs_expected) | |
336 | { | |
337 | arm_smmu_v3_test_cd_expect_transition(test, cur, target, | |
338 | num_syncs_expected, true); | |
339 | } | |
340 | ||
341 | static void arm_smmu_test_make_s1_cd(struct arm_smmu_cd *cd, unsigned int asid) | |
342 | { | |
343 | struct arm_smmu_master master = { | |
344 | .smmu = &smmu, | |
345 | }; | |
346 | struct io_pgtable io_pgtable = {}; | |
347 | struct arm_smmu_domain smmu_domain = { | |
348 | .pgtbl_ops = &io_pgtable.ops, | |
349 | .cd = { | |
350 | .asid = asid, | |
351 | }, | |
352 | }; | |
353 | ||
354 | io_pgtable.cfg.arm_lpae_s1_cfg.ttbr = 0xdaedbeefdeadbeefULL; | |
355 | io_pgtable.cfg.arm_lpae_s1_cfg.tcr.ips = 1; | |
356 | io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tg = 2; | |
357 | io_pgtable.cfg.arm_lpae_s1_cfg.tcr.sh = 3; | |
358 | io_pgtable.cfg.arm_lpae_s1_cfg.tcr.orgn = 1; | |
359 | io_pgtable.cfg.arm_lpae_s1_cfg.tcr.irgn = 2; | |
360 | io_pgtable.cfg.arm_lpae_s1_cfg.tcr.tsz = 4; | |
361 | io_pgtable.cfg.arm_lpae_s1_cfg.mair = 0xabcdef012345678ULL; | |
362 | ||
363 | arm_smmu_make_s1_cd(cd, &master, &smmu_domain); | |
364 | } | |
365 | ||
366 | static void arm_smmu_v3_write_cd_test_s1_clear(struct kunit *test) | |
367 | { | |
368 | struct arm_smmu_cd cd = {}; | |
369 | struct arm_smmu_cd cd_2; | |
370 | ||
371 | arm_smmu_test_make_s1_cd(&cd_2, 1997); | |
372 | arm_smmu_v3_test_cd_expect_non_hitless_transition( | |
373 | test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); | |
374 | arm_smmu_v3_test_cd_expect_non_hitless_transition( | |
375 | test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); | |
376 | } | |
377 | ||
378 | static void arm_smmu_v3_write_cd_test_s1_change_asid(struct kunit *test) | |
379 | { | |
380 | struct arm_smmu_cd cd = {}; | |
381 | struct arm_smmu_cd cd_2; | |
382 | ||
383 | arm_smmu_test_make_s1_cd(&cd, 778); | |
384 | arm_smmu_test_make_s1_cd(&cd_2, 1997); | |
385 | arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, | |
386 | NUM_EXPECTED_SYNCS(1)); | |
387 | arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, | |
388 | NUM_EXPECTED_SYNCS(1)); | |
389 | } | |
390 | ||
391 | static void arm_smmu_test_make_sva_cd(struct arm_smmu_cd *cd, unsigned int asid) | |
392 | { | |
393 | struct arm_smmu_master master = { | |
394 | .smmu = &smmu, | |
395 | }; | |
396 | ||
397 | arm_smmu_make_sva_cd(cd, &master, &sva_mm, asid); | |
398 | } | |
399 | ||
400 | static void arm_smmu_test_make_sva_release_cd(struct arm_smmu_cd *cd, | |
401 | unsigned int asid) | |
402 | { | |
403 | struct arm_smmu_master master = { | |
404 | .smmu = &smmu, | |
405 | }; | |
406 | ||
407 | arm_smmu_make_sva_cd(cd, &master, NULL, asid); | |
408 | } | |
409 | ||
410 | static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) | |
411 | { | |
412 | struct arm_smmu_cd cd = {}; | |
413 | struct arm_smmu_cd cd_2; | |
414 | ||
415 | arm_smmu_test_make_sva_cd(&cd_2, 1997); | |
416 | arm_smmu_v3_test_cd_expect_non_hitless_transition( | |
417 | test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); | |
418 | arm_smmu_v3_test_cd_expect_non_hitless_transition( | |
419 | test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); | |
420 | } | |
421 | ||
422 | static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) | |
423 | { | |
424 | struct arm_smmu_cd cd; | |
425 | struct arm_smmu_cd cd_2; | |
426 | ||
427 | arm_smmu_test_make_sva_cd(&cd, 1997); | |
428 | arm_smmu_test_make_sva_release_cd(&cd_2, 1997); | |
429 | arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd, &cd_2, | |
430 | NUM_EXPECTED_SYNCS(2)); | |
431 | arm_smmu_v3_test_cd_expect_hitless_transition(test, &cd_2, &cd, | |
432 | NUM_EXPECTED_SYNCS(2)); | |
433 | } | |
434 | ||
435 | static struct kunit_case arm_smmu_v3_test_cases[] = { | |
436 | KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort), | |
437 | KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass), | |
438 | KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_abort), | |
439 | KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_cdtable), | |
440 | KUNIT_CASE(arm_smmu_v3_write_ste_test_cdtable_to_bypass), | |
441 | KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_cdtable), | |
442 | KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_abort), | |
443 | KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_s2), | |
444 | KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_bypass), | |
445 | KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_s2), | |
446 | KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_clear), | |
447 | KUNIT_CASE(arm_smmu_v3_write_cd_test_s1_change_asid), | |
448 | KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), | |
449 | KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release), | |
450 | {}, | |
451 | }; | |
452 | ||
453 | static int arm_smmu_v3_test_suite_init(struct kunit_suite *test) | |
454 | { | |
455 | arm_smmu_make_bypass_ste(&smmu, &bypass_ste); | |
456 | arm_smmu_make_abort_ste(&abort_ste); | |
457 | return 0; | |
458 | } | |
459 | ||
460 | static struct kunit_suite arm_smmu_v3_test_module = { | |
461 | .name = "arm-smmu-v3-kunit-test", | |
462 | .suite_init = arm_smmu_v3_test_suite_init, | |
463 | .test_cases = arm_smmu_v3_test_cases, | |
464 | }; | |
465 | kunit_test_suites(&arm_smmu_v3_test_module); | |
da55da5a JG |
466 | |
467 | MODULE_IMPORT_NS(EXPORTED_FOR_KUNIT_TESTING); | |
468 | MODULE_LICENSE("GPL v2"); |