Commit | Line | Data |
---|---|---|
e881e783 JPB |
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* | |
3 | * IOMMU API for ARM architected SMMUv3 implementations. | |
4 | * | |
5 | * Copyright (C) 2015 ARM Limited | |
6 | */ | |
7 | ||
8 | #ifndef _ARM_SMMU_V3_H | |
9 | #define _ARM_SMMU_V3_H | |
10 | ||
11 | #include <linux/bitfield.h> | |
12 | #include <linux/iommu.h> | |
13 | #include <linux/kernel.h> | |
14 | #include <linux/mmzone.h> | |
15 | #include <linux/sizes.h> | |
16 | ||
17 | /* MMIO registers */ | |
18 | #define ARM_SMMU_IDR0 0x0 | |
19 | #define IDR0_ST_LVL GENMASK(28, 27) | |
20 | #define IDR0_ST_LVL_2LVL 1 | |
21 | #define IDR0_STALL_MODEL GENMASK(25, 24) | |
22 | #define IDR0_STALL_MODEL_STALL 0 | |
23 | #define IDR0_STALL_MODEL_FORCE 2 | |
24 | #define IDR0_TTENDIAN GENMASK(22, 21) | |
25 | #define IDR0_TTENDIAN_MIXED 0 | |
26 | #define IDR0_TTENDIAN_LE 2 | |
27 | #define IDR0_TTENDIAN_BE 3 | |
28 | #define IDR0_CD2L (1 << 19) | |
29 | #define IDR0_VMID16 (1 << 18) | |
30 | #define IDR0_PRI (1 << 16) | |
31 | #define IDR0_SEV (1 << 14) | |
32 | #define IDR0_MSI (1 << 13) | |
33 | #define IDR0_ASID16 (1 << 12) | |
34 | #define IDR0_ATS (1 << 10) | |
35 | #define IDR0_HYP (1 << 9) | |
36 | #define IDR0_COHACC (1 << 4) | |
37 | #define IDR0_TTF GENMASK(3, 2) | |
38 | #define IDR0_TTF_AARCH64 2 | |
39 | #define IDR0_TTF_AARCH32_64 3 | |
40 | #define IDR0_S1P (1 << 1) | |
41 | #define IDR0_S2P (1 << 0) | |
42 | ||
43 | #define ARM_SMMU_IDR1 0x4 | |
44 | #define IDR1_TABLES_PRESET (1 << 30) | |
45 | #define IDR1_QUEUES_PRESET (1 << 29) | |
46 | #define IDR1_REL (1 << 28) | |
47 | #define IDR1_CMDQS GENMASK(25, 21) | |
48 | #define IDR1_EVTQS GENMASK(20, 16) | |
49 | #define IDR1_PRIQS GENMASK(15, 11) | |
50 | #define IDR1_SSIDSIZE GENMASK(10, 6) | |
51 | #define IDR1_SIDSIZE GENMASK(5, 0) | |
52 | ||
53 | #define ARM_SMMU_IDR3 0xc | |
54 | #define IDR3_RIL (1 << 10) | |
55 | ||
56 | #define ARM_SMMU_IDR5 0x14 | |
57 | #define IDR5_STALL_MAX GENMASK(31, 16) | |
58 | #define IDR5_GRAN64K (1 << 6) | |
59 | #define IDR5_GRAN16K (1 << 5) | |
60 | #define IDR5_GRAN4K (1 << 4) | |
61 | #define IDR5_OAS GENMASK(2, 0) | |
62 | #define IDR5_OAS_32_BIT 0 | |
63 | #define IDR5_OAS_36_BIT 1 | |
64 | #define IDR5_OAS_40_BIT 2 | |
65 | #define IDR5_OAS_42_BIT 3 | |
66 | #define IDR5_OAS_44_BIT 4 | |
67 | #define IDR5_OAS_48_BIT 5 | |
68 | #define IDR5_OAS_52_BIT 6 | |
69 | #define IDR5_VAX GENMASK(11, 10) | |
70 | #define IDR5_VAX_52_BIT 1 | |
71 | ||
72 | #define ARM_SMMU_CR0 0x20 | |
73 | #define CR0_ATSCHK (1 << 4) | |
74 | #define CR0_CMDQEN (1 << 3) | |
75 | #define CR0_EVTQEN (1 << 2) | |
76 | #define CR0_PRIQEN (1 << 1) | |
77 | #define CR0_SMMUEN (1 << 0) | |
78 | ||
79 | #define ARM_SMMU_CR0ACK 0x24 | |
80 | ||
81 | #define ARM_SMMU_CR1 0x28 | |
82 | #define CR1_TABLE_SH GENMASK(11, 10) | |
83 | #define CR1_TABLE_OC GENMASK(9, 8) | |
84 | #define CR1_TABLE_IC GENMASK(7, 6) | |
85 | #define CR1_QUEUE_SH GENMASK(5, 4) | |
86 | #define CR1_QUEUE_OC GENMASK(3, 2) | |
87 | #define CR1_QUEUE_IC GENMASK(1, 0) | |
88 | /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */ | |
89 | #define CR1_CACHE_NC 0 | |
90 | #define CR1_CACHE_WB 1 | |
91 | #define CR1_CACHE_WT 2 | |
92 | ||
93 | #define ARM_SMMU_CR2 0x2c | |
94 | #define CR2_PTM (1 << 2) | |
95 | #define CR2_RECINVSID (1 << 1) | |
96 | #define CR2_E2H (1 << 0) | |
97 | ||
98 | #define ARM_SMMU_GBPA 0x44 | |
99 | #define GBPA_UPDATE (1 << 31) | |
100 | #define GBPA_ABORT (1 << 20) | |
101 | ||
102 | #define ARM_SMMU_IRQ_CTRL 0x50 | |
103 | #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) | |
104 | #define IRQ_CTRL_PRIQ_IRQEN (1 << 1) | |
105 | #define IRQ_CTRL_GERROR_IRQEN (1 << 0) | |
106 | ||
107 | #define ARM_SMMU_IRQ_CTRLACK 0x54 | |
108 | ||
109 | #define ARM_SMMU_GERROR 0x60 | |
110 | #define GERROR_SFM_ERR (1 << 8) | |
111 | #define GERROR_MSI_GERROR_ABT_ERR (1 << 7) | |
112 | #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6) | |
113 | #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5) | |
114 | #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4) | |
115 | #define GERROR_PRIQ_ABT_ERR (1 << 3) | |
116 | #define GERROR_EVTQ_ABT_ERR (1 << 2) | |
117 | #define GERROR_CMDQ_ERR (1 << 0) | |
655c447c | 118 | #define GERROR_ERR_MASK 0x1fd |
e881e783 JPB |
119 | |
120 | #define ARM_SMMU_GERRORN 0x64 | |
121 | ||
122 | #define ARM_SMMU_GERROR_IRQ_CFG0 0x68 | |
123 | #define ARM_SMMU_GERROR_IRQ_CFG1 0x70 | |
124 | #define ARM_SMMU_GERROR_IRQ_CFG2 0x74 | |
125 | ||
126 | #define ARM_SMMU_STRTAB_BASE 0x80 | |
127 | #define STRTAB_BASE_RA (1UL << 62) | |
128 | #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) | |
129 | ||
130 | #define ARM_SMMU_STRTAB_BASE_CFG 0x88 | |
131 | #define STRTAB_BASE_CFG_FMT GENMASK(17, 16) | |
132 | #define STRTAB_BASE_CFG_FMT_LINEAR 0 | |
133 | #define STRTAB_BASE_CFG_FMT_2LVL 1 | |
134 | #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6) | |
135 | #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0) | |
136 | ||
137 | #define ARM_SMMU_CMDQ_BASE 0x90 | |
138 | #define ARM_SMMU_CMDQ_PROD 0x98 | |
139 | #define ARM_SMMU_CMDQ_CONS 0x9c | |
140 | ||
141 | #define ARM_SMMU_EVTQ_BASE 0xa0 | |
86d2d921 RM |
142 | #define ARM_SMMU_EVTQ_PROD 0xa8 |
143 | #define ARM_SMMU_EVTQ_CONS 0xac | |
e881e783 JPB |
144 | #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0 |
145 | #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8 | |
146 | #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc | |
147 | ||
148 | #define ARM_SMMU_PRIQ_BASE 0xc0 | |
86d2d921 RM |
149 | #define ARM_SMMU_PRIQ_PROD 0xc8 |
150 | #define ARM_SMMU_PRIQ_CONS 0xcc | |
e881e783 JPB |
151 | #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0 |
152 | #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8 | |
153 | #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc | |
154 | ||
155 | #define ARM_SMMU_REG_SZ 0xe00 | |
156 | ||
157 | /* Common MSI config fields */ | |
158 | #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2) | |
159 | #define MSI_CFG2_SH GENMASK(5, 4) | |
160 | #define MSI_CFG2_MEMATTR GENMASK(3, 0) | |
161 | ||
162 | /* Common memory attribute values */ | |
163 | #define ARM_SMMU_SH_NSH 0 | |
164 | #define ARM_SMMU_SH_OSH 2 | |
165 | #define ARM_SMMU_SH_ISH 3 | |
166 | #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1 | |
167 | #define ARM_SMMU_MEMATTR_OIWB 0xf | |
168 | ||
169 | #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1)) | |
170 | #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift)) | |
171 | #define Q_OVERFLOW_FLAG (1U << 31) | |
172 | #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG) | |
173 | #define Q_ENT(q, p) ((q)->base + \ | |
174 | Q_IDX(&((q)->llq), p) * \ | |
175 | (q)->ent_dwords) | |
176 | ||
177 | #define Q_BASE_RWA (1UL << 62) | |
178 | #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) | |
179 | #define Q_BASE_LOG2SIZE GENMASK(4, 0) | |
180 | ||
181 | /* Ensure DMA allocations are naturally aligned */ | |
182 | #ifdef CONFIG_CMA_ALIGNMENT | |
183 | #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) | |
184 | #else | |
185 | #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1) | |
186 | #endif | |
f115f3c0 | 187 | #define Q_MIN_SZ_SHIFT (PAGE_SHIFT) |
e881e783 JPB |
188 | |
189 | /* | |
190 | * Stream table. | |
191 | * | |
192 | * Linear: Enough to cover 1 << IDR1.SIDSIZE entries | |
193 | * 2lvl: 128k L1 entries, | |
194 | * 256 lazy entries per table (each table covers a PCI bus) | |
195 | */ | |
196 | #define STRTAB_L1_SZ_SHIFT 20 | |
197 | #define STRTAB_SPLIT 8 | |
198 | ||
199 | #define STRTAB_L1_DESC_DWORDS 1 | |
200 | #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0) | |
201 | #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6) | |
202 | ||
203 | #define STRTAB_STE_DWORDS 8 | |
204 | #define STRTAB_STE_0_V (1UL << 0) | |
205 | #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1) | |
206 | #define STRTAB_STE_0_CFG_ABORT 0 | |
207 | #define STRTAB_STE_0_CFG_BYPASS 4 | |
208 | #define STRTAB_STE_0_CFG_S1_TRANS 5 | |
209 | #define STRTAB_STE_0_CFG_S2_TRANS 6 | |
210 | ||
211 | #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) | |
212 | #define STRTAB_STE_0_S1FMT_LINEAR 0 | |
213 | #define STRTAB_STE_0_S1FMT_64K_L2 2 | |
214 | #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) | |
215 | #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59) | |
216 | ||
217 | #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0) | |
218 | #define STRTAB_STE_1_S1DSS_TERMINATE 0x0 | |
219 | #define STRTAB_STE_1_S1DSS_BYPASS 0x1 | |
220 | #define STRTAB_STE_1_S1DSS_SSID0 0x2 | |
221 | ||
222 | #define STRTAB_STE_1_S1C_CACHE_NC 0UL | |
223 | #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL | |
224 | #define STRTAB_STE_1_S1C_CACHE_WT 2UL | |
225 | #define STRTAB_STE_1_S1C_CACHE_WB 3UL | |
226 | #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2) | |
227 | #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4) | |
228 | #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6) | |
229 | ||
230 | #define STRTAB_STE_1_S1STALLD (1UL << 27) | |
231 | ||
232 | #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28) | |
233 | #define STRTAB_STE_1_EATS_ABT 0UL | |
234 | #define STRTAB_STE_1_EATS_TRANS 1UL | |
235 | #define STRTAB_STE_1_EATS_S1CHK 2UL | |
236 | ||
237 | #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30) | |
238 | #define STRTAB_STE_1_STRW_NSEL1 0UL | |
239 | #define STRTAB_STE_1_STRW_EL2 2UL | |
240 | ||
241 | #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44) | |
242 | #define STRTAB_STE_1_SHCFG_INCOMING 1UL | |
243 | ||
244 | #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) | |
245 | #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) | |
246 | #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0) | |
247 | #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6) | |
248 | #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8) | |
249 | #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10) | |
250 | #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12) | |
251 | #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14) | |
252 | #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16) | |
253 | #define STRTAB_STE_2_S2AA64 (1UL << 51) | |
254 | #define STRTAB_STE_2_S2ENDI (1UL << 52) | |
255 | #define STRTAB_STE_2_S2PTW (1UL << 54) | |
256 | #define STRTAB_STE_2_S2R (1UL << 58) | |
257 | ||
258 | #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4) | |
259 | ||
260 | /* | |
261 | * Context descriptors. | |
262 | * | |
263 | * Linear: when less than 1024 SSIDs are supported | |
264 | * 2lvl: at most 1024 L1 entries, | |
265 | * 1024 lazy entries per table. | |
266 | */ | |
267 | #define CTXDESC_SPLIT 10 | |
268 | #define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT) | |
269 | ||
270 | #define CTXDESC_L1_DESC_DWORDS 1 | |
271 | #define CTXDESC_L1_DESC_V (1UL << 0) | |
272 | #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12) | |
273 | ||
274 | #define CTXDESC_CD_DWORDS 8 | |
275 | #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) | |
276 | #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) | |
277 | #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) | |
278 | #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) | |
279 | #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12) | |
280 | #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14) | |
281 | #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30) | |
282 | ||
283 | #define CTXDESC_CD_0_ENDI (1UL << 15) | |
284 | #define CTXDESC_CD_0_V (1UL << 31) | |
285 | ||
286 | #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32) | |
287 | #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38) | |
288 | ||
289 | #define CTXDESC_CD_0_AA64 (1UL << 41) | |
290 | #define CTXDESC_CD_0_S (1UL << 44) | |
291 | #define CTXDESC_CD_0_R (1UL << 45) | |
292 | #define CTXDESC_CD_0_A (1UL << 46) | |
293 | #define CTXDESC_CD_0_ASET (1UL << 47) | |
294 | #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) | |
295 | ||
296 | #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) | |
297 | ||
298 | /* | |
299 | * When the SMMU only supports linear context descriptor tables, pick a | |
300 | * reasonable size limit (64kB). | |
301 | */ | |
302 | #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3)) | |
303 | ||
304 | /* Command queue */ | |
305 | #define CMDQ_ENT_SZ_SHIFT 4 | |
306 | #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3) | |
307 | #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT) | |
308 | ||
309 | #define CMDQ_CONS_ERR GENMASK(30, 24) | |
310 | #define CMDQ_ERR_CERROR_NONE_IDX 0 | |
311 | #define CMDQ_ERR_CERROR_ILL_IDX 1 | |
312 | #define CMDQ_ERR_CERROR_ABT_IDX 2 | |
313 | #define CMDQ_ERR_CERROR_ATC_INV_IDX 3 | |
314 | ||
315 | #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG | |
316 | ||
317 | /* | |
318 | * This is used to size the command queue and therefore must be at least | |
319 | * BITS_PER_LONG so that the valid_map works correctly (it relies on the | |
320 | * total number of queue entries being a multiple of BITS_PER_LONG). | |
321 | */ | |
322 | #define CMDQ_BATCH_ENTRIES BITS_PER_LONG | |
323 | ||
324 | #define CMDQ_0_OP GENMASK_ULL(7, 0) | |
325 | #define CMDQ_0_SSV (1UL << 11) | |
326 | ||
327 | #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32) | |
328 | #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0) | |
329 | #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12) | |
330 | ||
331 | #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12) | |
332 | #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32) | |
333 | #define CMDQ_CFGI_1_LEAF (1UL << 0) | |
334 | #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0) | |
335 | ||
336 | #define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12) | |
337 | #define CMDQ_TLBI_RANGE_NUM_MAX 31 | |
338 | #define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20) | |
339 | #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32) | |
340 | #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48) | |
341 | #define CMDQ_TLBI_1_LEAF (1UL << 0) | |
342 | #define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8) | |
343 | #define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10) | |
344 | #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) | |
345 | #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12) | |
346 | ||
347 | #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12) | |
348 | #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32) | |
349 | #define CMDQ_ATC_0_GLOBAL (1UL << 9) | |
350 | #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0) | |
351 | #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12) | |
352 | ||
353 | #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) | |
354 | #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) | |
355 | #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) | |
356 | #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12) | |
357 | ||
395ad89d JPB |
358 | #define CMDQ_RESUME_0_RESP_TERM 0UL |
359 | #define CMDQ_RESUME_0_RESP_RETRY 1UL | |
360 | #define CMDQ_RESUME_0_RESP_ABORT 2UL | |
361 | #define CMDQ_RESUME_0_RESP GENMASK_ULL(13, 12) | |
362 | #define CMDQ_RESUME_0_SID GENMASK_ULL(63, 32) | |
363 | #define CMDQ_RESUME_1_STAG GENMASK_ULL(15, 0) | |
364 | ||
e881e783 JPB |
365 | #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12) |
366 | #define CMDQ_SYNC_0_CS_NONE 0 | |
367 | #define CMDQ_SYNC_0_CS_IRQ 1 | |
368 | #define CMDQ_SYNC_0_CS_SEV 2 | |
369 | #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22) | |
370 | #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24) | |
371 | #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32) | |
372 | #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2) | |
373 | ||
374 | /* Event queue */ | |
375 | #define EVTQ_ENT_SZ_SHIFT 5 | |
376 | #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) | |
f115f3c0 | 377 | #define EVTQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT) |
e881e783 JPB |
378 | |
379 | #define EVTQ_0_ID GENMASK_ULL(7, 0) | |
380 | ||
395ad89d JPB |
381 | #define EVT_ID_TRANSLATION_FAULT 0x10 |
382 | #define EVT_ID_ADDR_SIZE_FAULT 0x11 | |
383 | #define EVT_ID_ACCESS_FAULT 0x12 | |
384 | #define EVT_ID_PERMISSION_FAULT 0x13 | |
385 | ||
386 | #define EVTQ_0_SSV (1UL << 11) | |
387 | #define EVTQ_0_SSID GENMASK_ULL(31, 12) | |
388 | #define EVTQ_0_SID GENMASK_ULL(63, 32) | |
389 | #define EVTQ_1_STAG GENMASK_ULL(15, 0) | |
390 | #define EVTQ_1_STALL (1UL << 31) | |
391 | #define EVTQ_1_PnU (1UL << 33) | |
392 | #define EVTQ_1_InD (1UL << 34) | |
393 | #define EVTQ_1_RnW (1UL << 35) | |
394 | #define EVTQ_1_S2 (1UL << 39) | |
395 | #define EVTQ_1_CLASS GENMASK_ULL(41, 40) | |
396 | #define EVTQ_1_TT_READ (1UL << 44) | |
397 | #define EVTQ_2_ADDR GENMASK_ULL(63, 0) | |
398 | #define EVTQ_3_IPA GENMASK_ULL(51, 12) | |
399 | ||
e881e783 JPB |
400 | /* PRI queue */ |
401 | #define PRIQ_ENT_SZ_SHIFT 4 | |
402 | #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3) | |
f115f3c0 | 403 | #define PRIQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT) |
e881e783 JPB |
404 | |
405 | #define PRIQ_0_SID GENMASK_ULL(31, 0) | |
406 | #define PRIQ_0_SSID GENMASK_ULL(51, 32) | |
407 | #define PRIQ_0_PERM_PRIV (1UL << 58) | |
408 | #define PRIQ_0_PERM_EXEC (1UL << 59) | |
409 | #define PRIQ_0_PERM_READ (1UL << 60) | |
410 | #define PRIQ_0_PERM_WRITE (1UL << 61) | |
411 | #define PRIQ_0_PRG_LAST (1UL << 62) | |
412 | #define PRIQ_0_SSID_V (1UL << 63) | |
413 | ||
414 | #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0) | |
415 | #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12) | |
416 | ||
417 | /* High-level queue structures */ | |
418 | #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */ | |
419 | #define ARM_SMMU_POLL_SPIN_COUNT 10 | |
420 | ||
421 | #define MSI_IOVA_BASE 0x8000000 | |
422 | #define MSI_IOVA_LENGTH 0x100000 | |
423 | ||
424 | enum pri_resp { | |
425 | PRI_RESP_DENY = 0, | |
426 | PRI_RESP_FAIL = 1, | |
427 | PRI_RESP_SUCC = 2, | |
428 | }; | |
429 | ||
430 | struct arm_smmu_cmdq_ent { | |
431 | /* Common fields */ | |
432 | u8 opcode; | |
433 | bool substream_valid; | |
434 | ||
435 | /* Command-specific fields */ | |
436 | union { | |
437 | #define CMDQ_OP_PREFETCH_CFG 0x1 | |
438 | struct { | |
439 | u32 sid; | |
e881e783 JPB |
440 | } prefetch; |
441 | ||
442 | #define CMDQ_OP_CFGI_STE 0x3 | |
443 | #define CMDQ_OP_CFGI_ALL 0x4 | |
444 | #define CMDQ_OP_CFGI_CD 0x5 | |
445 | #define CMDQ_OP_CFGI_CD_ALL 0x6 | |
446 | struct { | |
447 | u32 sid; | |
448 | u32 ssid; | |
449 | union { | |
450 | bool leaf; | |
451 | u8 span; | |
452 | }; | |
453 | } cfgi; | |
454 | ||
455 | #define CMDQ_OP_TLBI_NH_ASID 0x11 | |
456 | #define CMDQ_OP_TLBI_NH_VA 0x12 | |
457 | #define CMDQ_OP_TLBI_EL2_ALL 0x20 | |
9111aebf JPB |
458 | #define CMDQ_OP_TLBI_EL2_ASID 0x21 |
459 | #define CMDQ_OP_TLBI_EL2_VA 0x22 | |
e881e783 JPB |
460 | #define CMDQ_OP_TLBI_S12_VMALL 0x28 |
461 | #define CMDQ_OP_TLBI_S2_IPA 0x2a | |
462 | #define CMDQ_OP_TLBI_NSNH_ALL 0x30 | |
463 | struct { | |
464 | u8 num; | |
465 | u8 scale; | |
466 | u16 asid; | |
467 | u16 vmid; | |
468 | bool leaf; | |
469 | u8 ttl; | |
470 | u8 tg; | |
471 | u64 addr; | |
472 | } tlbi; | |
473 | ||
474 | #define CMDQ_OP_ATC_INV 0x40 | |
475 | #define ATC_INV_SIZE_ALL 52 | |
476 | struct { | |
477 | u32 sid; | |
478 | u32 ssid; | |
479 | u64 addr; | |
480 | u8 size; | |
481 | bool global; | |
482 | } atc; | |
483 | ||
484 | #define CMDQ_OP_PRI_RESP 0x41 | |
485 | struct { | |
486 | u32 sid; | |
487 | u32 ssid; | |
488 | u16 grpid; | |
489 | enum pri_resp resp; | |
490 | } pri; | |
491 | ||
395ad89d JPB |
492 | #define CMDQ_OP_RESUME 0x44 |
493 | struct { | |
494 | u32 sid; | |
495 | u16 stag; | |
496 | u8 resp; | |
497 | } resume; | |
498 | ||
e881e783 JPB |
499 | #define CMDQ_OP_CMD_SYNC 0x46 |
500 | struct { | |
501 | u64 msiaddr; | |
502 | } sync; | |
503 | }; | |
504 | }; | |
505 | ||
506 | struct arm_smmu_ll_queue { | |
507 | union { | |
508 | u64 val; | |
509 | struct { | |
510 | u32 prod; | |
511 | u32 cons; | |
512 | }; | |
513 | struct { | |
514 | atomic_t prod; | |
515 | atomic_t cons; | |
516 | } atomic; | |
517 | u8 __pad[SMP_CACHE_BYTES]; | |
518 | } ____cacheline_aligned_in_smp; | |
519 | u32 max_n_shift; | |
520 | }; | |
521 | ||
522 | struct arm_smmu_queue { | |
523 | struct arm_smmu_ll_queue llq; | |
524 | int irq; /* Wired interrupt */ | |
525 | ||
526 | __le64 *base; | |
527 | dma_addr_t base_dma; | |
528 | u64 q_base; | |
529 | ||
530 | size_t ent_dwords; | |
531 | ||
532 | u32 __iomem *prod_reg; | |
533 | u32 __iomem *cons_reg; | |
534 | }; | |
535 | ||
536 | struct arm_smmu_queue_poll { | |
537 | ktime_t timeout; | |
538 | unsigned int delay; | |
539 | unsigned int spin_cnt; | |
540 | bool wfe; | |
541 | }; | |
542 | ||
543 | struct arm_smmu_cmdq { | |
544 | struct arm_smmu_queue q; | |
545 | atomic_long_t *valid_map; | |
546 | atomic_t owner_prod; | |
547 | atomic_t lock; | |
548 | }; | |
549 | ||
550 | struct arm_smmu_cmdq_batch { | |
551 | u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS]; | |
552 | int num; | |
553 | }; | |
554 | ||
555 | struct arm_smmu_evtq { | |
556 | struct arm_smmu_queue q; | |
395ad89d | 557 | struct iopf_queue *iopf; |
e881e783 JPB |
558 | u32 max_stalls; |
559 | }; | |
560 | ||
561 | struct arm_smmu_priq { | |
562 | struct arm_smmu_queue q; | |
563 | }; | |
564 | ||
565 | /* High-level stream table and context descriptor structures */ | |
566 | struct arm_smmu_strtab_l1_desc { | |
567 | u8 span; | |
568 | ||
569 | __le64 *l2ptr; | |
570 | dma_addr_t l2ptr_dma; | |
571 | }; | |
572 | ||
573 | struct arm_smmu_ctx_desc { | |
574 | u16 asid; | |
575 | u64 ttbr; | |
576 | u64 tcr; | |
577 | u64 mair; | |
3f1ce8e8 JPB |
578 | |
579 | refcount_t refs; | |
580 | struct mm_struct *mm; | |
e881e783 JPB |
581 | }; |
582 | ||
583 | struct arm_smmu_l1_ctx_desc { | |
584 | __le64 *l2ptr; | |
585 | dma_addr_t l2ptr_dma; | |
586 | }; | |
587 | ||
588 | struct arm_smmu_ctx_desc_cfg { | |
589 | __le64 *cdtab; | |
590 | dma_addr_t cdtab_dma; | |
591 | struct arm_smmu_l1_ctx_desc *l1_desc; | |
592 | unsigned int num_l1_ents; | |
593 | }; | |
594 | ||
595 | struct arm_smmu_s1_cfg { | |
596 | struct arm_smmu_ctx_desc_cfg cdcfg; | |
597 | struct arm_smmu_ctx_desc cd; | |
598 | u8 s1fmt; | |
599 | u8 s1cdmax; | |
600 | }; | |
601 | ||
602 | struct arm_smmu_s2_cfg { | |
603 | u16 vmid; | |
604 | u64 vttbr; | |
605 | u64 vtcr; | |
606 | }; | |
607 | ||
608 | struct arm_smmu_strtab_cfg { | |
609 | __le64 *strtab; | |
610 | dma_addr_t strtab_dma; | |
611 | struct arm_smmu_strtab_l1_desc *l1_desc; | |
612 | unsigned int num_l1_ents; | |
613 | ||
614 | u64 strtab_base; | |
615 | u32 strtab_base_cfg; | |
616 | }; | |
617 | ||
618 | /* An SMMUv3 instance */ | |
619 | struct arm_smmu_device { | |
620 | struct device *dev; | |
621 | void __iomem *base; | |
622 | void __iomem *page1; | |
623 | ||
624 | #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0) | |
625 | #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1) | |
626 | #define ARM_SMMU_FEAT_TT_LE (1 << 2) | |
627 | #define ARM_SMMU_FEAT_TT_BE (1 << 3) | |
628 | #define ARM_SMMU_FEAT_PRI (1 << 4) | |
629 | #define ARM_SMMU_FEAT_ATS (1 << 5) | |
630 | #define ARM_SMMU_FEAT_SEV (1 << 6) | |
631 | #define ARM_SMMU_FEAT_MSI (1 << 7) | |
632 | #define ARM_SMMU_FEAT_COHERENCY (1 << 8) | |
633 | #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9) | |
634 | #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10) | |
635 | #define ARM_SMMU_FEAT_STALLS (1 << 11) | |
636 | #define ARM_SMMU_FEAT_HYP (1 << 12) | |
637 | #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13) | |
638 | #define ARM_SMMU_FEAT_VAX (1 << 14) | |
639 | #define ARM_SMMU_FEAT_RANGE_INV (1 << 15) | |
d744f9e6 JPB |
640 | #define ARM_SMMU_FEAT_BTM (1 << 16) |
641 | #define ARM_SMMU_FEAT_SVA (1 << 17) | |
9111aebf | 642 | #define ARM_SMMU_FEAT_E2H (1 << 18) |
e881e783 JPB |
643 | u32 features; |
644 | ||
645 | #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) | |
646 | #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1) | |
647 | #define ARM_SMMU_OPT_MSIPOLL (1 << 2) | |
648 | u32 options; | |
649 | ||
650 | struct arm_smmu_cmdq cmdq; | |
651 | struct arm_smmu_evtq evtq; | |
652 | struct arm_smmu_priq priq; | |
653 | ||
654 | int gerr_irq; | |
655 | int combined_irq; | |
656 | ||
657 | unsigned long ias; /* IPA */ | |
658 | unsigned long oas; /* PA */ | |
659 | unsigned long pgsize_bitmap; | |
660 | ||
661 | #define ARM_SMMU_MAX_ASIDS (1 << 16) | |
662 | unsigned int asid_bits; | |
663 | ||
664 | #define ARM_SMMU_MAX_VMIDS (1 << 16) | |
665 | unsigned int vmid_bits; | |
666 | DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS); | |
667 | ||
668 | unsigned int ssid_bits; | |
669 | unsigned int sid_bits; | |
670 | ||
671 | struct arm_smmu_strtab_cfg strtab_cfg; | |
672 | ||
673 | /* IOMMU core code handle */ | |
674 | struct iommu_device iommu; | |
cdf315f9 JPB |
675 | |
676 | struct rb_root streams; | |
677 | struct mutex streams_mutex; | |
678 | }; | |
679 | ||
680 | struct arm_smmu_stream { | |
681 | u32 id; | |
682 | struct arm_smmu_master *master; | |
683 | struct rb_node node; | |
e881e783 JPB |
684 | }; |
685 | ||
686 | /* SMMU private data for each master */ | |
687 | struct arm_smmu_master { | |
688 | struct arm_smmu_device *smmu; | |
689 | struct device *dev; | |
690 | struct arm_smmu_domain *domain; | |
691 | struct list_head domain_head; | |
cdf315f9 JPB |
692 | struct arm_smmu_stream *streams; |
693 | unsigned int num_streams; | |
e881e783 | 694 | bool ats_enabled; |
395ad89d | 695 | bool stall_enabled; |
f534d98b | 696 | bool sva_enabled; |
395ad89d | 697 | bool iopf_enabled; |
f534d98b | 698 | struct list_head bonds; |
e881e783 JPB |
699 | unsigned int ssid_bits; |
700 | }; | |
701 | ||
702 | /* SMMU private data for an IOMMU domain */ | |
703 | enum arm_smmu_domain_stage { | |
704 | ARM_SMMU_DOMAIN_S1 = 0, | |
705 | ARM_SMMU_DOMAIN_S2, | |
706 | ARM_SMMU_DOMAIN_NESTED, | |
707 | ARM_SMMU_DOMAIN_BYPASS, | |
708 | }; | |
709 | ||
710 | struct arm_smmu_domain { | |
711 | struct arm_smmu_device *smmu; | |
712 | struct mutex init_mutex; /* Protects smmu pointer */ | |
713 | ||
714 | struct io_pgtable_ops *pgtbl_ops; | |
395ad89d | 715 | bool stall_enabled; |
e881e783 JPB |
716 | atomic_t nr_ats_masters; |
717 | ||
718 | enum arm_smmu_domain_stage stage; | |
719 | union { | |
720 | struct arm_smmu_s1_cfg s1_cfg; | |
721 | struct arm_smmu_s2_cfg s2_cfg; | |
722 | }; | |
723 | ||
724 | struct iommu_domain domain; | |
725 | ||
726 | struct list_head devices; | |
727 | spinlock_t devices_lock; | |
32784a95 JPB |
728 | |
729 | struct list_head mmu_notifiers; | |
e881e783 JPB |
730 | }; |
731 | ||
32784a95 JPB |
732 | static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) |
733 | { | |
734 | return container_of(dom, struct arm_smmu_domain, domain); | |
735 | } | |
736 | ||
3f1ce8e8 JPB |
737 | extern struct xarray arm_smmu_asid_xa; |
738 | extern struct mutex arm_smmu_asid_lock; | |
32784a95 | 739 | extern struct arm_smmu_ctx_desc quiet_cd; |
3f1ce8e8 | 740 | |
3e630336 JPB |
741 | int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid, |
742 | struct arm_smmu_ctx_desc *cd); | |
743 | void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid); | |
51d113c3 JPB |
744 | void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid, |
745 | size_t granule, bool leaf, | |
746 | struct arm_smmu_domain *smmu_domain); | |
3f1ce8e8 | 747 | bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd); |
2f7e8c55 JPB |
748 | int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid, |
749 | unsigned long iova, size_t size); | |
3f1ce8e8 | 750 | |
d744f9e6 JPB |
751 | #ifdef CONFIG_ARM_SMMU_V3_SVA |
752 | bool arm_smmu_sva_supported(struct arm_smmu_device *smmu); | |
f534d98b JPB |
753 | bool arm_smmu_master_sva_supported(struct arm_smmu_master *master); |
754 | bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master); | |
755 | int arm_smmu_master_enable_sva(struct arm_smmu_master *master); | |
756 | int arm_smmu_master_disable_sva(struct arm_smmu_master *master); | |
395ad89d | 757 | bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master); |
32784a95 JPB |
758 | struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, |
759 | void *drvdata); | |
760 | void arm_smmu_sva_unbind(struct iommu_sva *handle); | |
761 | u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle); | |
762 | void arm_smmu_sva_notifier_synchronize(void); | |
d744f9e6 JPB |
763 | #else /* CONFIG_ARM_SMMU_V3_SVA */ |
764 | static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) | |
765 | { | |
766 | return false; | |
767 | } | |
f534d98b JPB |
768 | |
769 | static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master) | |
770 | { | |
771 | return false; | |
772 | } | |
773 | ||
774 | static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master) | |
775 | { | |
776 | return false; | |
777 | } | |
778 | ||
779 | static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master) | |
780 | { | |
781 | return -ENODEV; | |
782 | } | |
783 | ||
784 | static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master) | |
785 | { | |
786 | return -ENODEV; | |
787 | } | |
32784a95 | 788 | |
395ad89d JPB |
789 | static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) |
790 | { | |
791 | return false; | |
792 | } | |
793 | ||
32784a95 JPB |
794 | static inline struct iommu_sva * |
795 | arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata) | |
796 | { | |
797 | return ERR_PTR(-ENODEV); | |
798 | } | |
799 | ||
800 | static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {} | |
801 | ||
802 | static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle) | |
803 | { | |
804 | return IOMMU_PASID_INVALID; | |
805 | } | |
806 | ||
807 | static inline void arm_smmu_sva_notifier_synchronize(void) {} | |
d744f9e6 | 808 | #endif /* CONFIG_ARM_SMMU_V3_SVA */ |
e881e783 | 809 | #endif /* _ARM_SMMU_V3_H */ |