Commit | Line | Data |
---|---|---|
7a31f6f4 | 1 | /* |
89184651 | 2 | * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved. |
7a31f6f4 | 3 | * |
89184651 TR |
4 | * This program is free software; you can redistribute it and/or modify |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7a31f6f4 HD |
7 | */ |
8 | ||
bc5e6dea | 9 | #include <linux/err.h> |
7a31f6f4 | 10 | #include <linux/iommu.h> |
89184651 | 11 | #include <linux/kernel.h> |
0760e8fa | 12 | #include <linux/of.h> |
89184651 TR |
13 | #include <linux/of_device.h> |
14 | #include <linux/platform_device.h> | |
15 | #include <linux/slab.h> | |
306a7f91 TR |
16 | |
17 | #include <soc/tegra/ahb.h> | |
89184651 | 18 | #include <soc/tegra/mc.h> |
7a31f6f4 | 19 | |
89184651 TR |
20 | struct tegra_smmu { |
21 | void __iomem *regs; | |
22 | struct device *dev; | |
e6bc5933 | 23 | |
89184651 TR |
24 | struct tegra_mc *mc; |
25 | const struct tegra_smmu_soc *soc; | |
39abf8aa | 26 | |
89184651 TR |
27 | unsigned long *asids; |
28 | struct mutex lock; | |
39abf8aa | 29 | |
89184651 | 30 | struct list_head list; |
7a31f6f4 | 31 | }; |
7a31f6f4 | 32 | |
89184651 TR |
33 | struct tegra_smmu_as { |
34 | struct iommu_domain *domain; | |
35 | struct tegra_smmu *smmu; | |
36 | unsigned int use_count; | |
37 | struct page *count; | |
38 | struct page *pd; | |
39 | unsigned id; | |
40 | u32 attr; | |
7a31f6f4 HD |
41 | }; |
42 | ||
89184651 TR |
43 | static inline void smmu_writel(struct tegra_smmu *smmu, u32 value, |
44 | unsigned long offset) | |
45 | { | |
46 | writel(value, smmu->regs + offset); | |
47 | } | |
7a31f6f4 | 48 | |
89184651 TR |
49 | static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) |
50 | { | |
51 | return readl(smmu->regs + offset); | |
52 | } | |
5a2c937a | 53 | |
89184651 TR |
54 | #define SMMU_CONFIG 0x010 |
55 | #define SMMU_CONFIG_ENABLE (1 << 0) | |
7a31f6f4 | 56 | |
89184651 TR |
57 | #define SMMU_TLB_CONFIG 0x14 |
58 | #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) | |
59 | #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) | |
60 | #define SMMU_TLB_CONFIG_ACTIVE_LINES(x) ((x) & 0x3f) | |
0760e8fa | 61 | |
89184651 TR |
62 | #define SMMU_PTC_CONFIG 0x18 |
63 | #define SMMU_PTC_CONFIG_ENABLE (1 << 29) | |
64 | #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24) | |
65 | #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f) | |
39abf8aa | 66 | |
89184651 TR |
67 | #define SMMU_PTB_ASID 0x01c |
68 | #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) | |
a3b24915 | 69 | |
89184651 TR |
70 | #define SMMU_PTB_DATA 0x020 |
71 | #define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr)) | |
7a31f6f4 | 72 | |
89184651 | 73 | #define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr)) |
7a31f6f4 | 74 | |
89184651 TR |
75 | #define SMMU_TLB_FLUSH 0x030 |
76 | #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) | |
77 | #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) | |
78 | #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) | |
79 | #define SMMU_TLB_FLUSH_ASID(x) (((x) & 0x7f) << 24) | |
80 | #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ | |
81 | SMMU_TLB_FLUSH_VA_MATCH_SECTION) | |
82 | #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ | |
83 | SMMU_TLB_FLUSH_VA_MATCH_GROUP) | |
84 | #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31) | |
a6870e92 | 85 | |
89184651 TR |
86 | #define SMMU_PTC_FLUSH 0x034 |
87 | #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0) | |
88 | #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0) | |
a6870e92 | 89 | |
89184651 TR |
90 | #define SMMU_PTC_FLUSH_HI 0x9b8 |
91 | #define SMMU_PTC_FLUSH_HI_MASK 0x3 | |
7a31f6f4 | 92 | |
89184651 TR |
93 | /* per-SWGROUP SMMU_*_ASID register */ |
94 | #define SMMU_ASID_ENABLE (1 << 31) | |
95 | #define SMMU_ASID_MASK 0x7f | |
96 | #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK) | |
a6870e92 | 97 | |
89184651 TR |
98 | /* page table definitions */ |
99 | #define SMMU_NUM_PDE 1024 | |
100 | #define SMMU_NUM_PTE 1024 | |
a6870e92 | 101 | |
89184651 TR |
102 | #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4) |
103 | #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4) | |
7a31f6f4 | 104 | |
89184651 TR |
105 | #define SMMU_PDE_SHIFT 22 |
106 | #define SMMU_PTE_SHIFT 12 | |
fe1229b9 | 107 | |
89184651 | 108 | #define SMMU_PFN_MASK 0x000fffff |
a6870e92 | 109 | |
89184651 TR |
110 | #define SMMU_PD_READABLE (1 << 31) |
111 | #define SMMU_PD_WRITABLE (1 << 30) | |
112 | #define SMMU_PD_NONSECURE (1 << 29) | |
7a31f6f4 | 113 | |
89184651 TR |
114 | #define SMMU_PDE_READABLE (1 << 31) |
115 | #define SMMU_PDE_WRITABLE (1 << 30) | |
116 | #define SMMU_PDE_NONSECURE (1 << 29) | |
117 | #define SMMU_PDE_NEXT (1 << 28) | |
7a31f6f4 | 118 | |
89184651 TR |
119 | #define SMMU_PTE_READABLE (1 << 31) |
120 | #define SMMU_PTE_WRITABLE (1 << 30) | |
121 | #define SMMU_PTE_NONSECURE (1 << 29) | |
7a31f6f4 | 122 | |
89184651 TR |
123 | #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ |
124 | SMMU_PDE_NONSECURE) | |
125 | #define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \ | |
126 | SMMU_PTE_NONSECURE) | |
7a31f6f4 | 127 | |
89184651 TR |
128 | static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, |
129 | unsigned long offset) | |
7a31f6f4 | 130 | { |
89184651 TR |
131 | phys_addr_t phys = page ? page_to_phys(page) : 0; |
132 | u32 value; | |
133 | ||
134 | if (page) { | |
135 | offset &= ~(smmu->mc->soc->atom_size - 1); | |
136 | ||
137 | if (smmu->mc->soc->num_address_bits > 32) { | |
138 | #ifdef CONFIG_PHYS_ADDR_T_64BIT | |
139 | value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; | |
140 | #else | |
141 | value = 0; | |
142 | #endif | |
143 | smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); | |
7a31f6f4 | 144 | } |
7a31f6f4 | 145 | |
89184651 TR |
146 | value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR; |
147 | } else { | |
148 | value = SMMU_PTC_FLUSH_TYPE_ALL; | |
7a31f6f4 | 149 | } |
89184651 TR |
150 | |
151 | smmu_writel(smmu, value, SMMU_PTC_FLUSH); | |
7a31f6f4 HD |
152 | } |
153 | ||
89184651 | 154 | static inline void smmu_flush_tlb(struct tegra_smmu *smmu) |
7a31f6f4 | 155 | { |
89184651 | 156 | smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH); |
7a31f6f4 HD |
157 | } |
158 | ||
89184651 TR |
159 | static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu, |
160 | unsigned long asid) | |
7a31f6f4 | 161 | { |
89184651 | 162 | u32 value; |
7a31f6f4 | 163 | |
89184651 TR |
164 | value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | |
165 | SMMU_TLB_FLUSH_VA_MATCH_ALL; | |
166 | smmu_writel(smmu, value, SMMU_TLB_FLUSH); | |
7a31f6f4 HD |
167 | } |
168 | ||
89184651 TR |
169 | static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu, |
170 | unsigned long asid, | |
171 | unsigned long iova) | |
7a31f6f4 | 172 | { |
89184651 | 173 | u32 value; |
7a31f6f4 | 174 | |
89184651 TR |
175 | value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | |
176 | SMMU_TLB_FLUSH_VA_SECTION(iova); | |
177 | smmu_writel(smmu, value, SMMU_TLB_FLUSH); | |
7a31f6f4 HD |
178 | } |
179 | ||
89184651 TR |
180 | static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu, |
181 | unsigned long asid, | |
182 | unsigned long iova) | |
7a31f6f4 | 183 | { |
89184651 | 184 | u32 value; |
7a31f6f4 | 185 | |
89184651 TR |
186 | value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | |
187 | SMMU_TLB_FLUSH_VA_GROUP(iova); | |
188 | smmu_writel(smmu, value, SMMU_TLB_FLUSH); | |
7a31f6f4 HD |
189 | } |
190 | ||
89184651 | 191 | static inline void smmu_flush(struct tegra_smmu *smmu) |
7a31f6f4 | 192 | { |
89184651 | 193 | smmu_readl(smmu, SMMU_CONFIG); |
7a31f6f4 HD |
194 | } |
195 | ||
89184651 | 196 | static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp) |
7a31f6f4 | 197 | { |
89184651 | 198 | unsigned long id; |
7a31f6f4 | 199 | |
89184651 | 200 | mutex_lock(&smmu->lock); |
7a31f6f4 | 201 | |
89184651 TR |
202 | id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids); |
203 | if (id >= smmu->soc->num_asids) { | |
204 | mutex_unlock(&smmu->lock); | |
205 | return -ENOSPC; | |
7a31f6f4 | 206 | } |
7a31f6f4 | 207 | |
89184651 TR |
208 | set_bit(id, smmu->asids); |
209 | *idp = id; | |
210 | ||
211 | mutex_unlock(&smmu->lock); | |
212 | return 0; | |
7a31f6f4 HD |
213 | } |
214 | ||
89184651 | 215 | static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) |
7a31f6f4 | 216 | { |
89184651 TR |
217 | mutex_lock(&smmu->lock); |
218 | clear_bit(id, smmu->asids); | |
219 | mutex_unlock(&smmu->lock); | |
7a31f6f4 | 220 | } |
89184651 TR |
221 | |
222 | static bool tegra_smmu_capable(enum iommu_cap cap) | |
7a31f6f4 | 223 | { |
89184651 | 224 | return false; |
7a31f6f4 | 225 | } |
7a31f6f4 | 226 | |
89184651 | 227 | static int tegra_smmu_domain_init(struct iommu_domain *domain) |
7a31f6f4 | 228 | { |
89184651 TR |
229 | struct tegra_smmu_as *as; |
230 | unsigned int i; | |
231 | uint32_t *pd; | |
7a31f6f4 | 232 | |
89184651 TR |
233 | as = kzalloc(sizeof(*as), GFP_KERNEL); |
234 | if (!as) | |
235 | return -ENOMEM; | |
7a31f6f4 | 236 | |
89184651 TR |
237 | as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; |
238 | as->domain = domain; | |
7a31f6f4 | 239 | |
89184651 TR |
240 | as->pd = alloc_page(GFP_KERNEL | __GFP_DMA); |
241 | if (!as->pd) { | |
242 | kfree(as); | |
243 | return -ENOMEM; | |
7a31f6f4 | 244 | } |
9e971a03 | 245 | |
89184651 TR |
246 | as->count = alloc_page(GFP_KERNEL); |
247 | if (!as->count) { | |
248 | __free_page(as->pd); | |
249 | kfree(as); | |
250 | return -ENOMEM; | |
7a31f6f4 | 251 | } |
9e971a03 | 252 | |
89184651 TR |
253 | /* clear PDEs */ |
254 | pd = page_address(as->pd); | |
255 | SetPageReserved(as->pd); | |
9e971a03 | 256 | |
89184651 TR |
257 | for (i = 0; i < SMMU_NUM_PDE; i++) |
258 | pd[i] = 0; | |
7a31f6f4 | 259 | |
89184651 TR |
260 | /* clear PDE usage counters */ |
261 | pd = page_address(as->count); | |
262 | SetPageReserved(as->count); | |
7a31f6f4 | 263 | |
89184651 TR |
264 | for (i = 0; i < SMMU_NUM_PDE; i++) |
265 | pd[i] = 0; | |
9e971a03 | 266 | |
89184651 | 267 | domain->priv = as; |
f9a4f063 | 268 | |
89184651 | 269 | return 0; |
7a31f6f4 HD |
270 | } |
271 | ||
89184651 | 272 | static void tegra_smmu_domain_destroy(struct iommu_domain *domain) |
7a31f6f4 | 273 | { |
89184651 | 274 | struct tegra_smmu_as *as = domain->priv; |
7a31f6f4 | 275 | |
89184651 TR |
276 | /* TODO: free page directory and page tables */ |
277 | ClearPageReserved(as->pd); | |
7a31f6f4 | 278 | |
89184651 | 279 | kfree(as); |
7a31f6f4 HD |
280 | } |
281 | ||
89184651 TR |
282 | static const struct tegra_smmu_swgroup * |
283 | tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup) | |
7a31f6f4 | 284 | { |
89184651 TR |
285 | const struct tegra_smmu_swgroup *group = NULL; |
286 | unsigned int i; | |
7a31f6f4 | 287 | |
89184651 TR |
288 | for (i = 0; i < smmu->soc->num_swgroups; i++) { |
289 | if (smmu->soc->swgroups[i].swgroup == swgroup) { | |
290 | group = &smmu->soc->swgroups[i]; | |
291 | break; | |
292 | } | |
293 | } | |
7a31f6f4 | 294 | |
89184651 | 295 | return group; |
7a31f6f4 HD |
296 | } |
297 | ||
89184651 TR |
298 | static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup, |
299 | unsigned int asid) | |
7a31f6f4 | 300 | { |
89184651 TR |
301 | const struct tegra_smmu_swgroup *group; |
302 | unsigned int i; | |
303 | u32 value; | |
7a31f6f4 | 304 | |
89184651 TR |
305 | for (i = 0; i < smmu->soc->num_clients; i++) { |
306 | const struct tegra_mc_client *client = &smmu->soc->clients[i]; | |
7a31f6f4 | 307 | |
89184651 TR |
308 | if (client->swgroup != swgroup) |
309 | continue; | |
7a31f6f4 | 310 | |
89184651 TR |
311 | value = smmu_readl(smmu, client->smmu.reg); |
312 | value |= BIT(client->smmu.bit); | |
313 | smmu_writel(smmu, value, client->smmu.reg); | |
314 | } | |
7a31f6f4 | 315 | |
89184651 TR |
316 | group = tegra_smmu_find_swgroup(smmu, swgroup); |
317 | if (group) { | |
318 | value = smmu_readl(smmu, group->reg); | |
319 | value &= ~SMMU_ASID_MASK; | |
320 | value |= SMMU_ASID_VALUE(asid); | |
321 | value |= SMMU_ASID_ENABLE; | |
322 | smmu_writel(smmu, value, group->reg); | |
323 | } | |
7a31f6f4 HD |
324 | } |
325 | ||
89184651 TR |
326 | static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup, |
327 | unsigned int asid) | |
7a31f6f4 | 328 | { |
89184651 TR |
329 | const struct tegra_smmu_swgroup *group; |
330 | unsigned int i; | |
331 | u32 value; | |
7a31f6f4 | 332 | |
89184651 TR |
333 | group = tegra_smmu_find_swgroup(smmu, swgroup); |
334 | if (group) { | |
335 | value = smmu_readl(smmu, group->reg); | |
336 | value &= ~SMMU_ASID_MASK; | |
337 | value |= SMMU_ASID_VALUE(asid); | |
338 | value &= ~SMMU_ASID_ENABLE; | |
339 | smmu_writel(smmu, value, group->reg); | |
340 | } | |
7a31f6f4 | 341 | |
89184651 TR |
342 | for (i = 0; i < smmu->soc->num_clients; i++) { |
343 | const struct tegra_mc_client *client = &smmu->soc->clients[i]; | |
7a31f6f4 | 344 | |
89184651 TR |
345 | if (client->swgroup != swgroup) |
346 | continue; | |
7a31f6f4 | 347 | |
89184651 TR |
348 | value = smmu_readl(smmu, client->smmu.reg); |
349 | value &= ~BIT(client->smmu.bit); | |
350 | smmu_writel(smmu, value, client->smmu.reg); | |
351 | } | |
7a31f6f4 HD |
352 | } |
353 | ||
89184651 TR |
354 | static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, |
355 | struct tegra_smmu_as *as) | |
7a31f6f4 | 356 | { |
89184651 | 357 | u32 value; |
7a31f6f4 HD |
358 | int err; |
359 | ||
89184651 TR |
360 | if (as->use_count > 0) { |
361 | as->use_count++; | |
362 | return 0; | |
7a31f6f4 | 363 | } |
7a31f6f4 | 364 | |
89184651 TR |
365 | err = tegra_smmu_alloc_asid(smmu, &as->id); |
366 | if (err < 0) | |
367 | return err; | |
7a31f6f4 | 368 | |
89184651 TR |
369 | smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD); |
370 | smmu_flush_ptc(smmu, as->pd, 0); | |
371 | smmu_flush_tlb_asid(smmu, as->id); | |
7a31f6f4 | 372 | |
89184651 TR |
373 | smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); |
374 | value = SMMU_PTB_DATA_VALUE(as->pd, as->attr); | |
375 | smmu_writel(smmu, value, SMMU_PTB_DATA); | |
376 | smmu_flush(smmu); | |
7a31f6f4 | 377 | |
89184651 TR |
378 | as->smmu = smmu; |
379 | as->use_count++; | |
7a31f6f4 | 380 | |
89184651 | 381 | return 0; |
7a31f6f4 HD |
382 | } |
383 | ||
89184651 TR |
384 | static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, |
385 | struct tegra_smmu_as *as) | |
7a31f6f4 | 386 | { |
89184651 TR |
387 | if (--as->use_count > 0) |
388 | return; | |
389 | ||
390 | tegra_smmu_free_asid(smmu, as->id); | |
391 | as->smmu = NULL; | |
7a31f6f4 HD |
392 | } |
393 | ||
89184651 TR |
394 | static int tegra_smmu_attach_dev(struct iommu_domain *domain, |
395 | struct device *dev) | |
7a31f6f4 | 396 | { |
89184651 TR |
397 | struct tegra_smmu *smmu = dev->archdata.iommu; |
398 | struct tegra_smmu_as *as = domain->priv; | |
399 | struct device_node *np = dev->of_node; | |
400 | struct of_phandle_args args; | |
401 | unsigned int index = 0; | |
402 | int err = 0; | |
7a31f6f4 | 403 | |
89184651 TR |
404 | while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, |
405 | &args)) { | |
406 | unsigned int swgroup = args.args[0]; | |
d2453b2c | 407 | |
89184651 TR |
408 | if (args.np != smmu->dev->of_node) { |
409 | of_node_put(args.np); | |
d2453b2c | 410 | continue; |
89184651 | 411 | } |
d2453b2c | 412 | |
89184651 | 413 | of_node_put(args.np); |
d2453b2c | 414 | |
89184651 TR |
415 | err = tegra_smmu_as_prepare(smmu, as); |
416 | if (err < 0) | |
417 | return err; | |
418 | ||
419 | tegra_smmu_enable(smmu, swgroup, as->id); | |
420 | index++; | |
7a31f6f4 | 421 | } |
7a31f6f4 | 422 | |
89184651 TR |
423 | if (index == 0) |
424 | return -ENODEV; | |
7a31f6f4 | 425 | |
89184651 TR |
426 | return 0; |
427 | } | |
7a31f6f4 | 428 | |
89184651 TR |
429 | static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) |
430 | { | |
431 | struct tegra_smmu_as *as = domain->priv; | |
432 | struct device_node *np = dev->of_node; | |
433 | struct tegra_smmu *smmu = as->smmu; | |
434 | struct of_phandle_args args; | |
435 | unsigned int index = 0; | |
7a31f6f4 | 436 | |
89184651 TR |
437 | while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, |
438 | &args)) { | |
439 | unsigned int swgroup = args.args[0]; | |
7a31f6f4 | 440 | |
89184651 TR |
441 | if (args.np != smmu->dev->of_node) { |
442 | of_node_put(args.np); | |
443 | continue; | |
444 | } | |
23349902 | 445 | |
89184651 | 446 | of_node_put(args.np); |
7a31f6f4 | 447 | |
89184651 TR |
448 | tegra_smmu_disable(smmu, swgroup, as->id); |
449 | tegra_smmu_as_unprepare(smmu, as); | |
450 | index++; | |
451 | } | |
7a31f6f4 HD |
452 | } |
453 | ||
89184651 TR |
454 | static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, |
455 | struct page **pagep) | |
7a31f6f4 | 456 | { |
89184651 TR |
457 | u32 *pd = page_address(as->pd), *pt, *count; |
458 | u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; | |
459 | u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; | |
460 | struct tegra_smmu *smmu = as->smmu; | |
461 | struct page *page; | |
462 | unsigned int i; | |
463 | ||
464 | if (pd[pde] == 0) { | |
465 | page = alloc_page(GFP_KERNEL | __GFP_DMA); | |
466 | if (!page) | |
467 | return NULL; | |
7a31f6f4 | 468 | |
89184651 TR |
469 | pt = page_address(page); |
470 | SetPageReserved(page); | |
7a31f6f4 | 471 | |
89184651 TR |
472 | for (i = 0; i < SMMU_NUM_PTE; i++) |
473 | pt[i] = 0; | |
7a31f6f4 | 474 | |
89184651 | 475 | smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT); |
7a31f6f4 | 476 | |
89184651 | 477 | pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); |
7a31f6f4 | 478 | |
89184651 TR |
479 | smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4); |
480 | smmu_flush_ptc(smmu, as->pd, pde << 2); | |
481 | smmu_flush_tlb_section(smmu, as->id, iova); | |
482 | smmu_flush(smmu); | |
483 | } else { | |
484 | page = pfn_to_page(pd[pde] & SMMU_PFN_MASK); | |
485 | pt = page_address(page); | |
7a31f6f4 HD |
486 | } |
487 | ||
89184651 | 488 | *pagep = page; |
7a31f6f4 | 489 | |
89184651 TR |
490 | /* Keep track of entries in this page table. */ |
491 | count = page_address(as->count); | |
492 | if (pt[pte] == 0) | |
493 | count[pde]++; | |
7a31f6f4 | 494 | |
89184651 TR |
495 | return &pt[pte]; |
496 | } | |
39abf8aa | 497 | |
89184651 | 498 | static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova) |
39abf8aa | 499 | { |
89184651 TR |
500 | u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; |
501 | u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; | |
502 | u32 *count = page_address(as->count); | |
503 | u32 *pd = page_address(as->pd), *pt; | |
504 | struct page *page; | |
39abf8aa | 505 | |
89184651 TR |
506 | page = pfn_to_page(pd[pde] & SMMU_PFN_MASK); |
507 | pt = page_address(page); | |
39abf8aa | 508 | |
89184651 TR |
509 | /* |
510 | * When no entries in this page table are used anymore, return the | |
511 | * memory page to the system. | |
512 | */ | |
513 | if (pt[pte] != 0) { | |
514 | if (--count[pde] == 0) { | |
515 | ClearPageReserved(page); | |
516 | __free_page(page); | |
517 | pd[pde] = 0; | |
518 | } | |
39abf8aa | 519 | |
89184651 | 520 | pt[pte] = 0; |
39abf8aa | 521 | } |
39abf8aa HD |
522 | } |
523 | ||
89184651 TR |
524 | static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, |
525 | phys_addr_t paddr, size_t size, int prot) | |
39abf8aa | 526 | { |
89184651 TR |
527 | struct tegra_smmu_as *as = domain->priv; |
528 | struct tegra_smmu *smmu = as->smmu; | |
529 | unsigned long offset; | |
530 | struct page *page; | |
531 | u32 *pte; | |
39abf8aa | 532 | |
89184651 TR |
533 | pte = as_get_pte(as, iova, &page); |
534 | if (!pte) | |
535 | return -ENOMEM; | |
39abf8aa | 536 | |
89184651 TR |
537 | *pte = __phys_to_pfn(paddr) | SMMU_PTE_ATTR; |
538 | offset = offset_in_page(pte); | |
39abf8aa | 539 | |
89184651 TR |
540 | smmu->soc->ops->flush_dcache(page, offset, 4); |
541 | smmu_flush_ptc(smmu, page, offset); | |
542 | smmu_flush_tlb_group(smmu, as->id, iova); | |
543 | smmu_flush(smmu); | |
39abf8aa | 544 | |
39abf8aa HD |
545 | return 0; |
546 | } | |
547 | ||
89184651 TR |
548 | static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, |
549 | size_t size) | |
39abf8aa | 550 | { |
89184651 TR |
551 | struct tegra_smmu_as *as = domain->priv; |
552 | struct tegra_smmu *smmu = as->smmu; | |
553 | unsigned long offset; | |
554 | struct page *page; | |
555 | u32 *pte; | |
39abf8aa | 556 | |
89184651 TR |
557 | pte = as_get_pte(as, iova, &page); |
558 | if (!pte) | |
559 | return 0; | |
39abf8aa | 560 | |
89184651 TR |
561 | offset = offset_in_page(pte); |
562 | as_put_pte(as, iova); | |
563 | ||
564 | smmu->soc->ops->flush_dcache(page, offset, 4); | |
565 | smmu_flush_ptc(smmu, page, offset); | |
566 | smmu_flush_tlb_group(smmu, as->id, iova); | |
567 | smmu_flush(smmu); | |
568 | ||
569 | return size; | |
39abf8aa HD |
570 | } |
571 | ||
89184651 TR |
572 | static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, |
573 | dma_addr_t iova) | |
39abf8aa | 574 | { |
89184651 TR |
575 | struct tegra_smmu_as *as = domain->priv; |
576 | struct page *page; | |
577 | unsigned long pfn; | |
578 | u32 *pte; | |
39abf8aa | 579 | |
89184651 TR |
580 | pte = as_get_pte(as, iova, &page); |
581 | pfn = *pte & SMMU_PFN_MASK; | |
39abf8aa | 582 | |
89184651 | 583 | return PFN_PHYS(pfn); |
39abf8aa HD |
584 | } |
585 | ||
89184651 | 586 | static struct tegra_smmu *tegra_smmu_find(struct device_node *np) |
7a31f6f4 | 587 | { |
89184651 TR |
588 | struct platform_device *pdev; |
589 | struct tegra_mc *mc; | |
7a31f6f4 | 590 | |
89184651 TR |
591 | pdev = of_find_device_by_node(np); |
592 | if (!pdev) | |
593 | return NULL; | |
594 | ||
595 | mc = platform_get_drvdata(pdev); | |
596 | if (!mc) | |
597 | return NULL; | |
598 | ||
599 | return mc->smmu; | |
7a31f6f4 HD |
600 | } |
601 | ||
89184651 | 602 | static int tegra_smmu_add_device(struct device *dev) |
7a31f6f4 | 603 | { |
89184651 TR |
604 | struct device_node *np = dev->of_node; |
605 | struct of_phandle_args args; | |
606 | unsigned int index = 0; | |
7a31f6f4 | 607 | |
89184651 TR |
608 | while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, |
609 | &args) == 0) { | |
610 | struct tegra_smmu *smmu; | |
611 | ||
612 | smmu = tegra_smmu_find(args.np); | |
613 | if (smmu) { | |
614 | /* | |
615 | * Only a single IOMMU master interface is currently | |
616 | * supported by the Linux kernel, so abort after the | |
617 | * first match. | |
618 | */ | |
619 | dev->archdata.iommu = smmu; | |
620 | break; | |
621 | } | |
622 | ||
623 | index++; | |
624 | } | |
625 | ||
626 | return 0; | |
7a31f6f4 HD |
627 | } |
628 | ||
89184651 | 629 | static void tegra_smmu_remove_device(struct device *dev) |
7a31f6f4 | 630 | { |
89184651 TR |
631 | dev->archdata.iommu = NULL; |
632 | } | |
7a31f6f4 | 633 | |
89184651 TR |
634 | static const struct iommu_ops tegra_smmu_ops = { |
635 | .capable = tegra_smmu_capable, | |
636 | .domain_init = tegra_smmu_domain_init, | |
637 | .domain_destroy = tegra_smmu_domain_destroy, | |
638 | .attach_dev = tegra_smmu_attach_dev, | |
639 | .detach_dev = tegra_smmu_detach_dev, | |
640 | .add_device = tegra_smmu_add_device, | |
641 | .remove_device = tegra_smmu_remove_device, | |
642 | .map = tegra_smmu_map, | |
643 | .unmap = tegra_smmu_unmap, | |
644 | .map_sg = default_iommu_map_sg, | |
645 | .iova_to_phys = tegra_smmu_iova_to_phys, | |
7a31f6f4 | 646 | |
89184651 TR |
647 | .pgsize_bitmap = SZ_4K, |
648 | }; | |
7a31f6f4 | 649 | |
89184651 TR |
650 | static void tegra_smmu_ahb_enable(void) |
651 | { | |
652 | static const struct of_device_id ahb_match[] = { | |
653 | { .compatible = "nvidia,tegra30-ahb", }, | |
654 | { } | |
655 | }; | |
656 | struct device_node *ahb; | |
7a31f6f4 | 657 | |
89184651 TR |
658 | ahb = of_find_matching_node(NULL, ahb_match); |
659 | if (ahb) { | |
660 | tegra_ahb_enable_smmu(ahb); | |
661 | of_node_put(ahb); | |
7a31f6f4 | 662 | } |
89184651 | 663 | } |
7a31f6f4 | 664 | |
89184651 TR |
665 | struct tegra_smmu *tegra_smmu_probe(struct device *dev, |
666 | const struct tegra_smmu_soc *soc, | |
667 | struct tegra_mc *mc) | |
668 | { | |
669 | struct tegra_smmu *smmu; | |
670 | size_t size; | |
671 | u32 value; | |
672 | int err; | |
7a31f6f4 | 673 | |
89184651 TR |
674 | /* This can happen on Tegra20 which doesn't have an SMMU */ |
675 | if (!soc) | |
676 | return NULL; | |
0760e8fa | 677 | |
89184651 TR |
678 | smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); |
679 | if (!smmu) | |
680 | return ERR_PTR(-ENOMEM); | |
0760e8fa | 681 | |
89184651 TR |
682 | /* |
683 | * This is a bit of a hack. Ideally we'd want to simply return this | |
684 | * value. However the IOMMU registration process will attempt to add | |
685 | * all devices to the IOMMU when bus_set_iommu() is called. In order | |
686 | * not to rely on global variables to track the IOMMU instance, we | |
687 | * set it here so that it can be looked up from the .add_device() | |
688 | * callback via the IOMMU device's .drvdata field. | |
689 | */ | |
690 | mc->smmu = smmu; | |
0760e8fa | 691 | |
89184651 | 692 | size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); |
0760e8fa | 693 | |
89184651 TR |
694 | smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); |
695 | if (!smmu->asids) | |
696 | return ERR_PTR(-ENOMEM); | |
7a31f6f4 | 697 | |
89184651 | 698 | mutex_init(&smmu->lock); |
7a31f6f4 | 699 | |
89184651 TR |
700 | smmu->regs = mc->regs; |
701 | smmu->soc = soc; | |
702 | smmu->dev = dev; | |
703 | smmu->mc = mc; | |
7a31f6f4 | 704 | |
89184651 | 705 | value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); |
7a31f6f4 | 706 | |
89184651 TR |
707 | if (soc->supports_request_limit) |
708 | value |= SMMU_PTC_CONFIG_REQ_LIMIT(8); | |
39abf8aa | 709 | |
89184651 | 710 | smmu_writel(smmu, value, SMMU_PTC_CONFIG); |
7a31f6f4 | 711 | |
89184651 TR |
712 | value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | |
713 | SMMU_TLB_CONFIG_ACTIVE_LINES(0x20); | |
7a31f6f4 | 714 | |
89184651 TR |
715 | if (soc->supports_round_robin_arbitration) |
716 | value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; | |
7a31f6f4 | 717 | |
89184651 | 718 | smmu_writel(smmu, value, SMMU_TLB_CONFIG); |
7a31f6f4 | 719 | |
89184651 TR |
720 | smmu_flush_ptc(smmu, NULL, 0); |
721 | smmu_flush_tlb(smmu); | |
722 | smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); | |
723 | smmu_flush(smmu); | |
724 | ||
725 | tegra_smmu_ahb_enable(); | |
7a31f6f4 | 726 | |
89184651 TR |
727 | err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); |
728 | if (err < 0) | |
729 | return ERR_PTR(err); | |
7a31f6f4 | 730 | |
89184651 TR |
731 | return smmu; |
732 | } |