Commit | Line | Data |
---|---|---|
a61127c2 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
d53e54b4 | 2 | /* |
70722309 | 3 | * IOMMU API for Graphics Address Relocation Table on Tegra20 |
d53e54b4 HD |
4 | * |
5 | * Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved. | |
6 | * | |
39fcbbcc | 7 | * Author: Hiroshi DOYU <hdoyu@nvidia.com> |
d53e54b4 HD |
8 | */ |
9 | ||
5dd82cdb DO |
10 | #define dev_fmt(fmt) "gart: " fmt |
11 | ||
4f821c10 DO |
12 | #include <linux/io.h> |
13 | #include <linux/iommu.h> | |
39fcbbcc | 14 | #include <linux/moduleparam.h> |
ce2785a7 | 15 | #include <linux/platform_device.h> |
d53e54b4 | 16 | #include <linux/slab.h> |
4f821c10 | 17 | #include <linux/spinlock.h> |
d53e54b4 | 18 | #include <linux/vmalloc.h> |
d53e54b4 | 19 | |
ce2785a7 DO |
20 | #include <soc/tegra/mc.h> |
21 | ||
774dfc9b HD |
22 | #define GART_REG_BASE 0x24 |
23 | #define GART_CONFIG (0x24 - GART_REG_BASE) | |
24 | #define GART_ENTRY_ADDR (0x28 - GART_REG_BASE) | |
25 | #define GART_ENTRY_DATA (0x2c - GART_REG_BASE) | |
70722309 DO |
26 | |
27 | #define GART_ENTRY_PHYS_ADDR_VALID BIT(31) | |
d53e54b4 HD |
28 | |
29 | #define GART_PAGE_SHIFT 12 | |
30 | #define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT) | |
70722309 DO |
31 | #define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT) |
32 | ||
33 | /* bitmap of the page sizes currently supported */ | |
34 | #define GART_IOMMU_PGSIZES (GART_PAGE_SIZE) | |
d53e54b4 | 35 | |
d53e54b4 HD |
36 | struct gart_device { |
37 | void __iomem *regs; | |
38 | u32 *savedata; | |
70722309 DO |
39 | unsigned long iovmm_base; /* offset to vmm_area start */ |
40 | unsigned long iovmm_end; /* offset to vmm_area end */ | |
d53e54b4 | 41 | spinlock_t pte_lock; /* for pagetable */ |
e7e23670 DO |
42 | spinlock_t dom_lock; /* for active domain */ |
43 | unsigned int active_devices; /* number of active devices */ | |
7d849b7b | 44 | struct iommu_domain *active_domain; /* current active domain */ |
c184ae83 | 45 | struct iommu_device iommu; /* IOMMU Core handle */ |
70722309 | 46 | struct device *dev; |
d53e54b4 HD |
47 | }; |
48 | ||
49 | static struct gart_device *gart_handle; /* unique for a system */ | |
50 | ||
40c9b882 DO |
51 | static bool gart_debug; |
52 | ||
d53e54b4 HD |
53 | /* |
54 | * Any interaction between any block on PPSB and a block on APB or AHB | |
55 | * must have these read-back to ensure the APB/AHB bus transaction is | |
56 | * complete before initiating activity on the PPSB block. | |
57 | */ | |
70722309 | 58 | #define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG) |
d53e54b4 HD |
59 | |
60 | #define for_each_gart_pte(gart, iova) \ | |
61 | for (iova = gart->iovmm_base; \ | |
70722309 | 62 | iova < gart->iovmm_end; \ |
d53e54b4 HD |
63 | iova += GART_PAGE_SIZE) |
64 | ||
65 | static inline void gart_set_pte(struct gart_device *gart, | |
70722309 | 66 | unsigned long iova, unsigned long pte) |
d53e54b4 | 67 | { |
70722309 DO |
68 | writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); |
69 | writel_relaxed(pte, gart->regs + GART_ENTRY_DATA); | |
d53e54b4 HD |
70 | } |
71 | ||
72 | static inline unsigned long gart_read_pte(struct gart_device *gart, | |
70722309 | 73 | unsigned long iova) |
d53e54b4 HD |
74 | { |
75 | unsigned long pte; | |
76 | ||
70722309 DO |
77 | writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); |
78 | pte = readl_relaxed(gart->regs + GART_ENTRY_DATA); | |
d53e54b4 HD |
79 | |
80 | return pte; | |
81 | } | |
82 | ||
83 | static void do_gart_setup(struct gart_device *gart, const u32 *data) | |
84 | { | |
85 | unsigned long iova; | |
86 | ||
87 | for_each_gart_pte(gart, iova) | |
88 | gart_set_pte(gart, iova, data ? *(data++) : 0); | |
89 | ||
70722309 | 90 | writel_relaxed(1, gart->regs + GART_CONFIG); |
d53e54b4 HD |
91 | FLUSH_GART_REGS(gart); |
92 | } | |
93 | ||
70722309 DO |
94 | static inline bool gart_iova_range_invalid(struct gart_device *gart, |
95 | unsigned long iova, size_t bytes) | |
d53e54b4 | 96 | { |
70722309 DO |
97 | return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE || |
98 | iova + bytes > gart->iovmm_end); | |
d53e54b4 | 99 | } |
d53e54b4 | 100 | |
70722309 | 101 | static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova) |
d53e54b4 | 102 | { |
70722309 | 103 | return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID); |
d53e54b4 HD |
104 | } |
105 | ||
106 | static int gart_iommu_attach_dev(struct iommu_domain *domain, | |
107 | struct device *dev) | |
108 | { | |
cc0e1205 | 109 | struct gart_device *gart = gart_handle; |
e7e23670 | 110 | int ret = 0; |
d53e54b4 | 111 | |
e7e23670 | 112 | spin_lock(&gart->dom_lock); |
d53e54b4 | 113 | |
e7e23670 | 114 | if (gart->active_domain && gart->active_domain != domain) { |
f4a14773 | 115 | ret = -EINVAL; |
a5616e24 JR |
116 | } else if (dev_iommu_priv_get(dev) != domain) { |
117 | dev_iommu_priv_set(dev, domain); | |
e7e23670 DO |
118 | gart->active_domain = domain; |
119 | gart->active_devices++; | |
d53e54b4 | 120 | } |
c3086fad | 121 | |
e7e23670 DO |
122 | spin_unlock(&gart->dom_lock); |
123 | ||
124 | return ret; | |
c3086fad DO |
125 | } |
126 | ||
c1fe9119 | 127 | static void gart_iommu_set_platform_dma(struct device *dev) |
c3086fad | 128 | { |
c1fe9119 | 129 | struct iommu_domain *domain = iommu_get_domain_for_dev(dev); |
e7e23670 DO |
130 | struct gart_device *gart = gart_handle; |
131 | ||
132 | spin_lock(&gart->dom_lock); | |
c3086fad | 133 | |
a5616e24 JR |
134 | if (dev_iommu_priv_get(dev) == domain) { |
135 | dev_iommu_priv_set(dev, NULL); | |
e7e23670 DO |
136 | |
137 | if (--gart->active_devices == 0) | |
138 | gart->active_domain = NULL; | |
139 | } | |
140 | ||
141 | spin_unlock(&gart->dom_lock); | |
d53e54b4 HD |
142 | } |
143 | ||
b5cbb386 | 144 | static struct iommu_domain *gart_iommu_domain_alloc(unsigned type) |
d53e54b4 | 145 | { |
e7e23670 | 146 | struct iommu_domain *domain; |
d53e54b4 | 147 | |
b5cbb386 JR |
148 | if (type != IOMMU_DOMAIN_UNMANAGED) |
149 | return NULL; | |
d53e54b4 | 150 | |
e7e23670 DO |
151 | domain = kzalloc(sizeof(*domain), GFP_KERNEL); |
152 | if (domain) { | |
70722309 DO |
153 | domain->geometry.aperture_start = gart_handle->iovmm_base; |
154 | domain->geometry.aperture_end = gart_handle->iovmm_end - 1; | |
e7e23670 DO |
155 | domain->geometry.force_aperture = true; |
156 | } | |
836a8ac9 | 157 | |
e7e23670 | 158 | return domain; |
d53e54b4 HD |
159 | } |
160 | ||
b5cbb386 | 161 | static void gart_iommu_domain_free(struct iommu_domain *domain) |
d53e54b4 | 162 | { |
e7e23670 DO |
163 | WARN_ON(gart_handle->active_domain == domain); |
164 | kfree(domain); | |
d53e54b4 HD |
165 | } |
166 | ||
70722309 DO |
167 | static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova, |
168 | unsigned long pa) | |
169 | { | |
170 | if (unlikely(gart_debug && gart_pte_valid(gart, iova))) { | |
171 | dev_err(gart->dev, "Page entry is in-use\n"); | |
172 | return -EINVAL; | |
173 | } | |
174 | ||
175 | gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa); | |
176 | ||
177 | return 0; | |
178 | } | |
179 | ||
d53e54b4 | 180 | static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova, |
781ca2de | 181 | phys_addr_t pa, size_t bytes, int prot, gfp_t gfp) |
d53e54b4 | 182 | { |
e7e23670 | 183 | struct gart_device *gart = gart_handle; |
70722309 | 184 | int ret; |
d53e54b4 | 185 | |
70722309 | 186 | if (gart_iova_range_invalid(gart, iova, bytes)) |
d53e54b4 HD |
187 | return -EINVAL; |
188 | ||
70722309 DO |
189 | spin_lock(&gart->pte_lock); |
190 | ret = __gart_iommu_map(gart, iova, (unsigned long)pa); | |
191 | spin_unlock(&gart->pte_lock); | |
192 | ||
193 | return ret; | |
194 | } | |
195 | ||
196 | static inline int __gart_iommu_unmap(struct gart_device *gart, | |
197 | unsigned long iova) | |
198 | { | |
199 | if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) { | |
200 | dev_err(gart->dev, "Page entry is invalid\n"); | |
d53e54b4 HD |
201 | return -EINVAL; |
202 | } | |
70722309 DO |
203 | |
204 | gart_set_pte(gart, iova, 0); | |
205 | ||
d53e54b4 HD |
206 | return 0; |
207 | } | |
208 | ||
209 | static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, | |
56f8af5e | 210 | size_t bytes, struct iommu_iotlb_gather *gather) |
d53e54b4 | 211 | { |
e7e23670 | 212 | struct gart_device *gart = gart_handle; |
70722309 | 213 | int err; |
d53e54b4 | 214 | |
70722309 | 215 | if (gart_iova_range_invalid(gart, iova, bytes)) |
d53e54b4 HD |
216 | return 0; |
217 | ||
70722309 DO |
218 | spin_lock(&gart->pte_lock); |
219 | err = __gart_iommu_unmap(gart, iova); | |
220 | spin_unlock(&gart->pte_lock); | |
221 | ||
222 | return err ? 0 : bytes; | |
d53e54b4 HD |
223 | } |
224 | ||
225 | static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, | |
bb5547ac | 226 | dma_addr_t iova) |
d53e54b4 | 227 | { |
e7e23670 | 228 | struct gart_device *gart = gart_handle; |
d53e54b4 | 229 | unsigned long pte; |
d53e54b4 | 230 | |
70722309 | 231 | if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE)) |
d53e54b4 HD |
232 | return -EINVAL; |
233 | ||
70722309 | 234 | spin_lock(&gart->pte_lock); |
d53e54b4 | 235 | pte = gart_read_pte(gart, iova); |
70722309 | 236 | spin_unlock(&gart->pte_lock); |
d53e54b4 | 237 | |
70722309 | 238 | return pte & GART_PAGE_MASK; |
d53e54b4 HD |
239 | } |
240 | ||
b287ba73 | 241 | static struct iommu_device *gart_iommu_probe_device(struct device *dev) |
15f9a310 | 242 | { |
8c3d6923 | 243 | if (!dev_iommu_fwspec_get(dev)) |
b287ba73 | 244 | return ERR_PTR(-ENODEV); |
c184ae83 | 245 | |
b287ba73 | 246 | return &gart_handle->iommu; |
15f9a310 RM |
247 | } |
248 | ||
4b6f0ea3 DO |
249 | static int gart_iommu_of_xlate(struct device *dev, |
250 | struct of_phandle_args *args) | |
251 | { | |
252 | return 0; | |
253 | } | |
254 | ||
2ebbd258 YW |
255 | static void gart_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, |
256 | size_t size) | |
2fc0ac18 | 257 | { |
70722309 | 258 | FLUSH_GART_REGS(gart_handle); |
2fc0ac18 DO |
259 | } |
260 | ||
56f8af5e WD |
261 | static void gart_iommu_sync(struct iommu_domain *domain, |
262 | struct iommu_iotlb_gather *gather) | |
263 | { | |
862c3715 | 264 | size_t length = gather->end - gather->start + 1; |
2ebbd258 YW |
265 | |
266 | gart_iommu_sync_map(domain, gather->start, length); | |
56f8af5e WD |
267 | } |
268 | ||
b22f6434 | 269 | static const struct iommu_ops gart_iommu_ops = { |
b5cbb386 | 270 | .domain_alloc = gart_iommu_domain_alloc, |
b287ba73 | 271 | .probe_device = gart_iommu_probe_device, |
15f9a310 | 272 | .device_group = generic_device_group, |
c1fe9119 | 273 | .set_platform_dma_ops = gart_iommu_set_platform_dma, |
d53e54b4 | 274 | .pgsize_bitmap = GART_IOMMU_PGSIZES, |
4b6f0ea3 | 275 | .of_xlate = gart_iommu_of_xlate, |
9a630a4b LB |
276 | .default_domain_ops = &(const struct iommu_domain_ops) { |
277 | .attach_dev = gart_iommu_attach_dev, | |
9a630a4b LB |
278 | .map = gart_iommu_map, |
279 | .unmap = gart_iommu_unmap, | |
280 | .iova_to_phys = gart_iommu_iova_to_phys, | |
281 | .iotlb_sync_map = gart_iommu_sync_map, | |
282 | .iotlb_sync = gart_iommu_sync, | |
283 | .free = gart_iommu_domain_free, | |
284 | } | |
d53e54b4 HD |
285 | }; |
286 | ||
ce2785a7 | 287 | int tegra_gart_suspend(struct gart_device *gart) |
d53e54b4 | 288 | { |
d53e54b4 | 289 | u32 *data = gart->savedata; |
70722309 DO |
290 | unsigned long iova; |
291 | ||
292 | /* | |
293 | * All GART users shall be suspended at this point. Disable | |
294 | * address translation to trap all GART accesses as invalid | |
295 | * memory accesses. | |
296 | */ | |
297 | writel_relaxed(0, gart->regs + GART_CONFIG); | |
298 | FLUSH_GART_REGS(gart); | |
d53e54b4 | 299 | |
d53e54b4 HD |
300 | for_each_gart_pte(gart, iova) |
301 | *(data++) = gart_read_pte(gart, iova); | |
70722309 | 302 | |
d53e54b4 HD |
303 | return 0; |
304 | } | |
305 | ||
ce2785a7 | 306 | int tegra_gart_resume(struct gart_device *gart) |
d53e54b4 | 307 | { |
d53e54b4 | 308 | do_gart_setup(gart, gart->savedata); |
70722309 | 309 | |
d53e54b4 HD |
310 | return 0; |
311 | } | |
312 | ||
ce2785a7 | 313 | struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc) |
d53e54b4 HD |
314 | { |
315 | struct gart_device *gart; | |
70722309 DO |
316 | struct resource *res; |
317 | int err; | |
d53e54b4 | 318 | |
d53e54b4 HD |
319 | BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT); |
320 | ||
321 | /* the GART memory aperture is required */ | |
70722309 DO |
322 | res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1); |
323 | if (!res) { | |
324 | dev_err(dev, "Memory aperture resource unavailable\n"); | |
ce2785a7 | 325 | return ERR_PTR(-ENXIO); |
d53e54b4 HD |
326 | } |
327 | ||
167d67d5 | 328 | gart = kzalloc(sizeof(*gart), GFP_KERNEL); |
70722309 | 329 | if (!gart) |
ce2785a7 | 330 | return ERR_PTR(-ENOMEM); |
d53e54b4 | 331 | |
70722309 DO |
332 | gart_handle = gart; |
333 | ||
334 | gart->dev = dev; | |
335 | gart->regs = mc->regs + GART_REG_BASE; | |
336 | gart->iovmm_base = res->start; | |
337 | gart->iovmm_end = res->end + 1; | |
338 | spin_lock_init(&gart->pte_lock); | |
339 | spin_lock_init(&gart->dom_lock); | |
340 | ||
341 | do_gart_setup(gart, NULL); | |
342 | ||
343 | err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart"); | |
344 | if (err) | |
167d67d5 | 345 | goto free_gart; |
c184ae83 | 346 | |
2d471b20 | 347 | err = iommu_device_register(&gart->iommu, &gart_iommu_ops, dev); |
70722309 | 348 | if (err) |
ae95c46d | 349 | goto remove_sysfs; |
c184ae83 | 350 | |
70722309 DO |
351 | gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE * |
352 | sizeof(u32)); | |
d53e54b4 | 353 | if (!gart->savedata) { |
70722309 | 354 | err = -ENOMEM; |
ae95c46d | 355 | goto unregister_iommu; |
d53e54b4 HD |
356 | } |
357 | ||
ce2785a7 | 358 | return gart; |
ae95c46d DO |
359 | |
360 | unregister_iommu: | |
361 | iommu_device_unregister(&gart->iommu); | |
362 | remove_sysfs: | |
363 | iommu_device_sysfs_remove(&gart->iommu); | |
167d67d5 DO |
364 | free_gart: |
365 | kfree(gart); | |
ae95c46d | 366 | |
70722309 | 367 | return ERR_PTR(err); |
d53e54b4 | 368 | } |
d53e54b4 | 369 | |
39fcbbcc | 370 | module_param(gart_debug, bool, 0644); |
40c9b882 | 371 | MODULE_PARM_DESC(gart_debug, "Enable GART debugging"); |