Commit | Line | Data |
---|---|---|
d53e54b4 | 1 | /* |
70722309 | 2 | * IOMMU API for Graphics Address Relocation Table on Tegra20 |
d53e54b4 HD |
3 | * |
4 | * Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved. | |
5 | * | |
39fcbbcc PG |
6 | * Author: Hiroshi DOYU <hdoyu@nvidia.com> |
7 | * | |
d53e54b4 HD |
8 | * This program is free software; you can redistribute it and/or modify it |
9 | * under the terms and conditions of the GNU General Public License, | |
10 | * version 2, as published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope it will be useful, but WITHOUT | |
13 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
15 | * more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License along with | |
18 | * this program; if not, write to the Free Software Foundation, Inc., | |
19 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
20 | */ | |
21 | ||
5dd82cdb DO |
22 | #define dev_fmt(fmt) "gart: " fmt |
23 | ||
4f821c10 DO |
24 | #include <linux/io.h> |
25 | #include <linux/iommu.h> | |
39fcbbcc | 26 | #include <linux/moduleparam.h> |
ce2785a7 | 27 | #include <linux/platform_device.h> |
d53e54b4 | 28 | #include <linux/slab.h> |
4f821c10 | 29 | #include <linux/spinlock.h> |
d53e54b4 | 30 | #include <linux/vmalloc.h> |
d53e54b4 | 31 | |
ce2785a7 DO |
32 | #include <soc/tegra/mc.h> |
33 | ||
774dfc9b HD |
34 | #define GART_REG_BASE 0x24 |
35 | #define GART_CONFIG (0x24 - GART_REG_BASE) | |
36 | #define GART_ENTRY_ADDR (0x28 - GART_REG_BASE) | |
37 | #define GART_ENTRY_DATA (0x2c - GART_REG_BASE) | |
70722309 DO |
38 | |
39 | #define GART_ENTRY_PHYS_ADDR_VALID BIT(31) | |
d53e54b4 HD |
40 | |
41 | #define GART_PAGE_SHIFT 12 | |
42 | #define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT) | |
70722309 DO |
43 | #define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT) |
44 | ||
45 | /* bitmap of the page sizes currently supported */ | |
46 | #define GART_IOMMU_PGSIZES (GART_PAGE_SIZE) | |
d53e54b4 | 47 | |
d53e54b4 HD |
48 | struct gart_device { |
49 | void __iomem *regs; | |
50 | u32 *savedata; | |
70722309 DO |
51 | unsigned long iovmm_base; /* offset to vmm_area start */ |
52 | unsigned long iovmm_end; /* offset to vmm_area end */ | |
d53e54b4 | 53 | spinlock_t pte_lock; /* for pagetable */ |
e7e23670 DO |
54 | spinlock_t dom_lock; /* for active domain */ |
55 | unsigned int active_devices; /* number of active devices */ | |
7d849b7b | 56 | struct iommu_domain *active_domain; /* current active domain */ |
c184ae83 | 57 | struct iommu_device iommu; /* IOMMU Core handle */ |
70722309 | 58 | struct device *dev; |
d53e54b4 HD |
59 | }; |
60 | ||
61 | static struct gart_device *gart_handle; /* unique for a system */ | |
62 | ||
40c9b882 DO |
63 | static bool gart_debug; |
64 | ||
d53e54b4 HD |
65 | /* |
66 | * Any interaction between any block on PPSB and a block on APB or AHB | |
67 | * must have these read-back to ensure the APB/AHB bus transaction is | |
68 | * complete before initiating activity on the PPSB block. | |
69 | */ | |
70722309 | 70 | #define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG) |
d53e54b4 HD |
71 | |
72 | #define for_each_gart_pte(gart, iova) \ | |
73 | for (iova = gart->iovmm_base; \ | |
70722309 | 74 | iova < gart->iovmm_end; \ |
d53e54b4 HD |
75 | iova += GART_PAGE_SIZE) |
76 | ||
77 | static inline void gart_set_pte(struct gart_device *gart, | |
70722309 | 78 | unsigned long iova, unsigned long pte) |
d53e54b4 | 79 | { |
70722309 DO |
80 | writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); |
81 | writel_relaxed(pte, gart->regs + GART_ENTRY_DATA); | |
d53e54b4 HD |
82 | } |
83 | ||
84 | static inline unsigned long gart_read_pte(struct gart_device *gart, | |
70722309 | 85 | unsigned long iova) |
d53e54b4 HD |
86 | { |
87 | unsigned long pte; | |
88 | ||
70722309 DO |
89 | writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); |
90 | pte = readl_relaxed(gart->regs + GART_ENTRY_DATA); | |
d53e54b4 HD |
91 | |
92 | return pte; | |
93 | } | |
94 | ||
95 | static void do_gart_setup(struct gart_device *gart, const u32 *data) | |
96 | { | |
97 | unsigned long iova; | |
98 | ||
99 | for_each_gart_pte(gart, iova) | |
100 | gart_set_pte(gart, iova, data ? *(data++) : 0); | |
101 | ||
70722309 | 102 | writel_relaxed(1, gart->regs + GART_CONFIG); |
d53e54b4 HD |
103 | FLUSH_GART_REGS(gart); |
104 | } | |
105 | ||
70722309 DO |
106 | static inline bool gart_iova_range_invalid(struct gart_device *gart, |
107 | unsigned long iova, size_t bytes) | |
d53e54b4 | 108 | { |
70722309 DO |
109 | return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE || |
110 | iova + bytes > gart->iovmm_end); | |
d53e54b4 | 111 | } |
d53e54b4 | 112 | |
70722309 | 113 | static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova) |
d53e54b4 | 114 | { |
70722309 | 115 | return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID); |
d53e54b4 HD |
116 | } |
117 | ||
118 | static int gart_iommu_attach_dev(struct iommu_domain *domain, | |
119 | struct device *dev) | |
120 | { | |
cc0e1205 | 121 | struct gart_device *gart = gart_handle; |
e7e23670 | 122 | int ret = 0; |
d53e54b4 | 123 | |
e7e23670 | 124 | spin_lock(&gart->dom_lock); |
d53e54b4 | 125 | |
e7e23670 DO |
126 | if (gart->active_domain && gart->active_domain != domain) { |
127 | ret = -EBUSY; | |
128 | } else if (dev->archdata.iommu != domain) { | |
129 | dev->archdata.iommu = domain; | |
130 | gart->active_domain = domain; | |
131 | gart->active_devices++; | |
d53e54b4 | 132 | } |
c3086fad | 133 | |
e7e23670 DO |
134 | spin_unlock(&gart->dom_lock); |
135 | ||
136 | return ret; | |
c3086fad DO |
137 | } |
138 | ||
139 | static void gart_iommu_detach_dev(struct iommu_domain *domain, | |
140 | struct device *dev) | |
141 | { | |
e7e23670 DO |
142 | struct gart_device *gart = gart_handle; |
143 | ||
144 | spin_lock(&gart->dom_lock); | |
c3086fad | 145 | |
e7e23670 DO |
146 | if (dev->archdata.iommu == domain) { |
147 | dev->archdata.iommu = NULL; | |
148 | ||
149 | if (--gart->active_devices == 0) | |
150 | gart->active_domain = NULL; | |
151 | } | |
152 | ||
153 | spin_unlock(&gart->dom_lock); | |
d53e54b4 HD |
154 | } |
155 | ||
b5cbb386 | 156 | static struct iommu_domain *gart_iommu_domain_alloc(unsigned type) |
d53e54b4 | 157 | { |
e7e23670 | 158 | struct iommu_domain *domain; |
d53e54b4 | 159 | |
b5cbb386 JR |
160 | if (type != IOMMU_DOMAIN_UNMANAGED) |
161 | return NULL; | |
d53e54b4 | 162 | |
e7e23670 DO |
163 | domain = kzalloc(sizeof(*domain), GFP_KERNEL); |
164 | if (domain) { | |
70722309 DO |
165 | domain->geometry.aperture_start = gart_handle->iovmm_base; |
166 | domain->geometry.aperture_end = gart_handle->iovmm_end - 1; | |
e7e23670 DO |
167 | domain->geometry.force_aperture = true; |
168 | } | |
836a8ac9 | 169 | |
e7e23670 | 170 | return domain; |
d53e54b4 HD |
171 | } |
172 | ||
b5cbb386 | 173 | static void gart_iommu_domain_free(struct iommu_domain *domain) |
d53e54b4 | 174 | { |
e7e23670 DO |
175 | WARN_ON(gart_handle->active_domain == domain); |
176 | kfree(domain); | |
d53e54b4 HD |
177 | } |
178 | ||
70722309 DO |
179 | static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova, |
180 | unsigned long pa) | |
181 | { | |
182 | if (unlikely(gart_debug && gart_pte_valid(gart, iova))) { | |
183 | dev_err(gart->dev, "Page entry is in-use\n"); | |
184 | return -EINVAL; | |
185 | } | |
186 | ||
187 | gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa); | |
188 | ||
189 | return 0; | |
190 | } | |
191 | ||
d53e54b4 HD |
192 | static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova, |
193 | phys_addr_t pa, size_t bytes, int prot) | |
194 | { | |
e7e23670 | 195 | struct gart_device *gart = gart_handle; |
70722309 | 196 | int ret; |
d53e54b4 | 197 | |
70722309 | 198 | if (gart_iova_range_invalid(gart, iova, bytes)) |
d53e54b4 HD |
199 | return -EINVAL; |
200 | ||
70722309 DO |
201 | spin_lock(&gart->pte_lock); |
202 | ret = __gart_iommu_map(gart, iova, (unsigned long)pa); | |
203 | spin_unlock(&gart->pte_lock); | |
204 | ||
205 | return ret; | |
206 | } | |
207 | ||
208 | static inline int __gart_iommu_unmap(struct gart_device *gart, | |
209 | unsigned long iova) | |
210 | { | |
211 | if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) { | |
212 | dev_err(gart->dev, "Page entry is invalid\n"); | |
d53e54b4 HD |
213 | return -EINVAL; |
214 | } | |
70722309 DO |
215 | |
216 | gart_set_pte(gart, iova, 0); | |
217 | ||
d53e54b4 HD |
218 | return 0; |
219 | } | |
220 | ||
221 | static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, | |
222 | size_t bytes) | |
223 | { | |
e7e23670 | 224 | struct gart_device *gart = gart_handle; |
70722309 | 225 | int err; |
d53e54b4 | 226 | |
70722309 | 227 | if (gart_iova_range_invalid(gart, iova, bytes)) |
d53e54b4 HD |
228 | return 0; |
229 | ||
70722309 DO |
230 | spin_lock(&gart->pte_lock); |
231 | err = __gart_iommu_unmap(gart, iova); | |
232 | spin_unlock(&gart->pte_lock); | |
233 | ||
234 | return err ? 0 : bytes; | |
d53e54b4 HD |
235 | } |
236 | ||
237 | static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, | |
bb5547ac | 238 | dma_addr_t iova) |
d53e54b4 | 239 | { |
e7e23670 | 240 | struct gart_device *gart = gart_handle; |
d53e54b4 | 241 | unsigned long pte; |
d53e54b4 | 242 | |
70722309 | 243 | if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE)) |
d53e54b4 HD |
244 | return -EINVAL; |
245 | ||
70722309 | 246 | spin_lock(&gart->pte_lock); |
d53e54b4 | 247 | pte = gart_read_pte(gart, iova); |
70722309 | 248 | spin_unlock(&gart->pte_lock); |
d53e54b4 | 249 | |
70722309 | 250 | return pte & GART_PAGE_MASK; |
d53e54b4 HD |
251 | } |
252 | ||
7c2aa644 | 253 | static bool gart_iommu_capable(enum iommu_cap cap) |
d53e54b4 | 254 | { |
7c2aa644 | 255 | return false; |
d53e54b4 HD |
256 | } |
257 | ||
15f9a310 RM |
258 | static int gart_iommu_add_device(struct device *dev) |
259 | { | |
4b6f0ea3 | 260 | struct iommu_group *group; |
15f9a310 | 261 | |
4b6f0ea3 DO |
262 | if (!dev->iommu_fwspec) |
263 | return -ENODEV; | |
264 | ||
265 | group = iommu_group_get_for_dev(dev); | |
15f9a310 RM |
266 | if (IS_ERR(group)) |
267 | return PTR_ERR(group); | |
268 | ||
269 | iommu_group_put(group); | |
c184ae83 JR |
270 | |
271 | iommu_device_link(&gart_handle->iommu, dev); | |
272 | ||
15f9a310 RM |
273 | return 0; |
274 | } | |
275 | ||
276 | static void gart_iommu_remove_device(struct device *dev) | |
277 | { | |
278 | iommu_group_remove_device(dev); | |
c184ae83 | 279 | iommu_device_unlink(&gart_handle->iommu, dev); |
15f9a310 RM |
280 | } |
281 | ||
4b6f0ea3 DO |
282 | static int gart_iommu_of_xlate(struct device *dev, |
283 | struct of_phandle_args *args) | |
284 | { | |
285 | return 0; | |
286 | } | |
287 | ||
2fc0ac18 DO |
288 | static void gart_iommu_sync(struct iommu_domain *domain) |
289 | { | |
70722309 | 290 | FLUSH_GART_REGS(gart_handle); |
2fc0ac18 DO |
291 | } |
292 | ||
b22f6434 | 293 | static const struct iommu_ops gart_iommu_ops = { |
7c2aa644 | 294 | .capable = gart_iommu_capable, |
b5cbb386 JR |
295 | .domain_alloc = gart_iommu_domain_alloc, |
296 | .domain_free = gart_iommu_domain_free, | |
d53e54b4 HD |
297 | .attach_dev = gart_iommu_attach_dev, |
298 | .detach_dev = gart_iommu_detach_dev, | |
15f9a310 RM |
299 | .add_device = gart_iommu_add_device, |
300 | .remove_device = gart_iommu_remove_device, | |
301 | .device_group = generic_device_group, | |
d53e54b4 HD |
302 | .map = gart_iommu_map, |
303 | .unmap = gart_iommu_unmap, | |
304 | .iova_to_phys = gart_iommu_iova_to_phys, | |
d53e54b4 | 305 | .pgsize_bitmap = GART_IOMMU_PGSIZES, |
4b6f0ea3 | 306 | .of_xlate = gart_iommu_of_xlate, |
2fc0ac18 DO |
307 | .iotlb_sync_map = gart_iommu_sync, |
308 | .iotlb_sync = gart_iommu_sync, | |
d53e54b4 HD |
309 | }; |
310 | ||
ce2785a7 | 311 | int tegra_gart_suspend(struct gart_device *gart) |
d53e54b4 | 312 | { |
d53e54b4 | 313 | u32 *data = gart->savedata; |
70722309 DO |
314 | unsigned long iova; |
315 | ||
316 | /* | |
317 | * All GART users shall be suspended at this point. Disable | |
318 | * address translation to trap all GART accesses as invalid | |
319 | * memory accesses. | |
320 | */ | |
321 | writel_relaxed(0, gart->regs + GART_CONFIG); | |
322 | FLUSH_GART_REGS(gart); | |
d53e54b4 | 323 | |
d53e54b4 HD |
324 | for_each_gart_pte(gart, iova) |
325 | *(data++) = gart_read_pte(gart, iova); | |
70722309 | 326 | |
d53e54b4 HD |
327 | return 0; |
328 | } | |
329 | ||
ce2785a7 | 330 | int tegra_gart_resume(struct gart_device *gart) |
d53e54b4 | 331 | { |
d53e54b4 | 332 | do_gart_setup(gart, gart->savedata); |
70722309 | 333 | |
d53e54b4 HD |
334 | return 0; |
335 | } | |
336 | ||
ce2785a7 | 337 | struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc) |
d53e54b4 HD |
338 | { |
339 | struct gart_device *gart; | |
70722309 DO |
340 | struct resource *res; |
341 | int err; | |
d53e54b4 | 342 | |
d53e54b4 HD |
343 | BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT); |
344 | ||
345 | /* the GART memory aperture is required */ | |
70722309 DO |
346 | res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1); |
347 | if (!res) { | |
348 | dev_err(dev, "Memory aperture resource unavailable\n"); | |
ce2785a7 | 349 | return ERR_PTR(-ENXIO); |
d53e54b4 HD |
350 | } |
351 | ||
167d67d5 | 352 | gart = kzalloc(sizeof(*gart), GFP_KERNEL); |
70722309 | 353 | if (!gart) |
ce2785a7 | 354 | return ERR_PTR(-ENOMEM); |
d53e54b4 | 355 | |
70722309 DO |
356 | gart_handle = gart; |
357 | ||
358 | gart->dev = dev; | |
359 | gart->regs = mc->regs + GART_REG_BASE; | |
360 | gart->iovmm_base = res->start; | |
361 | gart->iovmm_end = res->end + 1; | |
362 | spin_lock_init(&gart->pte_lock); | |
363 | spin_lock_init(&gart->dom_lock); | |
364 | ||
365 | do_gart_setup(gart, NULL); | |
366 | ||
367 | err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart"); | |
368 | if (err) | |
167d67d5 | 369 | goto free_gart; |
c184ae83 JR |
370 | |
371 | iommu_device_set_ops(&gart->iommu, &gart_iommu_ops); | |
4b6f0ea3 | 372 | iommu_device_set_fwnode(&gart->iommu, dev->fwnode); |
c184ae83 | 373 | |
70722309 DO |
374 | err = iommu_device_register(&gart->iommu); |
375 | if (err) | |
ae95c46d | 376 | goto remove_sysfs; |
c184ae83 | 377 | |
70722309 DO |
378 | gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE * |
379 | sizeof(u32)); | |
d53e54b4 | 380 | if (!gart->savedata) { |
70722309 | 381 | err = -ENOMEM; |
ae95c46d | 382 | goto unregister_iommu; |
d53e54b4 HD |
383 | } |
384 | ||
ce2785a7 | 385 | return gart; |
ae95c46d DO |
386 | |
387 | unregister_iommu: | |
388 | iommu_device_unregister(&gart->iommu); | |
389 | remove_sysfs: | |
390 | iommu_device_sysfs_remove(&gart->iommu); | |
167d67d5 DO |
391 | free_gart: |
392 | kfree(gart); | |
ae95c46d | 393 | |
70722309 | 394 | return ERR_PTR(err); |
d53e54b4 | 395 | } |
d53e54b4 | 396 | |
39fcbbcc | 397 | module_param(gart_debug, bool, 0644); |
40c9b882 | 398 | MODULE_PARM_DESC(gart_debug, "Enable GART debugging"); |