Commit | Line | Data |
---|---|---|
76aaf220 DV |
1 | /* |
2 | * Copyright © 2010 Daniel Vetter | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include "drmP.h" | |
26 | #include "drm.h" | |
27 | #include "i915_drm.h" | |
28 | #include "i915_drv.h" | |
29 | #include "i915_trace.h" | |
30 | #include "intel_drv.h" | |
31 | ||
1d2a314c DV |
32 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
33 | static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, | |
34 | unsigned first_entry, | |
35 | unsigned num_entries) | |
36 | { | |
37 | int i, j; | |
38 | uint32_t *pt_vaddr; | |
39 | uint32_t scratch_pte; | |
40 | ||
41 | scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr); | |
42 | scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC; | |
43 | ||
44 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
45 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[i]); | |
46 | ||
47 | for (j = 0; j < I915_PPGTT_PT_ENTRIES; j++) | |
48 | pt_vaddr[j] = scratch_pte; | |
49 | ||
50 | kunmap_atomic(pt_vaddr); | |
51 | } | |
52 | ||
53 | } | |
54 | ||
55 | int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) | |
56 | { | |
57 | struct drm_i915_private *dev_priv = dev->dev_private; | |
58 | struct i915_hw_ppgtt *ppgtt; | |
59 | uint32_t pd_entry; | |
60 | unsigned first_pd_entry_in_global_pt; | |
61 | uint32_t __iomem *pd_addr; | |
62 | int i; | |
63 | int ret = -ENOMEM; | |
64 | ||
65 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 | |
66 | * entries. For aliasing ppgtt support we just steal them at the end for | |
67 | * now. */ | |
68 | first_pd_entry_in_global_pt = 512*1024 - I915_PPGTT_PD_ENTRIES; | |
69 | ||
70 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); | |
71 | if (!ppgtt) | |
72 | return ret; | |
73 | ||
74 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; | |
75 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, | |
76 | GFP_KERNEL); | |
77 | if (!ppgtt->pt_pages) | |
78 | goto err_ppgtt; | |
79 | ||
80 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
81 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); | |
82 | if (!ppgtt->pt_pages[i]) | |
83 | goto err_pt_alloc; | |
84 | } | |
85 | ||
86 | if (dev_priv->mm.gtt->needs_dmar) { | |
87 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) | |
88 | *ppgtt->num_pd_entries, | |
89 | GFP_KERNEL); | |
90 | if (!ppgtt->pt_dma_addr) | |
91 | goto err_pt_alloc; | |
92 | } | |
93 | ||
94 | pd_addr = dev_priv->mm.gtt->gtt + first_pd_entry_in_global_pt; | |
95 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
96 | dma_addr_t pt_addr; | |
97 | if (dev_priv->mm.gtt->needs_dmar) { | |
98 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], | |
99 | 0, 4096, | |
100 | PCI_DMA_BIDIRECTIONAL); | |
101 | ||
102 | if (pci_dma_mapping_error(dev->pdev, | |
103 | pt_addr)) { | |
104 | ret = -EIO; | |
105 | goto err_pd_pin; | |
106 | ||
107 | } | |
108 | ppgtt->pt_dma_addr[i] = pt_addr; | |
109 | } else | |
110 | pt_addr = page_to_phys(ppgtt->pt_pages[i]); | |
111 | ||
112 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); | |
113 | pd_entry |= GEN6_PDE_VALID; | |
114 | ||
115 | writel(pd_entry, pd_addr + i); | |
116 | } | |
117 | readl(pd_addr); | |
118 | ||
119 | ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; | |
120 | ||
121 | i915_ppgtt_clear_range(ppgtt, 0, | |
122 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); | |
123 | ||
124 | ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t); | |
125 | ||
126 | dev_priv->mm.aliasing_ppgtt = ppgtt; | |
127 | ||
128 | return 0; | |
129 | ||
130 | err_pd_pin: | |
131 | if (ppgtt->pt_dma_addr) { | |
132 | for (i--; i >= 0; i--) | |
133 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], | |
134 | 4096, PCI_DMA_BIDIRECTIONAL); | |
135 | } | |
136 | err_pt_alloc: | |
137 | kfree(ppgtt->pt_dma_addr); | |
138 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
139 | if (ppgtt->pt_pages[i]) | |
140 | __free_page(ppgtt->pt_pages[i]); | |
141 | } | |
142 | kfree(ppgtt->pt_pages); | |
143 | err_ppgtt: | |
144 | kfree(ppgtt); | |
145 | ||
146 | return ret; | |
147 | } | |
148 | ||
149 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) | |
150 | { | |
151 | struct drm_i915_private *dev_priv = dev->dev_private; | |
152 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; | |
153 | int i; | |
154 | ||
155 | if (!ppgtt) | |
156 | return; | |
157 | ||
158 | if (ppgtt->pt_dma_addr) { | |
159 | for (i = 0; i < ppgtt->num_pd_entries; i++) | |
160 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], | |
161 | 4096, PCI_DMA_BIDIRECTIONAL); | |
162 | } | |
163 | ||
164 | kfree(ppgtt->pt_dma_addr); | |
165 | for (i = 0; i < ppgtt->num_pd_entries; i++) | |
166 | __free_page(ppgtt->pt_pages[i]); | |
167 | kfree(ppgtt->pt_pages); | |
168 | kfree(ppgtt); | |
169 | } | |
170 | ||
93dfb40c CW |
171 | /* XXX kill agp_type! */ |
172 | static unsigned int cache_level_to_agp_type(struct drm_device *dev, | |
173 | enum i915_cache_level cache_level) | |
174 | { | |
175 | switch (cache_level) { | |
176 | case I915_CACHE_LLC_MLC: | |
177 | if (INTEL_INFO(dev)->gen >= 6) | |
178 | return AGP_USER_CACHED_MEMORY_LLC_MLC; | |
179 | /* Older chipsets do not have this extra level of CPU | |
180 | * cacheing, so fallthrough and request the PTE simply | |
181 | * as cached. | |
182 | */ | |
183 | case I915_CACHE_LLC: | |
184 | return AGP_USER_CACHED_MEMORY; | |
185 | default: | |
186 | case I915_CACHE_NONE: | |
187 | return AGP_USER_MEMORY; | |
188 | } | |
189 | } | |
190 | ||
5c042287 BW |
191 | static bool do_idling(struct drm_i915_private *dev_priv) |
192 | { | |
193 | bool ret = dev_priv->mm.interruptible; | |
194 | ||
195 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { | |
196 | dev_priv->mm.interruptible = false; | |
8436473a | 197 | if (i915_gpu_idle(dev_priv->dev, false)) { |
5c042287 BW |
198 | DRM_ERROR("Couldn't idle GPU\n"); |
199 | /* Wait a bit, in hopes it avoids the hang */ | |
200 | udelay(10); | |
201 | } | |
202 | } | |
203 | ||
204 | return ret; | |
205 | } | |
206 | ||
207 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) | |
208 | { | |
209 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) | |
210 | dev_priv->mm.interruptible = interruptible; | |
211 | } | |
212 | ||
76aaf220 DV |
213 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
214 | { | |
215 | struct drm_i915_private *dev_priv = dev->dev_private; | |
05394f39 | 216 | struct drm_i915_gem_object *obj; |
76aaf220 | 217 | |
bee4a186 CW |
218 | /* First fill our portion of the GTT with scratch pages */ |
219 | intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, | |
220 | (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); | |
221 | ||
05394f39 | 222 | list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { |
a8e93126 | 223 | i915_gem_clflush_object(obj); |
d5bd1449 | 224 | i915_gem_gtt_rebind_object(obj, obj->cache_level); |
76aaf220 DV |
225 | } |
226 | ||
76aaf220 DV |
227 | intel_gtt_chipset_flush(); |
228 | } | |
7c2e6fdf | 229 | |
05394f39 | 230 | int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj) |
7c2e6fdf | 231 | { |
05394f39 | 232 | struct drm_device *dev = obj->base.dev; |
185cbcb3 | 233 | struct drm_i915_private *dev_priv = dev->dev_private; |
93dfb40c | 234 | unsigned int agp_type = cache_level_to_agp_type(dev, obj->cache_level); |
185cbcb3 | 235 | int ret; |
7c2e6fdf | 236 | |
185cbcb3 | 237 | if (dev_priv->mm.gtt->needs_dmar) { |
05394f39 CW |
238 | ret = intel_gtt_map_memory(obj->pages, |
239 | obj->base.size >> PAGE_SHIFT, | |
240 | &obj->sg_list, | |
241 | &obj->num_sg); | |
185cbcb3 DV |
242 | if (ret != 0) |
243 | return ret; | |
244 | ||
05394f39 CW |
245 | intel_gtt_insert_sg_entries(obj->sg_list, |
246 | obj->num_sg, | |
247 | obj->gtt_space->start >> PAGE_SHIFT, | |
93dfb40c | 248 | agp_type); |
185cbcb3 | 249 | } else |
05394f39 CW |
250 | intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, |
251 | obj->base.size >> PAGE_SHIFT, | |
252 | obj->pages, | |
93dfb40c | 253 | agp_type); |
7c2e6fdf | 254 | |
185cbcb3 | 255 | return 0; |
7c2e6fdf DV |
256 | } |
257 | ||
e4ffd173 CW |
258 | void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj, |
259 | enum i915_cache_level cache_level) | |
d5bd1449 CW |
260 | { |
261 | struct drm_device *dev = obj->base.dev; | |
262 | struct drm_i915_private *dev_priv = dev->dev_private; | |
263 | unsigned int agp_type = cache_level_to_agp_type(dev, cache_level); | |
264 | ||
265 | if (dev_priv->mm.gtt->needs_dmar) { | |
266 | BUG_ON(!obj->sg_list); | |
267 | ||
268 | intel_gtt_insert_sg_entries(obj->sg_list, | |
269 | obj->num_sg, | |
270 | obj->gtt_space->start >> PAGE_SHIFT, | |
271 | agp_type); | |
272 | } else | |
273 | intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, | |
274 | obj->base.size >> PAGE_SHIFT, | |
275 | obj->pages, | |
276 | agp_type); | |
277 | } | |
278 | ||
05394f39 | 279 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
7c2e6fdf | 280 | { |
5c042287 BW |
281 | struct drm_device *dev = obj->base.dev; |
282 | struct drm_i915_private *dev_priv = dev->dev_private; | |
283 | bool interruptible; | |
284 | ||
285 | interruptible = do_idling(dev_priv); | |
286 | ||
d9126400 CW |
287 | intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, |
288 | obj->base.size >> PAGE_SHIFT); | |
7c2e6fdf | 289 | |
d9126400 | 290 | if (obj->sg_list) { |
05394f39 CW |
291 | intel_gtt_unmap_memory(obj->sg_list, obj->num_sg); |
292 | obj->sg_list = NULL; | |
185cbcb3 | 293 | } |
5c042287 BW |
294 | |
295 | undo_idling(dev_priv, interruptible); | |
7c2e6fdf | 296 | } |