Commit | Line | Data |
---|---|---|
76aaf220 DV |
1 | /* |
2 | * Copyright © 2010 Daniel Vetter | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include "drmP.h" | |
76aaf220 DV |
26 | #include "i915_drm.h" |
27 | #include "i915_drv.h" | |
28 | #include "i915_trace.h" | |
29 | #include "intel_drv.h" | |
30 | ||
1d2a314c DV |
31 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
32 | static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, | |
33 | unsigned first_entry, | |
34 | unsigned num_entries) | |
35 | { | |
1d2a314c DV |
36 | uint32_t *pt_vaddr; |
37 | uint32_t scratch_pte; | |
7bddb01f DV |
38 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
39 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; | |
40 | unsigned last_pte, i; | |
1d2a314c DV |
41 | |
42 | scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr); | |
43 | scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC; | |
44 | ||
7bddb01f DV |
45 | while (num_entries) { |
46 | last_pte = first_pte + num_entries; | |
47 | if (last_pte > I915_PPGTT_PT_ENTRIES) | |
48 | last_pte = I915_PPGTT_PT_ENTRIES; | |
49 | ||
50 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); | |
1d2a314c | 51 | |
7bddb01f DV |
52 | for (i = first_pte; i < last_pte; i++) |
53 | pt_vaddr[i] = scratch_pte; | |
1d2a314c DV |
54 | |
55 | kunmap_atomic(pt_vaddr); | |
1d2a314c | 56 | |
7bddb01f DV |
57 | num_entries -= last_pte - first_pte; |
58 | first_pte = 0; | |
59 | act_pd++; | |
60 | } | |
1d2a314c DV |
61 | } |
62 | ||
63 | int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) | |
64 | { | |
65 | struct drm_i915_private *dev_priv = dev->dev_private; | |
66 | struct i915_hw_ppgtt *ppgtt; | |
1d2a314c | 67 | unsigned first_pd_entry_in_global_pt; |
1d2a314c DV |
68 | int i; |
69 | int ret = -ENOMEM; | |
70 | ||
71 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 | |
72 | * entries. For aliasing ppgtt support we just steal them at the end for | |
73 | * now. */ | |
9a0f938b | 74 | first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES; |
1d2a314c DV |
75 | |
76 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); | |
77 | if (!ppgtt) | |
78 | return ret; | |
79 | ||
80 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; | |
81 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, | |
82 | GFP_KERNEL); | |
83 | if (!ppgtt->pt_pages) | |
84 | goto err_ppgtt; | |
85 | ||
86 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
87 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); | |
88 | if (!ppgtt->pt_pages[i]) | |
89 | goto err_pt_alloc; | |
90 | } | |
91 | ||
92 | if (dev_priv->mm.gtt->needs_dmar) { | |
93 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) | |
94 | *ppgtt->num_pd_entries, | |
95 | GFP_KERNEL); | |
96 | if (!ppgtt->pt_dma_addr) | |
97 | goto err_pt_alloc; | |
1d2a314c | 98 | |
211c568b DV |
99 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
100 | dma_addr_t pt_addr; | |
101 | ||
1d2a314c DV |
102 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], |
103 | 0, 4096, | |
104 | PCI_DMA_BIDIRECTIONAL); | |
105 | ||
106 | if (pci_dma_mapping_error(dev->pdev, | |
107 | pt_addr)) { | |
108 | ret = -EIO; | |
109 | goto err_pd_pin; | |
110 | ||
111 | } | |
112 | ppgtt->pt_dma_addr[i] = pt_addr; | |
211c568b | 113 | } |
1d2a314c | 114 | } |
1d2a314c DV |
115 | |
116 | ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; | |
117 | ||
118 | i915_ppgtt_clear_range(ppgtt, 0, | |
119 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); | |
120 | ||
121 | ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t); | |
122 | ||
123 | dev_priv->mm.aliasing_ppgtt = ppgtt; | |
124 | ||
125 | return 0; | |
126 | ||
127 | err_pd_pin: | |
128 | if (ppgtt->pt_dma_addr) { | |
129 | for (i--; i >= 0; i--) | |
130 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], | |
131 | 4096, PCI_DMA_BIDIRECTIONAL); | |
132 | } | |
133 | err_pt_alloc: | |
134 | kfree(ppgtt->pt_dma_addr); | |
135 | for (i = 0; i < ppgtt->num_pd_entries; i++) { | |
136 | if (ppgtt->pt_pages[i]) | |
137 | __free_page(ppgtt->pt_pages[i]); | |
138 | } | |
139 | kfree(ppgtt->pt_pages); | |
140 | err_ppgtt: | |
141 | kfree(ppgtt); | |
142 | ||
143 | return ret; | |
144 | } | |
145 | ||
146 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) | |
147 | { | |
148 | struct drm_i915_private *dev_priv = dev->dev_private; | |
149 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; | |
150 | int i; | |
151 | ||
152 | if (!ppgtt) | |
153 | return; | |
154 | ||
155 | if (ppgtt->pt_dma_addr) { | |
156 | for (i = 0; i < ppgtt->num_pd_entries; i++) | |
157 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], | |
158 | 4096, PCI_DMA_BIDIRECTIONAL); | |
159 | } | |
160 | ||
161 | kfree(ppgtt->pt_dma_addr); | |
162 | for (i = 0; i < ppgtt->num_pd_entries; i++) | |
163 | __free_page(ppgtt->pt_pages[i]); | |
164 | kfree(ppgtt->pt_pages); | |
165 | kfree(ppgtt); | |
166 | } | |
167 | ||
7bddb01f DV |
168 | static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt, |
169 | struct scatterlist *sg_list, | |
170 | unsigned sg_len, | |
171 | unsigned first_entry, | |
172 | uint32_t pte_flags) | |
173 | { | |
174 | uint32_t *pt_vaddr, pte; | |
175 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; | |
176 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; | |
177 | unsigned i, j, m, segment_len; | |
178 | dma_addr_t page_addr; | |
179 | struct scatterlist *sg; | |
180 | ||
181 | /* init sg walking */ | |
182 | sg = sg_list; | |
183 | i = 0; | |
184 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; | |
185 | m = 0; | |
186 | ||
187 | while (i < sg_len) { | |
188 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); | |
189 | ||
190 | for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { | |
191 | page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); | |
192 | pte = GEN6_PTE_ADDR_ENCODE(page_addr); | |
193 | pt_vaddr[j] = pte | pte_flags; | |
194 | ||
195 | /* grab the next page */ | |
196 | m++; | |
197 | if (m == segment_len) { | |
198 | sg = sg_next(sg); | |
199 | i++; | |
200 | if (i == sg_len) | |
201 | break; | |
202 | ||
203 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; | |
204 | m = 0; | |
205 | } | |
206 | } | |
207 | ||
208 | kunmap_atomic(pt_vaddr); | |
209 | ||
210 | first_pte = 0; | |
211 | act_pd++; | |
212 | } | |
213 | } | |
214 | ||
215 | static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, | |
216 | unsigned first_entry, unsigned num_entries, | |
217 | struct page **pages, uint32_t pte_flags) | |
218 | { | |
219 | uint32_t *pt_vaddr, pte; | |
220 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; | |
221 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; | |
222 | unsigned last_pte, i; | |
223 | dma_addr_t page_addr; | |
224 | ||
225 | while (num_entries) { | |
226 | last_pte = first_pte + num_entries; | |
227 | last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES); | |
228 | ||
229 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); | |
230 | ||
231 | for (i = first_pte; i < last_pte; i++) { | |
232 | page_addr = page_to_phys(*pages); | |
233 | pte = GEN6_PTE_ADDR_ENCODE(page_addr); | |
234 | pt_vaddr[i] = pte | pte_flags; | |
235 | ||
236 | pages++; | |
237 | } | |
238 | ||
239 | kunmap_atomic(pt_vaddr); | |
240 | ||
241 | num_entries -= last_pte - first_pte; | |
242 | first_pte = 0; | |
243 | act_pd++; | |
244 | } | |
245 | } | |
246 | ||
247 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, | |
248 | struct drm_i915_gem_object *obj, | |
249 | enum i915_cache_level cache_level) | |
250 | { | |
251 | struct drm_device *dev = obj->base.dev; | |
252 | struct drm_i915_private *dev_priv = dev->dev_private; | |
253 | uint32_t pte_flags = GEN6_PTE_VALID; | |
254 | ||
255 | switch (cache_level) { | |
256 | case I915_CACHE_LLC_MLC: | |
257 | pte_flags |= GEN6_PTE_CACHE_LLC_MLC; | |
258 | break; | |
259 | case I915_CACHE_LLC: | |
260 | pte_flags |= GEN6_PTE_CACHE_LLC; | |
261 | break; | |
262 | case I915_CACHE_NONE: | |
a843af18 DV |
263 | if (IS_HASWELL(dev)) |
264 | pte_flags |= HSW_PTE_UNCACHED; | |
265 | else | |
266 | pte_flags |= GEN6_PTE_UNCACHED; | |
7bddb01f DV |
267 | break; |
268 | default: | |
269 | BUG(); | |
270 | } | |
271 | ||
1286ff73 DV |
272 | if (obj->sg_table) { |
273 | i915_ppgtt_insert_sg_entries(ppgtt, | |
274 | obj->sg_table->sgl, | |
275 | obj->sg_table->nents, | |
276 | obj->gtt_space->start >> PAGE_SHIFT, | |
277 | pte_flags); | |
278 | } else if (dev_priv->mm.gtt->needs_dmar) { | |
7bddb01f DV |
279 | BUG_ON(!obj->sg_list); |
280 | ||
281 | i915_ppgtt_insert_sg_entries(ppgtt, | |
282 | obj->sg_list, | |
283 | obj->num_sg, | |
284 | obj->gtt_space->start >> PAGE_SHIFT, | |
285 | pte_flags); | |
286 | } else | |
287 | i915_ppgtt_insert_pages(ppgtt, | |
288 | obj->gtt_space->start >> PAGE_SHIFT, | |
289 | obj->base.size >> PAGE_SHIFT, | |
290 | obj->pages, | |
291 | pte_flags); | |
292 | } | |
293 | ||
294 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, | |
295 | struct drm_i915_gem_object *obj) | |
296 | { | |
297 | i915_ppgtt_clear_range(ppgtt, | |
298 | obj->gtt_space->start >> PAGE_SHIFT, | |
299 | obj->base.size >> PAGE_SHIFT); | |
300 | } | |
301 | ||
93dfb40c CW |
302 | /* XXX kill agp_type! */ |
303 | static unsigned int cache_level_to_agp_type(struct drm_device *dev, | |
304 | enum i915_cache_level cache_level) | |
305 | { | |
306 | switch (cache_level) { | |
307 | case I915_CACHE_LLC_MLC: | |
308 | if (INTEL_INFO(dev)->gen >= 6) | |
309 | return AGP_USER_CACHED_MEMORY_LLC_MLC; | |
310 | /* Older chipsets do not have this extra level of CPU | |
311 | * cacheing, so fallthrough and request the PTE simply | |
312 | * as cached. | |
313 | */ | |
314 | case I915_CACHE_LLC: | |
315 | return AGP_USER_CACHED_MEMORY; | |
316 | default: | |
317 | case I915_CACHE_NONE: | |
318 | return AGP_USER_MEMORY; | |
319 | } | |
320 | } | |
321 | ||
5c042287 BW |
322 | static bool do_idling(struct drm_i915_private *dev_priv) |
323 | { | |
324 | bool ret = dev_priv->mm.interruptible; | |
325 | ||
326 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { | |
327 | dev_priv->mm.interruptible = false; | |
b2da9fe5 | 328 | if (i915_gpu_idle(dev_priv->dev)) { |
5c042287 BW |
329 | DRM_ERROR("Couldn't idle GPU\n"); |
330 | /* Wait a bit, in hopes it avoids the hang */ | |
331 | udelay(10); | |
332 | } | |
333 | } | |
334 | ||
335 | return ret; | |
336 | } | |
337 | ||
338 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) | |
339 | { | |
340 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) | |
341 | dev_priv->mm.interruptible = interruptible; | |
342 | } | |
343 | ||
76aaf220 DV |
344 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
345 | { | |
346 | struct drm_i915_private *dev_priv = dev->dev_private; | |
05394f39 | 347 | struct drm_i915_gem_object *obj; |
76aaf220 | 348 | |
bee4a186 CW |
349 | /* First fill our portion of the GTT with scratch pages */ |
350 | intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, | |
351 | (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); | |
352 | ||
05394f39 | 353 | list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { |
a8e93126 | 354 | i915_gem_clflush_object(obj); |
74163907 | 355 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
76aaf220 DV |
356 | } |
357 | ||
76aaf220 DV |
358 | intel_gtt_chipset_flush(); |
359 | } | |
7c2e6fdf | 360 | |
74163907 | 361 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
7c2e6fdf | 362 | { |
05394f39 | 363 | struct drm_device *dev = obj->base.dev; |
185cbcb3 | 364 | struct drm_i915_private *dev_priv = dev->dev_private; |
7c2e6fdf | 365 | |
f00f9791 DA |
366 | /* don't map imported dma buf objects */ |
367 | if (dev_priv->mm.gtt->needs_dmar && !obj->sg_table) | |
74163907 DV |
368 | return intel_gtt_map_memory(obj->pages, |
369 | obj->base.size >> PAGE_SHIFT, | |
370 | &obj->sg_list, | |
371 | &obj->num_sg); | |
372 | else | |
373 | return 0; | |
7c2e6fdf DV |
374 | } |
375 | ||
74163907 DV |
376 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
377 | enum i915_cache_level cache_level) | |
d5bd1449 CW |
378 | { |
379 | struct drm_device *dev = obj->base.dev; | |
380 | struct drm_i915_private *dev_priv = dev->dev_private; | |
381 | unsigned int agp_type = cache_level_to_agp_type(dev, cache_level); | |
382 | ||
1286ff73 DV |
383 | if (obj->sg_table) { |
384 | intel_gtt_insert_sg_entries(obj->sg_table->sgl, | |
385 | obj->sg_table->nents, | |
386 | obj->gtt_space->start >> PAGE_SHIFT, | |
387 | agp_type); | |
388 | } else if (dev_priv->mm.gtt->needs_dmar) { | |
d5bd1449 CW |
389 | BUG_ON(!obj->sg_list); |
390 | ||
391 | intel_gtt_insert_sg_entries(obj->sg_list, | |
392 | obj->num_sg, | |
393 | obj->gtt_space->start >> PAGE_SHIFT, | |
394 | agp_type); | |
395 | } else | |
396 | intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, | |
397 | obj->base.size >> PAGE_SHIFT, | |
398 | obj->pages, | |
399 | agp_type); | |
74898d7e DV |
400 | |
401 | obj->has_global_gtt_mapping = 1; | |
d5bd1449 CW |
402 | } |
403 | ||
05394f39 | 404 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
74163907 DV |
405 | { |
406 | intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, | |
407 | obj->base.size >> PAGE_SHIFT); | |
74898d7e DV |
408 | |
409 | obj->has_global_gtt_mapping = 0; | |
74163907 DV |
410 | } |
411 | ||
412 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) | |
7c2e6fdf | 413 | { |
5c042287 BW |
414 | struct drm_device *dev = obj->base.dev; |
415 | struct drm_i915_private *dev_priv = dev->dev_private; | |
416 | bool interruptible; | |
417 | ||
418 | interruptible = do_idling(dev_priv); | |
419 | ||
d9126400 | 420 | if (obj->sg_list) { |
05394f39 CW |
421 | intel_gtt_unmap_memory(obj->sg_list, obj->num_sg); |
422 | obj->sg_list = NULL; | |
185cbcb3 | 423 | } |
5c042287 BW |
424 | |
425 | undo_idling(dev_priv, interruptible); | |
7c2e6fdf | 426 | } |
644ec02b DV |
427 | |
428 | void i915_gem_init_global_gtt(struct drm_device *dev, | |
429 | unsigned long start, | |
430 | unsigned long mappable_end, | |
431 | unsigned long end) | |
432 | { | |
433 | drm_i915_private_t *dev_priv = dev->dev_private; | |
434 | ||
d1dd20a9 DV |
435 | /* Substract the guard page ... */ |
436 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); | |
644ec02b DV |
437 | |
438 | dev_priv->mm.gtt_start = start; | |
439 | dev_priv->mm.gtt_mappable_end = mappable_end; | |
440 | dev_priv->mm.gtt_end = end; | |
441 | dev_priv->mm.gtt_total = end - start; | |
442 | dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; | |
443 | ||
d1dd20a9 | 444 | /* ... but ensure that we clear the entire range. */ |
644ec02b DV |
445 | intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); |
446 | } |