projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Propagate error from flushing the ring
[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
673a394b 37
88241785 38static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
05394f39
CW		 39static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
88241785
CW		 41static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
42 bool write);
43static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
44 uint64_t offset,
45 uint64_t size);
05394f39 46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
88241785
CW		 47static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
48 unsigned alignment,
49 bool map_and_fenceable);
d9e86c0e
CW		 50static void i915_gem_clear_fence_reg(struct drm_device *dev,
51 struct drm_i915_fence_reg *reg);
05394f39
CW		 52static int i915_gem_phys_pwrite(struct drm_device *dev,
53 struct drm_i915_gem_object *obj,
71acb5eb 54 struct drm_i915_gem_pwrite *args,
05394f39
CW		 55 struct drm_file *file);
56static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
673a394b 57
17250b71
CW		 58static int i915_gem_inactive_shrink(struct shrinker *shrinker,
59 int nr_to_scan,
60 gfp_t gfp_mask);
61
31169714 62
73aa808f
CW		 63/* some bookkeeping */
64static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
65 size_t size)
66{
67 dev_priv->mm.object_count++;
68 dev_priv->mm.object_memory += size;
69}
70
71static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
72 size_t size)
73{
74 dev_priv->mm.object_count--;
75 dev_priv->mm.object_memory -= size;
76}
77
30dbf0c0
CW		 78int
79i915_gem_check_is_wedged(struct drm_device *dev)
80{
81 struct drm_i915_private *dev_priv = dev->dev_private;
82 struct completion *x = &dev_priv->error_completion;
83 unsigned long flags;
84 int ret;
85
86 if (!atomic_read(&dev_priv->mm.wedged))
87 return 0;
88
89 ret = wait_for_completion_interruptible(x);
90 if (ret)
91 return ret;
92
93 /* Success, we reset the GPU! */
94 if (!atomic_read(&dev_priv->mm.wedged))
95 return 0;
96
97 /* GPU is hung, bump the completion count to account for
98 * the token we just consumed so that we never hit zero and
99 * end up waiting upon a subsequent completion event that
100 * will never happen.
101 */
102 spin_lock_irqsave(&x->wait.lock, flags);
103 x->done++;
104 spin_unlock_irqrestore(&x->wait.lock, flags);
105 return -EIO;
106}
107
54cf91dc 108int i915_mutex_lock_interruptible(struct drm_device *dev)
76c1dec1
CW		 109{
110 struct drm_i915_private *dev_priv = dev->dev_private;
111 int ret;
112
113 ret = i915_gem_check_is_wedged(dev);
114 if (ret)
115 return ret;
116
117 ret = mutex_lock_interruptible(&dev->struct_mutex);
118 if (ret)
119 return ret;
120
121 if (atomic_read(&dev_priv->mm.wedged)) {
122 mutex_unlock(&dev->struct_mutex);
123 return -EAGAIN;
124 }
125
23bc5982 126 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 127 return 0;
128}
30dbf0c0 129
7d1c4804 130static inline bool
05394f39 131i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
7d1c4804 132{
05394f39 133 return obj->gtt_space && !obj->active && obj->pin_count == 0;
7d1c4804
CW		 134}
135
2021746e
CW		 136void i915_gem_do_init(struct drm_device *dev,
137 unsigned long start,
138 unsigned long mappable_end,
139 unsigned long end)
673a394b
EA		 140{
141 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 142
79e53945
JB		 143 drm_mm_init(&dev_priv->mm.gtt_space, start,
144 end - start);
673a394b 145
73aa808f 146 dev_priv->mm.gtt_total = end - start;
fb7d516a 147 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
53984635 148 dev_priv->mm.gtt_mappable_end = mappable_end;
79e53945 149}
673a394b 150
79e53945
JB		 151int
152i915_gem_init_ioctl(struct drm_device *dev, void *data,
05394f39 153 struct drm_file *file)
79e53945
JB		 154{
155 struct drm_i915_gem_init *args = data;
2021746e
CW		 156
157 if (args->gtt_start >= args->gtt_end ||
158 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
159 return -EINVAL;
79e53945
JB		 160
161 mutex_lock(&dev->struct_mutex);
2021746e 162 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
673a394b
EA		 163 mutex_unlock(&dev->struct_mutex);
164
2021746e 165 return 0;
673a394b
EA		 166}
167
5a125c3c
EA		 168int
169i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
05394f39 170 struct drm_file *file)
5a125c3c 171{
73aa808f 172 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 173 struct drm_i915_gem_get_aperture *args = data;
6299f992
CW		 174 struct drm_i915_gem_object *obj;
175 size_t pinned;
5a125c3c
EA		 176
177 if (!(dev->driver->driver_features & DRIVER_GEM))
178 return -ENODEV;
179
6299f992 180 pinned = 0;
73aa808f 181 mutex_lock(&dev->struct_mutex);
6299f992
CW		 182 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
183 pinned += obj->gtt_space->size;
73aa808f 184 mutex_unlock(&dev->struct_mutex);
5a125c3c 185
6299f992
CW		 186 args->aper_size = dev_priv->mm.gtt_total;
187 args->aper_available_size = args->aper_size -pinned;
188
5a125c3c
EA		 189 return 0;
190}
191
673a394b
EA		 192/**
193 * Creates a new mm object and returns a handle to it.
194 */
195int
196i915_gem_create_ioctl(struct drm_device *dev, void *data,
05394f39 197 struct drm_file *file)
673a394b
EA		 198{
199 struct drm_i915_gem_create *args = data;
05394f39 200 struct drm_i915_gem_object *obj;
a1a2d1d3
PP		 201 int ret;
202 u32 handle;
673a394b
EA		 203
204 args->size = roundup(args->size, PAGE_SIZE);
205
206 /* Allocate the new object */
ac52bc56 207 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 208 if (obj == NULL)
209 return -ENOMEM;
210
05394f39 211 ret = drm_gem_handle_create(file, &obj->base, &handle);
1dfd9754 212 if (ret) {
05394f39
CW		 213 drm_gem_object_release(&obj->base);
214 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
202f2fef 215 kfree(obj);
673a394b 216 return ret;
1dfd9754 217 }
673a394b 218
202f2fef 219 /* drop reference from allocate - handle holds it now */
05394f39 220 drm_gem_object_unreference(&obj->base);
202f2fef
CW		 221 trace_i915_gem_object_create(obj);
222
1dfd9754 223 args->handle = handle;
673a394b
EA		 224 return 0;
225}
226
05394f39 227static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
280b713b 228{
05394f39 229 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280b713b
EA		 230
231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
05394f39 232 obj->tiling_mode != I915_TILING_NONE;
280b713b
EA		 233}
234
99a03df5 235static inline void
40123c1f
EA		 236slow_shmem_copy(struct page *dst_page,
237 int dst_offset,
238 struct page *src_page,
239 int src_offset,
240 int length)
241{
242 char *dst_vaddr, *src_vaddr;
243
99a03df5
CW		 244 dst_vaddr = kmap(dst_page);
245 src_vaddr = kmap(src_page);
40123c1f
EA		 246
247 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
248
99a03df5
CW		 249 kunmap(src_page);
250 kunmap(dst_page);
40123c1f
EA		 251}
252
99a03df5 253static inline void
280b713b
EA		 254slow_shmem_bit17_copy(struct page *gpu_page,
255 int gpu_offset,
256 struct page *cpu_page,
257 int cpu_offset,
258 int length,
259 int is_read)
260{
261 char *gpu_vaddr, *cpu_vaddr;
262
263 /* Use the unswizzled path if this page isn't affected. */
264 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
265 if (is_read)
266 return slow_shmem_copy(cpu_page, cpu_offset,
267 gpu_page, gpu_offset, length);
268 else
269 return slow_shmem_copy(gpu_page, gpu_offset,
270 cpu_page, cpu_offset, length);
271 }
272
99a03df5
CW		 273 gpu_vaddr = kmap(gpu_page);
274 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 275
276 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
277 * XORing with the other bits (A9 for Y, A9 and A10 for X)
278 */
279 while (length > 0) {
280 int cacheline_end = ALIGN(gpu_offset + 1, 64);
281 int this_length = min(cacheline_end - gpu_offset, length);
282 int swizzled_gpu_offset = gpu_offset ^ 64;
283
284 if (is_read) {
285 memcpy(cpu_vaddr + cpu_offset,
286 gpu_vaddr + swizzled_gpu_offset,
287 this_length);
288 } else {
289 memcpy(gpu_vaddr + swizzled_gpu_offset,
290 cpu_vaddr + cpu_offset,
291 this_length);
292 }
293 cpu_offset += this_length;
294 gpu_offset += this_length;
295 length -= this_length;
296 }
297
99a03df5
CW		 298 kunmap(cpu_page);
299 kunmap(gpu_page);
280b713b
EA		 300}
301
eb01459f
EA		 302/**
303 * This is the fast shmem pread path, which attempts to copy_from_user directly
304 * from the backing pages of the object to the user's address space. On a
305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
306 */
307static int
05394f39
CW		 308i915_gem_shmem_pread_fast(struct drm_device *dev,
309 struct drm_i915_gem_object *obj,
eb01459f 310 struct drm_i915_gem_pread *args,
05394f39 311 struct drm_file *file)
eb01459f 312{
05394f39 313 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f 314 ssize_t remain;
e5281ccd 315 loff_t offset;
eb01459f
EA		 316 char __user *user_data;
317 int page_offset, page_length;
eb01459f
EA		 318
319 user_data = (char __user *) (uintptr_t) args->data_ptr;
320 remain = args->size;
321
eb01459f
EA		 322 offset = args->offset;
323
324 while (remain > 0) {
e5281ccd
CW		 325 struct page *page;
326 char *vaddr;
327 int ret;
328
eb01459f
EA		 329 /* Operation in this page
330 *
eb01459f
EA		 331 * page_offset = offset within page
332 * page_length = bytes to copy for this page
333 */
eb01459f
EA		 334 page_offset = offset & (PAGE_SIZE-1);
335 page_length = remain;
336 if ((page_offset + remain) > PAGE_SIZE)
337 page_length = PAGE_SIZE - page_offset;
338
e5281ccd
CW		 339 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
340 GFP_HIGHUSER | __GFP_RECLAIMABLE);
341 if (IS_ERR(page))
342 return PTR_ERR(page);
343
344 vaddr = kmap_atomic(page);
345 ret = __copy_to_user_inatomic(user_data,
346 vaddr + page_offset,
347 page_length);
348 kunmap_atomic(vaddr);
349
350 mark_page_accessed(page);
351 page_cache_release(page);
352 if (ret)
4f27b75d 353 return -EFAULT;
eb01459f
EA		 354
355 remain -= page_length;
356 user_data += page_length;
357 offset += page_length;
358 }
359
4f27b75d 360 return 0;
eb01459f
EA		 361}
362
363/**
364 * This is the fallback shmem pread path, which allocates temporary storage
365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
366 * can copy out of the object's backing pages while holding the struct mutex
367 * and not take page faults.
368 */
369static int
05394f39
CW		 370i915_gem_shmem_pread_slow(struct drm_device *dev,
371 struct drm_i915_gem_object *obj,
eb01459f 372 struct drm_i915_gem_pread *args,
05394f39 373 struct drm_file *file)
eb01459f 374{
05394f39 375 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
eb01459f
EA		 376 struct mm_struct *mm = current->mm;
377 struct page **user_pages;
378 ssize_t remain;
379 loff_t offset, pinned_pages, i;
380 loff_t first_data_page, last_data_page, num_pages;
e5281ccd
CW		 381 int shmem_page_offset;
382 int data_page_index, data_page_offset;
eb01459f
EA		 383 int page_length;
384 int ret;
385 uint64_t data_ptr = args->data_ptr;
280b713b 386 int do_bit17_swizzling;
eb01459f
EA		 387
388 remain = args->size;
389
390 /* Pin the user pages containing the data. We can't fault while
391 * holding the struct mutex, yet we want to hold it while
392 * dereferencing the user data.
393 */
394 first_data_page = data_ptr / PAGE_SIZE;
395 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
396 num_pages = last_data_page - first_data_page + 1;
397
4f27b75d 398 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
eb01459f
EA		 399 if (user_pages == NULL)
400 return -ENOMEM;
401
4f27b75d 402 mutex_unlock(&dev->struct_mutex);
eb01459f
EA		 403 down_read(&mm->mmap_sem);
404 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 405 num_pages, 1, 0, user_pages, NULL);
eb01459f 406 up_read(&mm->mmap_sem);
4f27b75d 407 mutex_lock(&dev->struct_mutex);
eb01459f
EA		 408 if (pinned_pages < num_pages) {
409 ret = -EFAULT;
4f27b75d 410 goto out;
eb01459f
EA		 411 }
412
4f27b75d
CW		 413 ret = i915_gem_object_set_cpu_read_domain_range(obj,
414 args->offset,
415 args->size);
07f73f69 416 if (ret)
4f27b75d 417 goto out;
eb01459f 418
4f27b75d 419 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
eb01459f 420
eb01459f
EA		 421 offset = args->offset;
422
423 while (remain > 0) {
e5281ccd
CW		 424 struct page *page;
425
eb01459f
EA		 426 /* Operation in this page
427 *
eb01459f
EA		 428 * shmem_page_offset = offset within page in shmem file
429 * data_page_index = page number in get_user_pages return
430 * data_page_offset = offset with data_page_index page.
431 * page_length = bytes to copy for this page
432 */
eb01459f
EA		 433 shmem_page_offset = offset & ~PAGE_MASK;
434 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
435 data_page_offset = data_ptr & ~PAGE_MASK;
436
437 page_length = remain;
438 if ((shmem_page_offset + page_length) > PAGE_SIZE)
439 page_length = PAGE_SIZE - shmem_page_offset;
440 if ((data_page_offset + page_length) > PAGE_SIZE)
441 page_length = PAGE_SIZE - data_page_offset;
442
e5281ccd
CW		 443 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
444 GFP_HIGHUSER | __GFP_RECLAIMABLE);
445 if (IS_ERR(page))
446 return PTR_ERR(page);
447
280b713b 448 if (do_bit17_swizzling) {
e5281ccd 449 slow_shmem_bit17_copy(page,
280b713b 450 shmem_page_offset,
99a03df5
CW		 451 user_pages[data_page_index],
452 data_page_offset,
453 page_length,
454 1);
455 } else {
456 slow_shmem_copy(user_pages[data_page_index],
457 data_page_offset,
e5281ccd 458 page,
99a03df5
CW		 459 shmem_page_offset,
460 page_length);
280b713b 461 }
eb01459f 462
e5281ccd
CW		 463 mark_page_accessed(page);
464 page_cache_release(page);
465
eb01459f
EA		 466 remain -= page_length;
467 data_ptr += page_length;
468 offset += page_length;
469 }
470
4f27b75d 471out:
eb01459f
EA		 472 for (i = 0; i < pinned_pages; i++) {
473 SetPageDirty(user_pages[i]);
e5281ccd 474 mark_page_accessed(user_pages[i]);
eb01459f
EA		 475 page_cache_release(user_pages[i]);
476 }
8e7d2b2c 477 drm_free_large(user_pages);
eb01459f
EA		 478
479 return ret;
480}
481
673a394b
EA		 482/**
483 * Reads data from the object referenced by handle.
484 *
485 * On error, the contents of *data are undefined.
486 */
487int
488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
05394f39 489 struct drm_file *file)
673a394b
EA		 490{
491 struct drm_i915_gem_pread *args = data;
05394f39 492 struct drm_i915_gem_object *obj;
35b62a89 493 int ret = 0;
673a394b 494
51311d0a
CW		 495 if (args->size == 0)
496 return 0;
497
498 if (!access_ok(VERIFY_WRITE,
499 (char __user *)(uintptr_t)args->data_ptr,
500 args->size))
501 return -EFAULT;
502
503 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
504 args->size);
505 if (ret)
506 return -EFAULT;
507
4f27b75d 508 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 509 if (ret)
4f27b75d 510 return ret;
673a394b 511
05394f39 512 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 513 if (obj == NULL) {
514 ret = -ENOENT;
515 goto unlock;
4f27b75d 516 }
673a394b 517
7dcd2499 518 /* Bounds check source. */
05394f39
CW		 519 if (args->offset > obj->base.size ||
520 args->size > obj->base.size - args->offset) {
ce9d419d 521 ret = -EINVAL;
35b62a89 522 goto out;
ce9d419d
CW		 523 }
524
4f27b75d
CW		 525 ret = i915_gem_object_set_cpu_read_domain_range(obj,
526 args->offset,
527 args->size);
528 if (ret)
e5281ccd 529 goto out;
4f27b75d
CW		 530
531 ret = -EFAULT;
532 if (!i915_gem_object_needs_bit17_swizzle(obj))
05394f39 533 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
4f27b75d 534 if (ret == -EFAULT)
05394f39 535 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
673a394b 536
35b62a89 537out:
05394f39 538 drm_gem_object_unreference(&obj->base);
1d7cfea1 539unlock:
4f27b75d 540 mutex_unlock(&dev->struct_mutex);
eb01459f 541 return ret;
673a394b
EA		 542}
543
0839ccb8
KP		 544/* This is the fast write path which cannot handle
545 * page faults in the source data
9b7530cc 546 */
0839ccb8
KP		 547
548static inline int
549fast_user_write(struct io_mapping *mapping,
550 loff_t page_base, int page_offset,
551 char __user *user_data,
552 int length)
9b7530cc 553{
9b7530cc 554 char *vaddr_atomic;
0839ccb8 555 unsigned long unwritten;
9b7530cc 556
3e4d3af5 557 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
0839ccb8
KP		 558 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
559 user_data, length);
3e4d3af5 560 io_mapping_unmap_atomic(vaddr_atomic);
fbd5a26d 561 return unwritten;
0839ccb8
KP		 562}
563
564/* Here's the write path which can sleep for
565 * page faults
566 */
567
ab34c226 568static inline void
3de09aa3
EA		 569slow_kernel_write(struct io_mapping *mapping,
570 loff_t gtt_base, int gtt_offset,
571 struct page *user_page, int user_offset,
572 int length)
0839ccb8 573{
ab34c226
CW		 574 char __iomem *dst_vaddr;
575 char *src_vaddr;
0839ccb8 576
ab34c226
CW		 577 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
578 src_vaddr = kmap(user_page);
579
580 memcpy_toio(dst_vaddr + gtt_offset,
581 src_vaddr + user_offset,
582 length);
583
584 kunmap(user_page);
585 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 586}
587
3de09aa3
EA		 588/**
589 * This is the fast pwrite path, where we copy the data directly from the
590 * user into the GTT, uncached.
591 */
673a394b 592static int
05394f39
CW		 593i915_gem_gtt_pwrite_fast(struct drm_device *dev,
594 struct drm_i915_gem_object *obj,
3de09aa3 595 struct drm_i915_gem_pwrite *args,
05394f39 596 struct drm_file *file)
673a394b 597{
0839ccb8 598 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 599 ssize_t remain;
0839ccb8 600 loff_t offset, page_base;
673a394b 601 char __user *user_data;
0839ccb8 602 int page_offset, page_length;
673a394b
EA		 603
604 user_data = (char __user *) (uintptr_t) args->data_ptr;
605 remain = args->size;
673a394b 606
05394f39 607 offset = obj->gtt_offset + args->offset;
673a394b
EA		 608
609 while (remain > 0) {
610 /* Operation in this page
611 *
0839ccb8
KP		 612 * page_base = page offset within aperture
613 * page_offset = offset within page
614 * page_length = bytes to copy for this page
673a394b 615 */
0839ccb8
KP		 616 page_base = (offset & ~(PAGE_SIZE-1));
617 page_offset = offset & (PAGE_SIZE-1);
618 page_length = remain;
619 if ((page_offset + remain) > PAGE_SIZE)
620 page_length = PAGE_SIZE - page_offset;
621
0839ccb8 622 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 623 * source page isn't available. Return the error and we'll
624 * retry in the slow path.
0839ccb8 625 */
fbd5a26d
CW		 626 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
627 page_offset, user_data, page_length))
628
629 return -EFAULT;
673a394b 630
0839ccb8
KP		 631 remain -= page_length;
632 user_data += page_length;
633 offset += page_length;
673a394b 634 }
673a394b 635
fbd5a26d 636 return 0;
673a394b
EA		 637}
638
3de09aa3
EA		 639/**
640 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
641 * the memory and maps it using kmap_atomic for copying.
642 *
643 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
644 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
645 */
3043c60c 646static int
05394f39
CW		 647i915_gem_gtt_pwrite_slow(struct drm_device *dev,
648 struct drm_i915_gem_object *obj,
3de09aa3 649 struct drm_i915_gem_pwrite *args,
05394f39 650 struct drm_file *file)
673a394b 651{
3de09aa3
EA		 652 drm_i915_private_t *dev_priv = dev->dev_private;
653 ssize_t remain;
654 loff_t gtt_page_base, offset;
655 loff_t first_data_page, last_data_page, num_pages;
656 loff_t pinned_pages, i;
657 struct page **user_pages;
658 struct mm_struct *mm = current->mm;
659 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 660 int ret;
3de09aa3
EA		 661 uint64_t data_ptr = args->data_ptr;
662
663 remain = args->size;
664
665 /* Pin the user pages containing the data. We can't fault while
666 * holding the struct mutex, and all of the pwrite implementations
667 * want to hold it while dereferencing the user data.
668 */
669 first_data_page = data_ptr / PAGE_SIZE;
670 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
671 num_pages = last_data_page - first_data_page + 1;
672
fbd5a26d 673 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
3de09aa3
EA		 674 if (user_pages == NULL)
675 return -ENOMEM;
676
fbd5a26d 677 mutex_unlock(&dev->struct_mutex);
3de09aa3
EA		 678 down_read(&mm->mmap_sem);
679 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
680 num_pages, 0, 0, user_pages, NULL);
681 up_read(&mm->mmap_sem);
fbd5a26d 682 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 683 if (pinned_pages < num_pages) {
684 ret = -EFAULT;
685 goto out_unpin_pages;
686 }
673a394b 687
d9e86c0e
CW		 688 ret = i915_gem_object_set_to_gtt_domain(obj, true);
689 if (ret)
690 goto out_unpin_pages;
691
692 ret = i915_gem_object_put_fence(obj);
3de09aa3 693 if (ret)
fbd5a26d 694 goto out_unpin_pages;
3de09aa3 695
05394f39 696 offset = obj->gtt_offset + args->offset;
3de09aa3
EA		 697
698 while (remain > 0) {
699 /* Operation in this page
700 *
701 * gtt_page_base = page offset within aperture
702 * gtt_page_offset = offset within page in aperture
703 * data_page_index = page number in get_user_pages return
704 * data_page_offset = offset with data_page_index page.
705 * page_length = bytes to copy for this page
706 */
707 gtt_page_base = offset & PAGE_MASK;
708 gtt_page_offset = offset & ~PAGE_MASK;
709 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
710 data_page_offset = data_ptr & ~PAGE_MASK;
711
712 page_length = remain;
713 if ((gtt_page_offset + page_length) > PAGE_SIZE)
714 page_length = PAGE_SIZE - gtt_page_offset;
715 if ((data_page_offset + page_length) > PAGE_SIZE)
716 page_length = PAGE_SIZE - data_page_offset;
717
ab34c226
CW		 718 slow_kernel_write(dev_priv->mm.gtt_mapping,
719 gtt_page_base, gtt_page_offset,
720 user_pages[data_page_index],
721 data_page_offset,
722 page_length);
3de09aa3
EA		 723
724 remain -= page_length;
725 offset += page_length;
726 data_ptr += page_length;
727 }
728
3de09aa3
EA		 729out_unpin_pages:
730 for (i = 0; i < pinned_pages; i++)
731 page_cache_release(user_pages[i]);
8e7d2b2c 732 drm_free_large(user_pages);
3de09aa3
EA		 733
734 return ret;
735}
736
40123c1f
EA		 737/**
738 * This is the fast shmem pwrite path, which attempts to directly
739 * copy_from_user into the kmapped pages backing the object.
740 */
3043c60c 741static int
05394f39
CW		 742i915_gem_shmem_pwrite_fast(struct drm_device *dev,
743 struct drm_i915_gem_object *obj,
40123c1f 744 struct drm_i915_gem_pwrite *args,
05394f39 745 struct drm_file *file)
673a394b 746{
05394f39 747 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f 748 ssize_t remain;
e5281ccd 749 loff_t offset;
40123c1f
EA		 750 char __user *user_data;
751 int page_offset, page_length;
40123c1f
EA		 752
753 user_data = (char __user *) (uintptr_t) args->data_ptr;
754 remain = args->size;
673a394b 755
40123c1f 756 offset = args->offset;
05394f39 757 obj->dirty = 1;
40123c1f
EA		 758
759 while (remain > 0) {
e5281ccd
CW		 760 struct page *page;
761 char *vaddr;
762 int ret;
763
40123c1f
EA		 764 /* Operation in this page
765 *
40123c1f
EA		 766 * page_offset = offset within page
767 * page_length = bytes to copy for this page
768 */
40123c1f
EA		 769 page_offset = offset & (PAGE_SIZE-1);
770 page_length = remain;
771 if ((page_offset + remain) > PAGE_SIZE)
772 page_length = PAGE_SIZE - page_offset;
773
e5281ccd
CW		 774 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
775 GFP_HIGHUSER | __GFP_RECLAIMABLE);
776 if (IS_ERR(page))
777 return PTR_ERR(page);
778
779 vaddr = kmap_atomic(page, KM_USER0);
780 ret = __copy_from_user_inatomic(vaddr + page_offset,
781 user_data,
782 page_length);
783 kunmap_atomic(vaddr, KM_USER0);
784
785 set_page_dirty(page);
786 mark_page_accessed(page);
787 page_cache_release(page);
788
789 /* If we get a fault while copying data, then (presumably) our
790 * source page isn't available. Return the error and we'll
791 * retry in the slow path.
792 */
793 if (ret)
fbd5a26d 794 return -EFAULT;
40123c1f
EA		 795
796 remain -= page_length;
797 user_data += page_length;
798 offset += page_length;
799 }
800
fbd5a26d 801 return 0;
40123c1f
EA		 802}
803
804/**
805 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
806 * the memory and maps it using kmap_atomic for copying.
807 *
808 * This avoids taking mmap_sem for faulting on the user's address while the
809 * struct_mutex is held.
810 */
811static int
05394f39
CW		 812i915_gem_shmem_pwrite_slow(struct drm_device *dev,
813 struct drm_i915_gem_object *obj,
40123c1f 814 struct drm_i915_gem_pwrite *args,
05394f39 815 struct drm_file *file)
40123c1f 816{
05394f39 817 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
40123c1f
EA		 818 struct mm_struct *mm = current->mm;
819 struct page **user_pages;
820 ssize_t remain;
821 loff_t offset, pinned_pages, i;
822 loff_t first_data_page, last_data_page, num_pages;
e5281ccd 823 int shmem_page_offset;
40123c1f
EA		 824 int data_page_index, data_page_offset;
825 int page_length;
826 int ret;
827 uint64_t data_ptr = args->data_ptr;
280b713b 828 int do_bit17_swizzling;
40123c1f
EA		 829
830 remain = args->size;
831
832 /* Pin the user pages containing the data. We can't fault while
833 * holding the struct mutex, and all of the pwrite implementations
834 * want to hold it while dereferencing the user data.
835 */
836 first_data_page = data_ptr / PAGE_SIZE;
837 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
838 num_pages = last_data_page - first_data_page + 1;
839
4f27b75d 840 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
40123c1f
EA		 841 if (user_pages == NULL)
842 return -ENOMEM;
843
fbd5a26d 844 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 845 down_read(&mm->mmap_sem);
846 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
847 num_pages, 0, 0, user_pages, NULL);
848 up_read(&mm->mmap_sem);
fbd5a26d 849 mutex_lock(&dev->struct_mutex);
40123c1f
EA		 850 if (pinned_pages < num_pages) {
851 ret = -EFAULT;
fbd5a26d 852 goto out;
673a394b
EA		 853 }
854
fbd5a26d 855 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
07f73f69 856 if (ret)
fbd5a26d 857 goto out;
40123c1f 858
fbd5a26d 859 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
40123c1f 860
673a394b 861 offset = args->offset;
05394f39 862 obj->dirty = 1;
673a394b 863
40123c1f 864 while (remain > 0) {
e5281ccd
CW		 865 struct page *page;
866
40123c1f
EA		 867 /* Operation in this page
868 *
40123c1f
EA		 869 * shmem_page_offset = offset within page in shmem file
870 * data_page_index = page number in get_user_pages return
871 * data_page_offset = offset with data_page_index page.
872 * page_length = bytes to copy for this page
873 */
40123c1f
EA		 874 shmem_page_offset = offset & ~PAGE_MASK;
875 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
876 data_page_offset = data_ptr & ~PAGE_MASK;
877
878 page_length = remain;
879 if ((shmem_page_offset + page_length) > PAGE_SIZE)
880 page_length = PAGE_SIZE - shmem_page_offset;
881 if ((data_page_offset + page_length) > PAGE_SIZE)
882 page_length = PAGE_SIZE - data_page_offset;
883
e5281ccd
CW		 884 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
885 GFP_HIGHUSER | __GFP_RECLAIMABLE);
886 if (IS_ERR(page)) {
887 ret = PTR_ERR(page);
888 goto out;
889 }
890
280b713b 891 if (do_bit17_swizzling) {
e5281ccd 892 slow_shmem_bit17_copy(page,
280b713b
EA		 893 shmem_page_offset,
894 user_pages[data_page_index],
895 data_page_offset,
99a03df5
CW		 896 page_length,
897 0);
898 } else {
e5281ccd 899 slow_shmem_copy(page,
99a03df5
CW		 900 shmem_page_offset,
901 user_pages[data_page_index],
902 data_page_offset,
903 page_length);
280b713b 904 }
40123c1f 905
e5281ccd
CW		 906 set_page_dirty(page);
907 mark_page_accessed(page);
908 page_cache_release(page);
909
40123c1f
EA		 910 remain -= page_length;
911 data_ptr += page_length;
912 offset += page_length;
673a394b
EA		 913 }
914
fbd5a26d 915out:
40123c1f
EA		 916 for (i = 0; i < pinned_pages; i++)
917 page_cache_release(user_pages[i]);
8e7d2b2c 918 drm_free_large(user_pages);
673a394b 919
40123c1f 920 return ret;
673a394b
EA		 921}
922
923/**
924 * Writes data to the object referenced by handle.
925 *
926 * On error, the contents of the buffer that were to be modified are undefined.
927 */
928int
929i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
fbd5a26d 930 struct drm_file *file)
673a394b
EA		 931{
932 struct drm_i915_gem_pwrite *args = data;
05394f39 933 struct drm_i915_gem_object *obj;
51311d0a
CW		 934 int ret;
935
936 if (args->size == 0)
937 return 0;
938
939 if (!access_ok(VERIFY_READ,
940 (char __user *)(uintptr_t)args->data_ptr,
941 args->size))
942 return -EFAULT;
943
944 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
945 args->size);
946 if (ret)
947 return -EFAULT;
673a394b 948
fbd5a26d 949 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 950 if (ret)
fbd5a26d 951 return ret;
1d7cfea1 952
05394f39 953 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 954 if (obj == NULL) {
955 ret = -ENOENT;
956 goto unlock;
fbd5a26d 957 }
673a394b 958
7dcd2499 959 /* Bounds check destination. */
05394f39
CW		 960 if (args->offset > obj->base.size ||
961 args->size > obj->base.size - args->offset) {
ce9d419d 962 ret = -EINVAL;
35b62a89 963 goto out;
ce9d419d
CW		 964 }
965
673a394b
EA		 966 /* We can only do the GTT pwrite on untiled buffers, as otherwise
967 * it would end up going through the fenced access, and we'll get
968 * different detiling behavior between reading and writing.
969 * pread/pwrite currently are reading and writing from the CPU
970 * perspective, requiring manual detiling by the client.
971 */
05394f39 972 if (obj->phys_obj)
fbd5a26d 973 ret = i915_gem_phys_pwrite(dev, obj, args, file);
d9e86c0e 974 else if (obj->gtt_space &&
05394f39 975 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
75e9e915 976 ret = i915_gem_object_pin(obj, 0, true);
fbd5a26d
CW		 977 if (ret)
978 goto out;
979
d9e86c0e
CW		 980 ret = i915_gem_object_set_to_gtt_domain(obj, true);
981 if (ret)
982 goto out_unpin;
983
984 ret = i915_gem_object_put_fence(obj);
fbd5a26d
CW		 985 if (ret)
986 goto out_unpin;
987
988 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
989 if (ret == -EFAULT)
990 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
991
992out_unpin:
993 i915_gem_object_unpin(obj);
40123c1f 994 } else {
fbd5a26d
CW		 995 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
996 if (ret)
e5281ccd 997 goto out;
673a394b 998
fbd5a26d
CW		 999 ret = -EFAULT;
1000 if (!i915_gem_object_needs_bit17_swizzle(obj))
1001 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1002 if (ret == -EFAULT)
1003 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
fbd5a26d 1004 }
673a394b 1005
35b62a89 1006out:
05394f39 1007 drm_gem_object_unreference(&obj->base);
1d7cfea1 1008unlock:
fbd5a26d 1009 mutex_unlock(&dev->struct_mutex);
673a394b
EA		 1010 return ret;
1011}
1012
1013/**
2ef7eeaa
EA		 1014 * Called when user space prepares to use an object with the CPU, either
1015 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1016 */
1017int
1018i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
05394f39 1019 struct drm_file *file)
673a394b
EA		 1020{
1021 struct drm_i915_gem_set_domain *args = data;
05394f39 1022 struct drm_i915_gem_object *obj;
2ef7eeaa
EA		 1023 uint32_t read_domains = args->read_domains;
1024 uint32_t write_domain = args->write_domain;
673a394b
EA		 1025 int ret;
1026
1027 if (!(dev->driver->driver_features & DRIVER_GEM))
1028 return -ENODEV;
1029
2ef7eeaa 1030 /* Only handle setting domains to types used by the CPU. */
21d509e3 1031 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1032 return -EINVAL;
1033
21d509e3 1034 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1035 return -EINVAL;
1036
1037 /* Having something in the write domain implies it's in the read
1038 * domain, and only that read domain. Enforce that in the request.
1039 */
1040 if (write_domain != 0 && read_domains != write_domain)
1041 return -EINVAL;
1042
76c1dec1 1043 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1044 if (ret)
76c1dec1 1045 return ret;
1d7cfea1 1046
05394f39 1047 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1048 if (obj == NULL) {
1049 ret = -ENOENT;
1050 goto unlock;
76c1dec1 1051 }
673a394b 1052
2ef7eeaa
EA		 1053 if (read_domains & I915_GEM_DOMAIN_GTT) {
1054 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392
EA		 1055
1056 /* Silently promote "you're not bound, there was nothing to do"
1057 * to success, since the client was just asking us to
1058 * make sure everything was done.
1059 */
1060 if (ret == -EINVAL)
1061 ret = 0;
2ef7eeaa 1062 } else {
e47c68e9 1063 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1064 }
1065
05394f39 1066 drm_gem_object_unreference(&obj->base);
1d7cfea1 1067unlock:
673a394b
EA		 1068 mutex_unlock(&dev->struct_mutex);
1069 return ret;
1070}
1071
1072/**
1073 * Called when user space has done writes to this buffer
1074 */
1075int
1076i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
05394f39 1077 struct drm_file *file)
673a394b
EA		 1078{
1079 struct drm_i915_gem_sw_finish *args = data;
05394f39 1080 struct drm_i915_gem_object *obj;
673a394b
EA		 1081 int ret = 0;
1082
1083 if (!(dev->driver->driver_features & DRIVER_GEM))
1084 return -ENODEV;
1085
76c1dec1 1086 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1087 if (ret)
76c1dec1 1088 return ret;
1d7cfea1 1089
05394f39 1090 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 1091 if (obj == NULL) {
1d7cfea1
CW		 1092 ret = -ENOENT;
1093 goto unlock;
673a394b
EA		 1094 }
1095
673a394b 1096 /* Pinned buffers may be scanout, so flush the cache */
05394f39 1097 if (obj->pin_count)
e47c68e9
EA		 1098 i915_gem_object_flush_cpu_write_domain(obj);
1099
05394f39 1100 drm_gem_object_unreference(&obj->base);
1d7cfea1 1101unlock:
673a394b
EA		 1102 mutex_unlock(&dev->struct_mutex);
1103 return ret;
1104}
1105
1106/**
1107 * Maps the contents of an object, returning the address it is mapped
1108 * into.
1109 *
1110 * While the mapping holds a reference on the contents of the object, it doesn't
1111 * imply a ref on the object itself.
1112 */
1113int
1114i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
05394f39 1115 struct drm_file *file)
673a394b 1116{
da761a6e 1117 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1118 struct drm_i915_gem_mmap *args = data;
1119 struct drm_gem_object *obj;
1120 loff_t offset;
1121 unsigned long addr;
1122
1123 if (!(dev->driver->driver_features & DRIVER_GEM))
1124 return -ENODEV;
1125
05394f39 1126 obj = drm_gem_object_lookup(dev, file, args->handle);
673a394b 1127 if (obj == NULL)
bf79cb91 1128 return -ENOENT;
673a394b 1129
da761a6e
CW		 1130 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1131 drm_gem_object_unreference_unlocked(obj);
1132 return -E2BIG;
1133 }
1134
673a394b
EA		 1135 offset = args->offset;
1136
1137 down_write(&current->mm->mmap_sem);
1138 addr = do_mmap(obj->filp, 0, args->size,
1139 PROT_READ | PROT_WRITE, MAP_SHARED,
1140 args->offset);
1141 up_write(&current->mm->mmap_sem);
bc9025bd 1142 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1143 if (IS_ERR((void *)addr))
1144 return addr;
1145
1146 args->addr_ptr = (uint64_t) addr;
1147
1148 return 0;
1149}
1150
de151cf6
JB		 1151/**
1152 * i915_gem_fault - fault a page into the GTT
1153 * vma: VMA in question
1154 * vmf: fault info
1155 *
1156 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1157 * from userspace. The fault handler takes care of binding the object to
1158 * the GTT (if needed), allocating and programming a fence register (again,
1159 * only if needed based on whether the old reg is still valid or the object
1160 * is tiled) and inserting a new PTE into the faulting process.
1161 *
1162 * Note that the faulting process may involve evicting existing objects
1163 * from the GTT and/or fence registers to make room. So performance may
1164 * suffer if the GTT working set is large or there are few fence registers
1165 * left.
1166 */
1167int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1168{
05394f39
CW		 1169 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1170 struct drm_device *dev = obj->base.dev;
7d1c4804 1171 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 1172 pgoff_t page_offset;
1173 unsigned long pfn;
1174 int ret = 0;
0f973f27 1175 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1176
1177 /* We don't use vmf->pgoff since that has the fake offset */
1178 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1179 PAGE_SHIFT;
1180
1181 /* Now bind it into the GTT if needed */
1182 mutex_lock(&dev->struct_mutex);
a00b10c3 1183
919926ae
CW		 1184 if (!obj->map_and_fenceable) {
1185 ret = i915_gem_object_unbind(obj);
1186 if (ret)
1187 goto unlock;
a00b10c3 1188 }
05394f39 1189 if (!obj->gtt_space) {
75e9e915 1190 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
c715089f
CW		 1191 if (ret)
1192 goto unlock;
de151cf6
JB		 1193 }
1194
4a684a41
CW		 1195 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1196 if (ret)
1197 goto unlock;
1198
d9e86c0e
CW		 1199 if (obj->tiling_mode == I915_TILING_NONE)
1200 ret = i915_gem_object_put_fence(obj);
1201 else
1202 ret = i915_gem_object_get_fence(obj, NULL, true);
1203 if (ret)
1204 goto unlock;
de151cf6 1205
05394f39
CW		 1206 if (i915_gem_object_is_inactive(obj))
1207 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
7d1c4804 1208
6299f992
CW		 1209 obj->fault_mappable = true;
1210
05394f39 1211 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
de151cf6
JB		 1212 page_offset;
1213
1214 /* Finally, remap it using the new GTT offset */
1215 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1216unlock:
de151cf6
JB		 1217 mutex_unlock(&dev->struct_mutex);
1218
1219 switch (ret) {
045e769a
CW		 1220 case -EAGAIN:
1221 set_need_resched();
c715089f
CW		 1222 case 0:
1223 case -ERESTARTSYS:
1224 return VM_FAULT_NOPAGE;
de151cf6 1225 case -ENOMEM:
de151cf6 1226 return VM_FAULT_OOM;
de151cf6 1227 default:
c715089f 1228 return VM_FAULT_SIGBUS;
de151cf6
JB		 1229 }
1230}
1231
1232/**
1233 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1234 * @obj: obj in question
1235 *
1236 * GEM memory mapping works by handing back to userspace a fake mmap offset
1237 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1238 * up the object based on the offset and sets up the various memory mapping
1239 * structures.
1240 *
1241 * This routine allocates and attaches a fake offset for @obj.
1242 */
1243static int
05394f39 1244i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
de151cf6 1245{
05394f39 1246 struct drm_device *dev = obj->base.dev;
de151cf6 1247 struct drm_gem_mm *mm = dev->mm_private;
de151cf6 1248 struct drm_map_list *list;
f77d390c 1249 struct drm_local_map *map;
de151cf6
JB		 1250 int ret = 0;
1251
1252 /* Set the object up for mmap'ing */
05394f39 1253 list = &obj->base.map_list;
9a298b2a 1254 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1255 if (!list->map)
1256 return -ENOMEM;
1257
1258 map = list->map;
1259 map->type = _DRM_GEM;
05394f39 1260 map->size = obj->base.size;
de151cf6
JB		 1261 map->handle = obj;
1262
1263 /* Get a DRM GEM mmap offset allocated... */
1264 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
05394f39
CW		 1265 obj->base.size / PAGE_SIZE,
1266 0, 0);
de151cf6 1267 if (!list->file_offset_node) {
05394f39
CW		 1268 DRM_ERROR("failed to allocate offset for bo %d\n",
1269 obj->base.name);
9e0ae534 1270 ret = -ENOSPC;
de151cf6
JB		 1271 goto out_free_list;
1272 }
1273
1274 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
05394f39
CW		 1275 obj->base.size / PAGE_SIZE,
1276 0);
de151cf6
JB		 1277 if (!list->file_offset_node) {
1278 ret = -ENOMEM;
1279 goto out_free_list;
1280 }
1281
1282 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1283 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1284 if (ret) {
de151cf6
JB		 1285 DRM_ERROR("failed to add to map hash\n");
1286 goto out_free_mm;
1287 }
1288
de151cf6
JB		 1289 return 0;
1290
1291out_free_mm:
1292 drm_mm_put_block(list->file_offset_node);
1293out_free_list:
9a298b2a 1294 kfree(list->map);
39a01d1f 1295 list->map = NULL;
de151cf6
JB		 1296
1297 return ret;
1298}
1299
901782b2
CW		 1300/**
1301 * i915_gem_release_mmap - remove physical page mappings
1302 * @obj: obj in question
1303 *
af901ca1 1304 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1305 * relinquish ownership of the pages back to the system.
1306 *
1307 * It is vital that we remove the page mapping if we have mapped a tiled
1308 * object through the GTT and then lose the fence register due to
1309 * resource pressure. Similarly if the object has been moved out of the
1310 * aperture, than pages mapped into userspace must be revoked. Removing the
1311 * mapping will then trigger a page fault on the next user access, allowing
1312 * fixup by i915_gem_fault().
1313 */
d05ca301 1314void
05394f39 1315i915_gem_release_mmap(struct drm_i915_gem_object *obj)
901782b2 1316{
6299f992
CW		 1317 if (!obj->fault_mappable)
1318 return;
901782b2 1319
6299f992
CW		 1320 unmap_mapping_range(obj->base.dev->dev_mapping,
1321 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1322 obj->base.size, 1);
fb7d516a 1323
6299f992 1324 obj->fault_mappable = false;
901782b2
CW		 1325}
1326
ab00b3e5 1327static void
05394f39 1328i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
ab00b3e5 1329{
05394f39 1330 struct drm_device *dev = obj->base.dev;
ab00b3e5 1331 struct drm_gem_mm *mm = dev->mm_private;
05394f39 1332 struct drm_map_list *list = &obj->base.map_list;
ab00b3e5 1333
ab00b3e5 1334 drm_ht_remove_item(&mm->offset_hash, &list->hash);
39a01d1f
CW		 1335 drm_mm_put_block(list->file_offset_node);
1336 kfree(list->map);
1337 list->map = NULL;
ab00b3e5
JB		 1338}
1339
92b88aeb
CW		 1340static uint32_t
1341i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1342{
1343 struct drm_device *dev = obj->base.dev;
1344 uint32_t size;
1345
1346 if (INTEL_INFO(dev)->gen >= 4 ||
1347 obj->tiling_mode == I915_TILING_NONE)
1348 return obj->base.size;
1349
1350 /* Previous chips need a power-of-two fence region when tiling */
1351 if (INTEL_INFO(dev)->gen == 3)
1352 size = 1024*1024;
1353 else
1354 size = 512*1024;
1355
1356 while (size < obj->base.size)
1357 size <<= 1;
1358
1359 return size;
1360}
1361
de151cf6
JB		 1362/**
1363 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1364 * @obj: object to check
1365 *
1366 * Return the required GTT alignment for an object, taking into account
5e783301 1367 * potential fence register mapping.
de151cf6
JB		 1368 */
1369static uint32_t
05394f39 1370i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
de151cf6 1371{
05394f39 1372 struct drm_device *dev = obj->base.dev;
de151cf6
JB		 1373
1374 /*
1375 * Minimum alignment is 4k (GTT page size), but might be greater
1376 * if a fence register is needed for the object.
1377 */
a00b10c3 1378 if (INTEL_INFO(dev)->gen >= 4 ||
05394f39 1379 obj->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1380 return 4096;
1381
a00b10c3
CW		 1382 /*
1383 * Previous chips need to be aligned to the size of the smallest
1384 * fence register that can contain the object.
1385 */
05394f39 1386 return i915_gem_get_gtt_size(obj);
a00b10c3
CW		 1387}
1388
5e783301
DV		 1389/**
1390 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1391 * unfenced object
1392 * @obj: object to check
1393 *
1394 * Return the required GTT alignment for an object, only taking into account
1395 * unfenced tiled surface requirements.
1396 */
1397static uint32_t
05394f39 1398i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
5e783301 1399{
05394f39 1400 struct drm_device *dev = obj->base.dev;
5e783301
DV		 1401 int tile_height;
1402
1403 /*
1404 * Minimum alignment is 4k (GTT page size) for sane hw.
1405 */
1406 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
05394f39 1407 obj->tiling_mode == I915_TILING_NONE)
5e783301
DV		 1408 return 4096;
1409
1410 /*
1411 * Older chips need unfenced tiled buffers to be aligned to the left
1412 * edge of an even tile row (where tile rows are counted as if the bo is
1413 * placed in a fenced gtt region).
1414 */
1415 if (IS_GEN2(dev) ||
05394f39 1416 (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
5e783301
DV		 1417 tile_height = 32;
1418 else
1419 tile_height = 8;
1420
05394f39 1421 return tile_height * obj->stride * 2;
5e783301
DV		 1422}
1423
de151cf6
JB		 1424/**
1425 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1426 * @dev: DRM device
1427 * @data: GTT mapping ioctl data
05394f39 1428 * @file: GEM object info
de151cf6
JB		 1429 *
1430 * Simply returns the fake offset to userspace so it can mmap it.
1431 * The mmap call will end up in drm_gem_mmap(), which will set things
1432 * up so we can get faults in the handler above.
1433 *
1434 * The fault handler will take care of binding the object into the GTT
1435 * (since it may have been evicted to make room for something), allocating
1436 * a fence register, and mapping the appropriate aperture address into
1437 * userspace.
1438 */
1439int
1440i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
05394f39 1441 struct drm_file *file)
de151cf6 1442{
da761a6e 1443 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6 1444 struct drm_i915_gem_mmap_gtt *args = data;
05394f39 1445 struct drm_i915_gem_object *obj;
de151cf6
JB		 1446 int ret;
1447
1448 if (!(dev->driver->driver_features & DRIVER_GEM))
1449 return -ENODEV;
1450
76c1dec1 1451 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 1452 if (ret)
76c1dec1 1453 return ret;
de151cf6 1454
05394f39 1455 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1d7cfea1
CW		 1456 if (obj == NULL) {
1457 ret = -ENOENT;
1458 goto unlock;
1459 }
de151cf6 1460
05394f39 1461 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
da761a6e
CW		 1462 ret = -E2BIG;
1463 goto unlock;
1464 }
1465
05394f39 1466 if (obj->madv != I915_MADV_WILLNEED) {
ab18282d 1467 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1d7cfea1
CW		 1468 ret = -EINVAL;
1469 goto out;
ab18282d
CW		 1470 }
1471
05394f39 1472 if (!obj->base.map_list.map) {
de151cf6 1473 ret = i915_gem_create_mmap_offset(obj);
1d7cfea1
CW		 1474 if (ret)
1475 goto out;
de151cf6
JB		 1476 }
1477
05394f39 1478 args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
de151cf6 1479
1d7cfea1 1480out:
05394f39 1481 drm_gem_object_unreference(&obj->base);
1d7cfea1 1482unlock:
de151cf6 1483 mutex_unlock(&dev->struct_mutex);
1d7cfea1 1484 return ret;
de151cf6
JB		 1485}
1486
e5281ccd 1487static int
05394f39 1488i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
e5281ccd
CW		 1489 gfp_t gfpmask)
1490{
e5281ccd
CW		 1491 int page_count, i;
1492 struct address_space *mapping;
1493 struct inode *inode;
1494 struct page *page;
1495
1496 /* Get the list of pages out of our struct file. They'll be pinned
1497 * at this point until we release them.
1498 */
05394f39
CW		 1499 page_count = obj->base.size / PAGE_SIZE;
1500 BUG_ON(obj->pages != NULL);
1501 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1502 if (obj->pages == NULL)
e5281ccd
CW		 1503 return -ENOMEM;
1504
05394f39 1505 inode = obj->base.filp->f_path.dentry->d_inode;
e5281ccd
CW		 1506 mapping = inode->i_mapping;
1507 for (i = 0; i < page_count; i++) {
1508 page = read_cache_page_gfp(mapping, i,
1509 GFP_HIGHUSER |
1510 __GFP_COLD |
1511 __GFP_RECLAIMABLE |
1512 gfpmask);
1513 if (IS_ERR(page))
1514 goto err_pages;
1515
05394f39 1516 obj->pages[i] = page;
e5281ccd
CW		 1517 }
1518
05394f39 1519 if (obj->tiling_mode != I915_TILING_NONE)
e5281ccd
CW		 1520 i915_gem_object_do_bit_17_swizzle(obj);
1521
1522 return 0;
1523
1524err_pages:
1525 while (i--)
05394f39 1526 page_cache_release(obj->pages[i]);
e5281ccd 1527
05394f39
CW		 1528 drm_free_large(obj->pages);
1529 obj->pages = NULL;
e5281ccd
CW		 1530 return PTR_ERR(page);
1531}
1532
5cdf5881 1533static void
05394f39 1534i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
673a394b 1535{
05394f39 1536 int page_count = obj->base.size / PAGE_SIZE;
673a394b
EA		 1537 int i;
1538
05394f39 1539 BUG_ON(obj->madv == __I915_MADV_PURGED);
673a394b 1540
05394f39 1541 if (obj->tiling_mode != I915_TILING_NONE)
280b713b
EA		 1542 i915_gem_object_save_bit_17_swizzle(obj);
1543
05394f39
CW		 1544 if (obj->madv == I915_MADV_DONTNEED)
1545 obj->dirty = 0;
3ef94daa
CW		 1546
1547 for (i = 0; i < page_count; i++) {
05394f39
CW		 1548 if (obj->dirty)
1549 set_page_dirty(obj->pages[i]);
3ef94daa 1550
05394f39
CW		 1551 if (obj->madv == I915_MADV_WILLNEED)
1552 mark_page_accessed(obj->pages[i]);
3ef94daa 1553
05394f39 1554 page_cache_release(obj->pages[i]);
3ef94daa 1555 }
05394f39 1556 obj->dirty = 0;
673a394b 1557
05394f39
CW		 1558 drm_free_large(obj->pages);
1559 obj->pages = NULL;
673a394b
EA		 1560}
1561
54cf91dc 1562void
05394f39 1563i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1ec14ad3
CW		 1564 struct intel_ring_buffer *ring,
1565 u32 seqno)
673a394b 1566{
05394f39 1567 struct drm_device *dev = obj->base.dev;
69dc4987 1568 struct drm_i915_private *dev_priv = dev->dev_private;
617dbe27 1569
852835f3 1570 BUG_ON(ring == NULL);
05394f39 1571 obj->ring = ring;
673a394b
EA		 1572
1573 /* Add a reference if we're newly entering the active list. */
05394f39
CW		 1574 if (!obj->active) {
1575 drm_gem_object_reference(&obj->base);
1576 obj->active = 1;
673a394b 1577 }
e35a41de 1578
673a394b 1579 /* Move from whatever list we were on to the tail of execution. */
05394f39
CW		 1580 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1581 list_move_tail(&obj->ring_list, &ring->active_list);
caea7476 1582
05394f39 1583 obj->last_rendering_seqno = seqno;
caea7476
CW		 1584 if (obj->fenced_gpu_access) {
1585 struct drm_i915_fence_reg *reg;
1586
1587 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1588
1589 obj->last_fenced_seqno = seqno;
1590 obj->last_fenced_ring = ring;
1591
1592 reg = &dev_priv->fence_regs[obj->fence_reg];
1593 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1594 }
1595}
1596
1597static void
1598i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1599{
1600 list_del_init(&obj->ring_list);
1601 obj->last_rendering_seqno = 0;
673a394b
EA		 1602}
1603
ce44b0ea 1604static void
05394f39 1605i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
ce44b0ea 1606{
05394f39 1607 struct drm_device *dev = obj->base.dev;
ce44b0ea 1608 drm_i915_private_t *dev_priv = dev->dev_private;
ce44b0ea 1609
05394f39
CW		 1610 BUG_ON(!obj->active);
1611 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
caea7476
CW		 1612
1613 i915_gem_object_move_off_active(obj);
1614}
1615
1616static void
1617i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1618{
1619 struct drm_device *dev = obj->base.dev;
1620 struct drm_i915_private *dev_priv = dev->dev_private;
1621
1622 if (obj->pin_count != 0)
1623 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1624 else
1625 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1626
1627 BUG_ON(!list_empty(&obj->gpu_write_list));
1628 BUG_ON(!obj->active);
1629 obj->ring = NULL;
1630
1631 i915_gem_object_move_off_active(obj);
1632 obj->fenced_gpu_access = false;
caea7476
CW		 1633
1634 obj->active = 0;
87ca9c8a 1635 obj->pending_gpu_write = false;
caea7476
CW		 1636 drm_gem_object_unreference(&obj->base);
1637
1638 WARN_ON(i915_verify_lists(dev));
ce44b0ea 1639}
673a394b 1640
963b4836
CW		 1641/* Immediately discard the backing storage */
1642static void
05394f39 1643i915_gem_object_truncate(struct drm_i915_gem_object *obj)
963b4836 1644{
bb6baf76 1645 struct inode *inode;
963b4836 1646
ae9fed6b
CW		 1647 /* Our goal here is to return as much of the memory as
1648 * is possible back to the system as we are called from OOM.
1649 * To do this we must instruct the shmfs to drop all of its
1650 * backing pages, *now*. Here we mirror the actions taken
1651 * when by shmem_delete_inode() to release the backing store.
1652 */
05394f39 1653 inode = obj->base.filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1654 truncate_inode_pages(inode->i_mapping, 0);
1655 if (inode->i_op->truncate_range)
1656 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76 1657
05394f39 1658 obj->madv = __I915_MADV_PURGED;
963b4836
CW		 1659}
1660
1661static inline int
05394f39 1662i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
963b4836 1663{
05394f39 1664 return obj->madv == I915_MADV_DONTNEED;
963b4836
CW		 1665}
1666
63560396
DV		 1667static void
1668i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1669 uint32_t flush_domains,
852835f3 1670 struct intel_ring_buffer *ring)
63560396 1671{
05394f39 1672 struct drm_i915_gem_object *obj, *next;
63560396 1673
05394f39 1674 list_for_each_entry_safe(obj, next,
64193406 1675 &ring->gpu_write_list,
63560396 1676 gpu_write_list) {
05394f39
CW		 1677 if (obj->base.write_domain & flush_domains) {
1678 uint32_t old_write_domain = obj->base.write_domain;
63560396 1679
05394f39
CW		 1680 obj->base.write_domain = 0;
1681 list_del_init(&obj->gpu_write_list);
1ec14ad3
CW		 1682 i915_gem_object_move_to_active(obj, ring,
1683 i915_gem_next_request_seqno(dev, ring));
63560396 1684
63560396 1685 trace_i915_gem_object_change_domain(obj,
05394f39 1686 obj->base.read_domains,
63560396
DV		 1687 old_write_domain);
1688 }
1689 }
1690}
8187a2b7 1691
3cce469c 1692int
8a1a49f9 1693i915_add_request(struct drm_device *dev,
f787a5f5 1694 struct drm_file *file,
8dc5d147 1695 struct drm_i915_gem_request *request,
8a1a49f9 1696 struct intel_ring_buffer *ring)
673a394b
EA		 1697{
1698 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1699 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1700 uint32_t seqno;
1701 int was_empty;
3cce469c
CW		 1702 int ret;
1703
1704 BUG_ON(request == NULL);
673a394b 1705
f787a5f5
CW		 1706 if (file != NULL)
1707 file_priv = file->driver_priv;
b962442e 1708
3cce469c
CW		 1709 ret = ring->add_request(ring, &seqno);
1710 if (ret)
1711 return ret;
673a394b 1712
a56ba56c 1713 ring->outstanding_lazy_request = false;
673a394b
EA		 1714
1715 request->seqno = seqno;
852835f3 1716 request->ring = ring;
673a394b 1717 request->emitted_jiffies = jiffies;
852835f3
ZN		 1718 was_empty = list_empty(&ring->request_list);
1719 list_add_tail(&request->list, &ring->request_list);
1720
f787a5f5 1721 if (file_priv) {
1c25595f 1722 spin_lock(&file_priv->mm.lock);
f787a5f5 1723 request->file_priv = file_priv;
b962442e 1724 list_add_tail(&request->client_list,
f787a5f5 1725 &file_priv->mm.request_list);
1c25595f 1726 spin_unlock(&file_priv->mm.lock);
b962442e 1727 }
673a394b 1728
f65d9421 1729 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1730 mod_timer(&dev_priv->hangcheck_timer,
1731 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1732 if (was_empty)
b3b079db
CW		 1733 queue_delayed_work(dev_priv->wq,
1734 &dev_priv->mm.retire_work, HZ);
f65d9421 1735 }
3cce469c 1736 return 0;
673a394b
EA		 1737}
1738
f787a5f5
CW		 1739static inline void
1740i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1741{
1c25595f 1742 struct drm_i915_file_private *file_priv = request->file_priv;
673a394b 1743
1c25595f
CW		 1744 if (!file_priv)
1745 return;
1c5d22f7 1746
1c25595f
CW		 1747 spin_lock(&file_priv->mm.lock);
1748 list_del(&request->client_list);
1749 request->file_priv = NULL;
1750 spin_unlock(&file_priv->mm.lock);
673a394b 1751}
673a394b 1752
dfaae392
CW		 1753static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1754 struct intel_ring_buffer *ring)
9375e446 1755{
dfaae392
CW		 1756 while (!list_empty(&ring->request_list)) {
1757 struct drm_i915_gem_request *request;
673a394b 1758
dfaae392
CW		 1759 request = list_first_entry(&ring->request_list,
1760 struct drm_i915_gem_request,
1761 list);
de151cf6 1762
dfaae392 1763 list_del(&request->list);
f787a5f5 1764 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1765 kfree(request);
1766 }
673a394b 1767
dfaae392 1768 while (!list_empty(&ring->active_list)) {
05394f39 1769 struct drm_i915_gem_object *obj;
9375e446 1770
05394f39
CW		 1771 obj = list_first_entry(&ring->active_list,
1772 struct drm_i915_gem_object,
1773 ring_list);
9375e446 1774
05394f39
CW		 1775 obj->base.write_domain = 0;
1776 list_del_init(&obj->gpu_write_list);
1777 i915_gem_object_move_to_inactive(obj);
673a394b
EA		 1778 }
1779}
1780
312817a3
CW		 1781static void i915_gem_reset_fences(struct drm_device *dev)
1782{
1783 struct drm_i915_private *dev_priv = dev->dev_private;
1784 int i;
1785
1786 for (i = 0; i < 16; i++) {
1787 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
7d2cb39c
CW		 1788 struct drm_i915_gem_object *obj = reg->obj;
1789
1790 if (!obj)
1791 continue;
1792
1793 if (obj->tiling_mode)
1794 i915_gem_release_mmap(obj);
1795
d9e86c0e
CW		 1796 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1797 reg->obj->fenced_gpu_access = false;
1798 reg->obj->last_fenced_seqno = 0;
1799 reg->obj->last_fenced_ring = NULL;
1800 i915_gem_clear_fence_reg(dev, reg);
312817a3
CW		 1801 }
1802}
1803
069efc1d 1804void i915_gem_reset(struct drm_device *dev)
673a394b 1805{
77f01230 1806 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1807 struct drm_i915_gem_object *obj;
1ec14ad3 1808 int i;
673a394b 1809
1ec14ad3
CW		 1810 for (i = 0; i < I915_NUM_RINGS; i++)
1811 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
dfaae392
CW		 1812
1813 /* Remove anything from the flushing lists. The GPU cache is likely
1814 * to be lost on reset along with the data, so simply move the
1815 * lost bo to the inactive list.
1816 */
1817 while (!list_empty(&dev_priv->mm.flushing_list)) {
05394f39
CW		 1818 obj= list_first_entry(&dev_priv->mm.flushing_list,
1819 struct drm_i915_gem_object,
1820 mm_list);
dfaae392 1821
05394f39
CW		 1822 obj->base.write_domain = 0;
1823 list_del_init(&obj->gpu_write_list);
1824 i915_gem_object_move_to_inactive(obj);
dfaae392
CW		 1825 }
1826
1827 /* Move everything out of the GPU domains to ensure we do any
1828 * necessary invalidation upon reuse.
1829 */
05394f39 1830 list_for_each_entry(obj,
77f01230 1831 &dev_priv->mm.inactive_list,
69dc4987 1832 mm_list)
77f01230 1833 {
05394f39 1834 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
77f01230 1835 }
069efc1d
CW		 1836
1837 /* The fence registers are invalidated so clear them out */
312817a3 1838 i915_gem_reset_fences(dev);
673a394b
EA		 1839}
1840
1841/**
1842 * This function clears the request list as sequence numbers are passed.
1843 */
b09a1fec
CW		 1844static void
1845i915_gem_retire_requests_ring(struct drm_device *dev,
1846 struct intel_ring_buffer *ring)
673a394b
EA		 1847{
1848 drm_i915_private_t *dev_priv = dev->dev_private;
1849 uint32_t seqno;
1ec14ad3 1850 int i;
673a394b 1851
b84d5f0c
CW		 1852 if (!ring->status_page.page_addr ||
1853 list_empty(&ring->request_list))
6c0594a3
KW		 1854 return;
1855
23bc5982 1856 WARN_ON(i915_verify_lists(dev));
673a394b 1857
78501eac 1858 seqno = ring->get_seqno(ring);
1ec14ad3
CW		 1859
1860 for (i = 0; i < I915_NUM_RINGS; i++)
1861 if (seqno >= ring->sync_seqno[i])
1862 ring->sync_seqno[i] = 0;
1863
852835f3 1864 while (!list_empty(&ring->request_list)) {
673a394b 1865 struct drm_i915_gem_request *request;
673a394b 1866
852835f3 1867 request = list_first_entry(&ring->request_list,
673a394b
EA		 1868 struct drm_i915_gem_request,
1869 list);
673a394b 1870
dfaae392 1871 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1872 break;
1873
1874 trace_i915_gem_request_retire(dev, request->seqno);
1875
1876 list_del(&request->list);
f787a5f5 1877 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1878 kfree(request);
1879 }
673a394b 1880
b84d5f0c
CW		 1881 /* Move any buffers on the active list that are no longer referenced
1882 * by the ringbuffer to the flushing/inactive lists as appropriate.
1883 */
1884 while (!list_empty(&ring->active_list)) {
05394f39 1885 struct drm_i915_gem_object *obj;
b84d5f0c 1886
05394f39
CW		 1887 obj= list_first_entry(&ring->active_list,
1888 struct drm_i915_gem_object,
1889 ring_list);
673a394b 1890
05394f39 1891 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
673a394b 1892 break;
b84d5f0c 1893
05394f39 1894 if (obj->base.write_domain != 0)
b84d5f0c
CW		 1895 i915_gem_object_move_to_flushing(obj);
1896 else
1897 i915_gem_object_move_to_inactive(obj);
673a394b 1898 }
9d34e5db
CW		 1899
1900 if (unlikely (dev_priv->trace_irq_seqno &&
1901 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1ec14ad3 1902 ring->irq_put(ring);
9d34e5db
CW		 1903 dev_priv->trace_irq_seqno = 0;
1904 }
23bc5982
CW		 1905
1906 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1907}
1908
b09a1fec
CW		 1909void
1910i915_gem_retire_requests(struct drm_device *dev)
1911{
1912 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 1913 int i;
b09a1fec 1914
be72615b 1915 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
05394f39 1916 struct drm_i915_gem_object *obj, *next;
be72615b
CW		 1917
1918 /* We must be careful that during unbind() we do not
1919 * accidentally infinitely recurse into retire requests.
1920 * Currently:
1921 * retire -> free -> unbind -> wait -> retire_ring
1922 */
05394f39 1923 list_for_each_entry_safe(obj, next,
be72615b 1924 &dev_priv->mm.deferred_free_list,
69dc4987 1925 mm_list)
05394f39 1926 i915_gem_free_object_tail(obj);
be72615b
CW		 1927 }
1928
1ec14ad3
CW		 1929 for (i = 0; i < I915_NUM_RINGS; i++)
1930 i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]);
b09a1fec
CW		 1931}
1932
75ef9da2 1933static void
673a394b
EA		 1934i915_gem_retire_work_handler(struct work_struct *work)
1935{
1936 drm_i915_private_t *dev_priv;
1937 struct drm_device *dev;
1938
1939 dev_priv = container_of(work, drm_i915_private_t,
1940 mm.retire_work.work);
1941 dev = dev_priv->dev;
1942
891b48cf
CW		 1943 /* Come back later if the device is busy... */
1944 if (!mutex_trylock(&dev->struct_mutex)) {
1945 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1946 return;
1947 }
1948
b09a1fec 1949 i915_gem_retire_requests(dev);
d1b851fc 1950
6dbe2772 1951 if (!dev_priv->mm.suspended &&
1ec14ad3
CW		 1952 (!list_empty(&dev_priv->ring[RCS].request_list) ||
1953 !list_empty(&dev_priv->ring[VCS].request_list) ||
1954 !list_empty(&dev_priv->ring[BCS].request_list)))
9c9fe1f8 1955 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1956 mutex_unlock(&dev->struct_mutex);
1957}
1958
5a5a0c64 1959int
852835f3 1960i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1961 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 1962{
1963 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1964 u32 ier;
673a394b
EA		 1965 int ret = 0;
1966
1967 BUG_ON(seqno == 0);
1968
ba1234d1 1969 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0
CW		 1970 return -EAGAIN;
1971
5d97eb69 1972 if (seqno == ring->outstanding_lazy_request) {
3cce469c
CW		 1973 struct drm_i915_gem_request *request;
1974
1975 request = kzalloc(sizeof(*request), GFP_KERNEL);
1976 if (request == NULL)
e35a41de 1977 return -ENOMEM;
3cce469c
CW		 1978
1979 ret = i915_add_request(dev, NULL, request, ring);
1980 if (ret) {
1981 kfree(request);
1982 return ret;
1983 }
1984
1985 seqno = request->seqno;
e35a41de 1986 }
ffed1d09 1987
78501eac 1988 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
bad720ff 1989 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 1990 ier = I915_READ(DEIER) | I915_READ(GTIER);
1991 else
1992 ier = I915_READ(IER);
802c7eb6
JB		 1993 if (!ier) {
1994 DRM_ERROR("something (likely vbetool) disabled "
1995 "interrupts, re-enabling\n");
1996 i915_driver_irq_preinstall(dev);
1997 i915_driver_irq_postinstall(dev);
1998 }
1999
1c5d22f7
CW		 2000 trace_i915_gem_request_wait_begin(dev, seqno);
2001
b2223497 2002 ring->waiting_seqno = seqno;
b13c2b96
CW		 2003 if (ring->irq_get(ring)) {
2004 if (interruptible)
2005 ret = wait_event_interruptible(ring->irq_queue,
2006 i915_seqno_passed(ring->get_seqno(ring), seqno)
2007 || atomic_read(&dev_priv->mm.wedged));
2008 else
2009 wait_event(ring->irq_queue,
2010 i915_seqno_passed(ring->get_seqno(ring), seqno)
2011 || atomic_read(&dev_priv->mm.wedged));
2012
2013 ring->irq_put(ring);
b5ba177d
CW		 2014 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2015 seqno) ||
2016 atomic_read(&dev_priv->mm.wedged), 3000))
2017 ret = -EBUSY;
b2223497 2018 ring->waiting_seqno = 0;
1c5d22f7
CW		 2019
2020 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 2021 }
ba1234d1 2022 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2023 ret = -EAGAIN;
673a394b
EA		 2024
2025 if (ret && ret != -ERESTARTSYS)
8bff917c 2026 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
78501eac 2027 __func__, ret, seqno, ring->get_seqno(ring),
8bff917c 2028 dev_priv->next_seqno);
673a394b
EA		 2029
2030 /* Directly dispatch request retiring. While we have the work queue
2031 * to handle this, the waiter on a request often wants an associated
2032 * buffer to have made it to the inactive list, and we would need
2033 * a separate wait queue to handle that.
2034 */
2035 if (ret == 0)
b09a1fec 2036 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 2037
2038 return ret;
2039}
2040
48764bf4
DV		 2041/**
2042 * Waits for a sequence number to be signaled, and cleans up the
2043 * request and object lists appropriately for that event.
2044 */
2045static int
852835f3 2046i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 2047 struct intel_ring_buffer *ring)
48764bf4 2048{
852835f3 2049 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 2050}
2051
673a394b
EA		 2052/**
2053 * Ensures that all rendering to the object has completed and the object is
2054 * safe to unbind from the GTT or access from the CPU.
2055 */
54cf91dc 2056int
05394f39 2057i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2cf34d7b 2058 bool interruptible)
673a394b 2059{
05394f39 2060 struct drm_device *dev = obj->base.dev;
673a394b
EA		 2061 int ret;
2062
e47c68e9
EA		 2063 /* This function only exists to support waiting for existing rendering,
2064 * not for emitting required flushes.
673a394b 2065 */
05394f39 2066 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2067
2068 /* If there is rendering queued on the buffer being evicted, wait for
2069 * it.
2070 */
05394f39 2071 if (obj->active) {
2cf34d7b 2072 ret = i915_do_wait_request(dev,
05394f39 2073 obj->last_rendering_seqno,
2cf34d7b 2074 interruptible,
05394f39 2075 obj->ring);
2cf34d7b 2076 if (ret)
673a394b
EA		 2077 return ret;
2078 }
2079
2080 return 0;
2081}
2082
2083/**
2084 * Unbinds an object from the GTT aperture.
2085 */
0f973f27 2086int
05394f39 2087i915_gem_object_unbind(struct drm_i915_gem_object *obj)
673a394b 2088{
673a394b
EA		 2089 int ret = 0;
2090
05394f39 2091 if (obj->gtt_space == NULL)
673a394b
EA		 2092 return 0;
2093
05394f39 2094 if (obj->pin_count != 0) {
673a394b
EA		 2095 DRM_ERROR("Attempting to unbind pinned buffer\n");
2096 return -EINVAL;
2097 }
2098
5323fd04
EA		 2099 /* blow away mappings if mapped through GTT */
2100 i915_gem_release_mmap(obj);
2101
673a394b
EA		 2102 /* Move the object to the CPU domain to ensure that
2103 * any possible CPU writes while it's not in the GTT
2104 * are flushed when we go to remap it. This will
2105 * also ensure that all pending GPU writes are finished
2106 * before we unbind.
2107 */
e47c68e9 2108 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2109 if (ret == -ERESTARTSYS)
673a394b 2110 return ret;
8dc1775d
CW		 2111 /* Continue on if we fail due to EIO, the GPU is hung so we
2112 * should be safe and we need to cleanup or else we might
2113 * cause memory corruption through use-after-free.
2114 */
812ed492
CW		 2115 if (ret) {
2116 i915_gem_clflush_object(obj);
05394f39 2117 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
812ed492 2118 }
673a394b 2119
96b47b65 2120 /* release the fence reg _after_ flushing */
d9e86c0e
CW		 2121 ret = i915_gem_object_put_fence(obj);
2122 if (ret == -ERESTARTSYS)
2123 return ret;
96b47b65 2124
7c2e6fdf 2125 i915_gem_gtt_unbind_object(obj);
e5281ccd 2126 i915_gem_object_put_pages_gtt(obj);
673a394b 2127
6299f992 2128 list_del_init(&obj->gtt_list);
05394f39 2129 list_del_init(&obj->mm_list);
75e9e915 2130 /* Avoid an unnecessary call to unbind on rebind. */
05394f39 2131 obj->map_and_fenceable = true;
673a394b 2132
05394f39
CW		 2133 drm_mm_put_block(obj->gtt_space);
2134 obj->gtt_space = NULL;
2135 obj->gtt_offset = 0;
673a394b 2136
05394f39 2137 if (i915_gem_object_is_purgeable(obj))
963b4836
CW		 2138 i915_gem_object_truncate(obj);
2139
1c5d22f7
CW		 2140 trace_i915_gem_object_unbind(obj);
2141
8dc1775d 2142 return ret;
673a394b
EA		 2143}
2144
88241785 2145int
54cf91dc
CW		 2146i915_gem_flush_ring(struct drm_device *dev,
2147 struct intel_ring_buffer *ring,
2148 uint32_t invalidate_domains,
2149 uint32_t flush_domains)
2150{
88241785
CW		 2151 int ret;
2152
2153 ret = ring->flush(ring, invalidate_domains, flush_domains);
2154 if (ret)
2155 return ret;
2156
2157 i915_gem_process_flushing_list(dev, flush_domains, ring);
2158 return 0;
54cf91dc
CW		 2159}
2160
a56ba56c
CW		 2161static int i915_ring_idle(struct drm_device *dev,
2162 struct intel_ring_buffer *ring)
2163{
88241785
CW		 2164 int ret;
2165
395b70be 2166 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
64193406
CW		 2167 return 0;
2168
88241785
CW		 2169 if (!list_empty(&ring->gpu_write_list)) {
2170 ret = i915_gem_flush_ring(dev, ring,
0ac74c6b 2171 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
88241785
CW		 2172 if (ret)
2173 return ret;
2174 }
2175
a56ba56c
CW		 2176 return i915_wait_request(dev,
2177 i915_gem_next_request_seqno(dev, ring),
2178 ring);
2179}
2180
b47eb4a2 2181int
4df2faf4
DV		 2182i915_gpu_idle(struct drm_device *dev)
2183{
2184 drm_i915_private_t *dev_priv = dev->dev_private;
2185 bool lists_empty;
1ec14ad3 2186 int ret, i;
4df2faf4 2187
d1b851fc 2188 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
395b70be 2189 list_empty(&dev_priv->mm.active_list));
4df2faf4
DV		 2190 if (lists_empty)
2191 return 0;
2192
2193 /* Flush everything onto the inactive list. */
1ec14ad3
CW		 2194 for (i = 0; i < I915_NUM_RINGS; i++) {
2195 ret = i915_ring_idle(dev, &dev_priv->ring[i]);
2196 if (ret)
2197 return ret;
2198 }
4df2faf4 2199
8a1a49f9 2200 return 0;
4df2faf4
DV		 2201}
2202
c6642782
DV		 2203static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2204 struct intel_ring_buffer *pipelined)
4e901fdc 2205{
05394f39 2206 struct drm_device *dev = obj->base.dev;
4e901fdc 2207 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2208 u32 size = obj->gtt_space->size;
2209 int regnum = obj->fence_reg;
4e901fdc
EA		 2210 uint64_t val;
2211
05394f39 2212 val = (uint64_t)((obj->gtt_offset + size - 4096) &
c6642782 2213 0xfffff000) << 32;
05394f39
CW		 2214 val |= obj->gtt_offset & 0xfffff000;
2215 val |= (uint64_t)((obj->stride / 128) - 1) <<
4e901fdc
EA		 2216 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2217
05394f39 2218 if (obj->tiling_mode == I915_TILING_Y)
4e901fdc
EA		 2219 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2220 val |= I965_FENCE_REG_VALID;
2221
c6642782
DV		 2222 if (pipelined) {
2223 int ret = intel_ring_begin(pipelined, 6);
2224 if (ret)
2225 return ret;
2226
2227 intel_ring_emit(pipelined, MI_NOOP);
2228 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2229 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2230 intel_ring_emit(pipelined, (u32)val);
2231 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2232 intel_ring_emit(pipelined, (u32)(val >> 32));
2233 intel_ring_advance(pipelined);
2234 } else
2235 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2236
2237 return 0;
4e901fdc
EA		 2238}
2239
c6642782
DV		 2240static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2241 struct intel_ring_buffer *pipelined)
de151cf6 2242{
05394f39 2243 struct drm_device *dev = obj->base.dev;
de151cf6 2244 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2245 u32 size = obj->gtt_space->size;
2246 int regnum = obj->fence_reg;
de151cf6
JB		 2247 uint64_t val;
2248
05394f39 2249 val = (uint64_t)((obj->gtt_offset + size - 4096) &
de151cf6 2250 0xfffff000) << 32;
05394f39
CW		 2251 val |= obj->gtt_offset & 0xfffff000;
2252 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2253 if (obj->tiling_mode == I915_TILING_Y)
de151cf6
JB		 2254 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2255 val |= I965_FENCE_REG_VALID;
2256
c6642782
DV		 2257 if (pipelined) {
2258 int ret = intel_ring_begin(pipelined, 6);
2259 if (ret)
2260 return ret;
2261
2262 intel_ring_emit(pipelined, MI_NOOP);
2263 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2264 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2265 intel_ring_emit(pipelined, (u32)val);
2266 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2267 intel_ring_emit(pipelined, (u32)(val >> 32));
2268 intel_ring_advance(pipelined);
2269 } else
2270 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2271
2272 return 0;
de151cf6
JB		 2273}
2274
c6642782
DV		 2275static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2276 struct intel_ring_buffer *pipelined)
de151cf6 2277{
05394f39 2278 struct drm_device *dev = obj->base.dev;
de151cf6 2279 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39 2280 u32 size = obj->gtt_space->size;
c6642782 2281 u32 fence_reg, val, pitch_val;
0f973f27 2282 int tile_width;
de151cf6 2283
c6642782
DV		 2284 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2285 (size & -size) != size ||
2286 (obj->gtt_offset & (size - 1)),
2287 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2288 obj->gtt_offset, obj->map_and_fenceable, size))
2289 return -EINVAL;
de151cf6 2290
c6642782 2291 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
0f973f27 2292 tile_width = 128;
de151cf6 2293 else
0f973f27
JB		 2294 tile_width = 512;
2295
2296 /* Note: pitch better be a power of two tile widths */
05394f39 2297 pitch_val = obj->stride / tile_width;
0f973f27 2298 pitch_val = ffs(pitch_val) - 1;
de151cf6 2299
05394f39
CW		 2300 val = obj->gtt_offset;
2301 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2302 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
a00b10c3 2303 val |= I915_FENCE_SIZE_BITS(size);
de151cf6
JB		 2304 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2305 val |= I830_FENCE_REG_VALID;
2306
05394f39 2307 fence_reg = obj->fence_reg;
a00b10c3
CW		 2308 if (fence_reg < 8)
2309 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f 2310 else
a00b10c3 2311 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
c6642782
DV		 2312
2313 if (pipelined) {
2314 int ret = intel_ring_begin(pipelined, 4);
2315 if (ret)
2316 return ret;
2317
2318 intel_ring_emit(pipelined, MI_NOOP);
2319 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2320 intel_ring_emit(pipelined, fence_reg);
2321 intel_ring_emit(pipelined, val);
2322 intel_ring_advance(pipelined);
2323 } else
2324 I915_WRITE(fence_reg, val);
2325
2326 return 0;
de151cf6
JB		 2327}
2328
c6642782
DV		 2329static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2330 struct intel_ring_buffer *pipelined)
de151cf6 2331{
05394f39 2332 struct drm_device *dev = obj->base.dev;
de151cf6 2333 drm_i915_private_t *dev_priv = dev->dev_private;
05394f39
CW		 2334 u32 size = obj->gtt_space->size;
2335 int regnum = obj->fence_reg;
de151cf6
JB		 2336 uint32_t val;
2337 uint32_t pitch_val;
2338
c6642782
DV		 2339 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2340 (size & -size) != size ||
2341 (obj->gtt_offset & (size - 1)),
2342 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2343 obj->gtt_offset, size))
2344 return -EINVAL;
de151cf6 2345
05394f39 2346 pitch_val = obj->stride / 128;
e76a16de 2347 pitch_val = ffs(pitch_val) - 1;
e76a16de 2348
05394f39
CW		 2349 val = obj->gtt_offset;
2350 if (obj->tiling_mode == I915_TILING_Y)
de151cf6 2351 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
c6642782 2352 val |= I830_FENCE_SIZE_BITS(size);
de151cf6
JB		 2353 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2354 val |= I830_FENCE_REG_VALID;
2355
c6642782
DV		 2356 if (pipelined) {
2357 int ret = intel_ring_begin(pipelined, 4);
2358 if (ret)
2359 return ret;
2360
2361 intel_ring_emit(pipelined, MI_NOOP);
2362 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2363 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2364 intel_ring_emit(pipelined, val);
2365 intel_ring_advance(pipelined);
2366 } else
2367 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2368
2369 return 0;
de151cf6
JB		 2370}
2371
d9e86c0e
CW		 2372static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2373{
2374 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2375}
2376
2377static int
2378i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2379 struct intel_ring_buffer *pipelined,
2380 bool interruptible)
2381{
2382 int ret;
2383
2384 if (obj->fenced_gpu_access) {
88241785
CW		 2385 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2386 ret = i915_gem_flush_ring(obj->base.dev,
2387 obj->last_fenced_ring,
2388 0, obj->base.write_domain);
2389 if (ret)
2390 return ret;
2391 }
d9e86c0e
CW		 2392
2393 obj->fenced_gpu_access = false;
2394 }
2395
2396 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2397 if (!ring_passed_seqno(obj->last_fenced_ring,
2398 obj->last_fenced_seqno)) {
2399 ret = i915_do_wait_request(obj->base.dev,
2400 obj->last_fenced_seqno,
2401 interruptible,
2402 obj->last_fenced_ring);
2403 if (ret)
2404 return ret;
2405 }
2406
2407 obj->last_fenced_seqno = 0;
2408 obj->last_fenced_ring = NULL;
2409 }
2410
63256ec5
CW		 2411 /* Ensure that all CPU reads are completed before installing a fence
2412 * and all writes before removing the fence.
2413 */
2414 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2415 mb();
2416
d9e86c0e
CW		 2417 return 0;
2418}
2419
2420int
2421i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2422{
2423 int ret;
2424
2425 if (obj->tiling_mode)
2426 i915_gem_release_mmap(obj);
2427
2428 ret = i915_gem_object_flush_fence(obj, NULL, true);
2429 if (ret)
2430 return ret;
2431
2432 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2433 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2434 i915_gem_clear_fence_reg(obj->base.dev,
2435 &dev_priv->fence_regs[obj->fence_reg]);
2436
2437 obj->fence_reg = I915_FENCE_REG_NONE;
2438 }
2439
2440 return 0;
2441}
2442
2443static struct drm_i915_fence_reg *
2444i915_find_fence_reg(struct drm_device *dev,
2445 struct intel_ring_buffer *pipelined)
ae3db24a 2446{
ae3db24a 2447 struct drm_i915_private *dev_priv = dev->dev_private;
d9e86c0e
CW		 2448 struct drm_i915_fence_reg *reg, *first, *avail;
2449 int i;
ae3db24a
DV		 2450
2451 /* First try to find a free reg */
d9e86c0e 2452 avail = NULL;
ae3db24a
DV		 2453 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2454 reg = &dev_priv->fence_regs[i];
2455 if (!reg->obj)
d9e86c0e 2456 return reg;
ae3db24a 2457
05394f39 2458 if (!reg->obj->pin_count)
d9e86c0e 2459 avail = reg;
ae3db24a
DV		 2460 }
2461
d9e86c0e
CW		 2462 if (avail == NULL)
2463 return NULL;
ae3db24a
DV		 2464
2465 /* None available, try to steal one or wait for a user to finish */
d9e86c0e
CW		 2466 avail = first = NULL;
2467 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2468 if (reg->obj->pin_count)
ae3db24a
DV		 2469 continue;
2470
d9e86c0e
CW		 2471 if (first == NULL)
2472 first = reg;
2473
2474 if (!pipelined ||
2475 !reg->obj->last_fenced_ring ||
2476 reg->obj->last_fenced_ring == pipelined) {
2477 avail = reg;
2478 break;
2479 }
ae3db24a
DV		 2480 }
2481
d9e86c0e
CW		 2482 if (avail == NULL)
2483 avail = first;
ae3db24a 2484
a00b10c3 2485 return avail;
ae3db24a
DV		 2486}
2487
de151cf6 2488/**
d9e86c0e 2489 * i915_gem_object_get_fence - set up a fence reg for an object
de151cf6 2490 * @obj: object to map through a fence reg
d9e86c0e
CW		 2491 * @pipelined: ring on which to queue the change, or NULL for CPU access
2492 * @interruptible: must we wait uninterruptibly for the register to retire?
de151cf6
JB		 2493 *
2494 * When mapping objects through the GTT, userspace wants to be able to write
2495 * to them without having to worry about swizzling if the object is tiled.
2496 *
2497 * This function walks the fence regs looking for a free one for @obj,
2498 * stealing one if it can't find any.
2499 *
2500 * It then sets up the reg based on the object's properties: address, pitch
2501 * and tiling format.
2502 */
8c4b8c3f 2503int
d9e86c0e
CW		 2504i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2505 struct intel_ring_buffer *pipelined,
2506 bool interruptible)
de151cf6 2507{
05394f39 2508 struct drm_device *dev = obj->base.dev;
79e53945 2509 struct drm_i915_private *dev_priv = dev->dev_private;
d9e86c0e 2510 struct drm_i915_fence_reg *reg;
ae3db24a 2511 int ret;
de151cf6 2512
6bda10d1
CW		 2513 /* XXX disable pipelining. There are bugs. Shocking. */
2514 pipelined = NULL;
2515
d9e86c0e 2516 /* Just update our place in the LRU if our fence is getting reused. */
05394f39
CW		 2517 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2518 reg = &dev_priv->fence_regs[obj->fence_reg];
007cc8ac 2519 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
d9e86c0e
CW		 2520
2521 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2522 pipelined = NULL;
2523
2524 if (!pipelined) {
2525 if (reg->setup_seqno) {
2526 if (!ring_passed_seqno(obj->last_fenced_ring,
2527 reg->setup_seqno)) {
2528 ret = i915_do_wait_request(obj->base.dev,
2529 reg->setup_seqno,
2530 interruptible,
2531 obj->last_fenced_ring);
2532 if (ret)
2533 return ret;
2534 }
2535
2536 reg->setup_seqno = 0;
2537 }
2538 } else if (obj->last_fenced_ring &&
2539 obj->last_fenced_ring != pipelined) {
2540 ret = i915_gem_object_flush_fence(obj,
2541 pipelined,
2542 interruptible);
2543 if (ret)
2544 return ret;
2545 } else if (obj->tiling_changed) {
2546 if (obj->fenced_gpu_access) {
88241785
CW		 2547 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2548 ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
2549 0, obj->base.write_domain);
2550 if (ret)
2551 return ret;
2552 }
d9e86c0e
CW		 2553
2554 obj->fenced_gpu_access = false;
2555 }
2556 }
2557
2558 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2559 pipelined = NULL;
2560 BUG_ON(!pipelined && reg->setup_seqno);
2561
2562 if (obj->tiling_changed) {
2563 if (pipelined) {
2564 reg->setup_seqno =
2565 i915_gem_next_request_seqno(dev, pipelined);
2566 obj->last_fenced_seqno = reg->setup_seqno;
2567 obj->last_fenced_ring = pipelined;
2568 }
2569 goto update;
2570 }
2571
a09ba7fa
EA		 2572 return 0;
2573 }
2574
d9e86c0e
CW		 2575 reg = i915_find_fence_reg(dev, pipelined);
2576 if (reg == NULL)
2577 return -ENOSPC;
de151cf6 2578
d9e86c0e
CW		 2579 ret = i915_gem_object_flush_fence(obj, pipelined, interruptible);
2580 if (ret)
ae3db24a 2581 return ret;
de151cf6 2582
d9e86c0e
CW		 2583 if (reg->obj) {
2584 struct drm_i915_gem_object *old = reg->obj;
2585
2586 drm_gem_object_reference(&old->base);
2587
2588 if (old->tiling_mode)
2589 i915_gem_release_mmap(old);
2590
d9e86c0e 2591 ret = i915_gem_object_flush_fence(old,
6bda10d1 2592 pipelined,
d9e86c0e
CW		 2593 interruptible);
2594 if (ret) {
2595 drm_gem_object_unreference(&old->base);
2596 return ret;
2597 }
2598
2599 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2600 pipelined = NULL;
2601
2602 old->fence_reg = I915_FENCE_REG_NONE;
2603 old->last_fenced_ring = pipelined;
2604 old->last_fenced_seqno =
2605 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2606
2607 drm_gem_object_unreference(&old->base);
2608 } else if (obj->last_fenced_seqno == 0)
2609 pipelined = NULL;
a09ba7fa 2610
de151cf6 2611 reg->obj = obj;
d9e86c0e
CW		 2612 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2613 obj->fence_reg = reg - dev_priv->fence_regs;
2614 obj->last_fenced_ring = pipelined;
de151cf6 2615
d9e86c0e
CW		 2616 reg->setup_seqno =
2617 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2618 obj->last_fenced_seqno = reg->setup_seqno;
2619
2620update:
2621 obj->tiling_changed = false;
e259befd
CW		 2622 switch (INTEL_INFO(dev)->gen) {
2623 case 6:
c6642782 2624 ret = sandybridge_write_fence_reg(obj, pipelined);
e259befd
CW		 2625 break;
2626 case 5:
2627 case 4:
c6642782 2628 ret = i965_write_fence_reg(obj, pipelined);
e259befd
CW		 2629 break;
2630 case 3:
c6642782 2631 ret = i915_write_fence_reg(obj, pipelined);
e259befd
CW		 2632 break;
2633 case 2:
c6642782 2634 ret = i830_write_fence_reg(obj, pipelined);
e259befd
CW		 2635 break;
2636 }
d9ddcb96 2637
c6642782 2638 return ret;
de151cf6
JB		 2639}
2640
2641/**
2642 * i915_gem_clear_fence_reg - clear out fence register info
2643 * @obj: object to clear
2644 *
2645 * Zeroes out the fence register itself and clears out the associated
05394f39 2646 * data structures in dev_priv and obj.
de151cf6
JB		 2647 */
2648static void
d9e86c0e
CW		 2649i915_gem_clear_fence_reg(struct drm_device *dev,
2650 struct drm_i915_fence_reg *reg)
de151cf6 2651{
79e53945 2652 drm_i915_private_t *dev_priv = dev->dev_private;
d9e86c0e 2653 uint32_t fence_reg = reg - dev_priv->fence_regs;
de151cf6 2654
e259befd
CW		 2655 switch (INTEL_INFO(dev)->gen) {
2656 case 6:
d9e86c0e 2657 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
e259befd
CW		 2658 break;
2659 case 5:
2660 case 4:
d9e86c0e 2661 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
e259befd
CW		 2662 break;
2663 case 3:
d9e86c0e
CW		 2664 if (fence_reg >= 8)
2665 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
dc529a4f 2666 else
e259befd 2667 case 2:
d9e86c0e 2668 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
dc529a4f
EA		 2669
2670 I915_WRITE(fence_reg, 0);
e259befd 2671 break;
dc529a4f 2672 }
de151cf6 2673
007cc8ac 2674 list_del_init(&reg->lru_list);
d9e86c0e
CW		 2675 reg->obj = NULL;
2676 reg->setup_seqno = 0;
52dc7d32
CW		 2677}
2678
673a394b
EA		 2679/**
2680 * Finds free space in the GTT aperture and binds the object there.
2681 */
2682static int
05394f39 2683i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
920afa77 2684 unsigned alignment,
75e9e915 2685 bool map_and_fenceable)
673a394b 2686{
05394f39 2687 struct drm_device *dev = obj->base.dev;
673a394b 2688 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 2689 struct drm_mm_node *free_space;
a00b10c3 2690 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
5e783301 2691 u32 size, fence_size, fence_alignment, unfenced_alignment;
75e9e915 2692 bool mappable, fenceable;
07f73f69 2693 int ret;
673a394b 2694
05394f39 2695 if (obj->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2696 DRM_ERROR("Attempting to bind a purgeable object\n");
2697 return -EINVAL;
2698 }
2699
05394f39
CW		 2700 fence_size = i915_gem_get_gtt_size(obj);
2701 fence_alignment = i915_gem_get_gtt_alignment(obj);
2702 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
a00b10c3 2703
673a394b 2704 if (alignment == 0)
5e783301
DV		 2705 alignment = map_and_fenceable ? fence_alignment :
2706 unfenced_alignment;
75e9e915 2707 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
673a394b
EA		 2708 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2709 return -EINVAL;
2710 }
2711
05394f39 2712 size = map_and_fenceable ? fence_size : obj->base.size;
a00b10c3 2713
654fc607
CW		 2714 /* If the object is bigger than the entire aperture, reject it early
2715 * before evicting everything in a vain attempt to find space.
2716 */
05394f39 2717 if (obj->base.size >
75e9e915 2718 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
654fc607
CW		 2719 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2720 return -E2BIG;
2721 }
2722
673a394b 2723 search_free:
75e9e915 2724 if (map_and_fenceable)
920afa77
DV		 2725 free_space =
2726 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
a00b10c3 2727 size, alignment, 0,
920afa77
DV		 2728 dev_priv->mm.gtt_mappable_end,
2729 0);
2730 else
2731 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
a00b10c3 2732 size, alignment, 0);
920afa77
DV		 2733
2734 if (free_space != NULL) {
75e9e915 2735 if (map_and_fenceable)
05394f39 2736 obj->gtt_space =
920afa77 2737 drm_mm_get_block_range_generic(free_space,
a00b10c3 2738 size, alignment, 0,
920afa77
DV		 2739 dev_priv->mm.gtt_mappable_end,
2740 0);
2741 else
05394f39 2742 obj->gtt_space =
a00b10c3 2743 drm_mm_get_block(free_space, size, alignment);
920afa77 2744 }
05394f39 2745 if (obj->gtt_space == NULL) {
673a394b
EA		 2746 /* If the gtt is empty and we're still having trouble
2747 * fitting our object in, we're out of memory.
2748 */
75e9e915
DV		 2749 ret = i915_gem_evict_something(dev, size, alignment,
2750 map_and_fenceable);
9731129c 2751 if (ret)
673a394b 2752 return ret;
9731129c 2753
673a394b
EA		 2754 goto search_free;
2755 }
2756
e5281ccd 2757 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
673a394b 2758 if (ret) {
05394f39
CW		 2759 drm_mm_put_block(obj->gtt_space);
2760 obj->gtt_space = NULL;
07f73f69
CW		 2761
2762 if (ret == -ENOMEM) {
2763 /* first try to clear up some space from the GTT */
a00b10c3 2764 ret = i915_gem_evict_something(dev, size,
75e9e915
DV		 2765 alignment,
2766 map_and_fenceable);
07f73f69 2767 if (ret) {
07f73f69 2768 /* now try to shrink everyone else */
4bdadb97
CW		 2769 if (gfpmask) {
2770 gfpmask = 0;
2771 goto search_free;
07f73f69
CW		 2772 }
2773
2774 return ret;
2775 }
2776
2777 goto search_free;
2778 }
2779
673a394b
EA		 2780 return ret;
2781 }
2782
7c2e6fdf
DV		 2783 ret = i915_gem_gtt_bind_object(obj);
2784 if (ret) {
e5281ccd 2785 i915_gem_object_put_pages_gtt(obj);
05394f39
CW		 2786 drm_mm_put_block(obj->gtt_space);
2787 obj->gtt_space = NULL;
07f73f69 2788
a00b10c3 2789 ret = i915_gem_evict_something(dev, size,
75e9e915 2790 alignment, map_and_fenceable);
9731129c 2791 if (ret)
07f73f69 2792 return ret;
07f73f69
CW		 2793
2794 goto search_free;
673a394b 2795 }
673a394b 2796
6299f992 2797 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
05394f39 2798 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
bf1a1092 2799
673a394b
EA		 2800 /* Assert that the object is not currently in any GPU domain. As it
2801 * wasn't in the GTT, there shouldn't be any way it could have been in
2802 * a GPU cache
2803 */
05394f39
CW		 2804 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2805 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2806
6299f992 2807 obj->gtt_offset = obj->gtt_space->start;
1c5d22f7 2808
75e9e915 2809 fenceable =
05394f39
CW		 2810 obj->gtt_space->size == fence_size &&
2811 (obj->gtt_space->start & (fence_alignment -1)) == 0;
a00b10c3 2812
75e9e915 2813 mappable =
05394f39 2814 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
a00b10c3 2815
05394f39 2816 obj->map_and_fenceable = mappable && fenceable;
75e9e915 2817
6299f992 2818 trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
673a394b
EA		 2819 return 0;
2820}
2821
2822void
05394f39 2823i915_gem_clflush_object(struct drm_i915_gem_object *obj)
673a394b 2824{
673a394b
EA		 2825 /* If we don't have a page list set up, then we're not pinned
2826 * to GPU, and we can ignore the cache flush because it'll happen
2827 * again at bind time.
2828 */
05394f39 2829 if (obj->pages == NULL)
673a394b
EA		 2830 return;
2831
1c5d22f7 2832 trace_i915_gem_object_clflush(obj);
cfa16a0d 2833
05394f39 2834 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
673a394b
EA		 2835}
2836
e47c68e9 2837/** Flushes any GPU write domain for the object if it's dirty. */
88241785 2838static int
3619df03 2839i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2840{
05394f39 2841 struct drm_device *dev = obj->base.dev;
e47c68e9 2842
05394f39 2843 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
88241785 2844 return 0;
e47c68e9
EA		 2845
2846 /* Queue the GPU write cache flushing we need. */
88241785 2847 return i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
e47c68e9
EA		 2848}
2849
2850/** Flushes the GTT write domain for the object if it's dirty. */
2851static void
05394f39 2852i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2853{
1c5d22f7
CW		 2854 uint32_t old_write_domain;
2855
05394f39 2856 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
e47c68e9
EA		 2857 return;
2858
63256ec5 2859 /* No actual flushing is required for the GTT write domain. Writes
e47c68e9
EA		 2860 * to it immediately go to main memory as far as we know, so there's
2861 * no chipset flush. It also doesn't land in render cache.
63256ec5
CW		 2862 *
2863 * However, we do have to enforce the order so that all writes through
2864 * the GTT land before any writes to the device, such as updates to
2865 * the GATT itself.
e47c68e9 2866 */
63256ec5
CW		 2867 wmb();
2868
4a684a41
CW		 2869 i915_gem_release_mmap(obj);
2870
05394f39
CW		 2871 old_write_domain = obj->base.write_domain;
2872 obj->base.write_domain = 0;
1c5d22f7
CW		 2873
2874 trace_i915_gem_object_change_domain(obj,
05394f39 2875 obj->base.read_domains,
1c5d22f7 2876 old_write_domain);
e47c68e9
EA		 2877}
2878
2879/** Flushes the CPU write domain for the object if it's dirty. */
2880static void
05394f39 2881i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
e47c68e9 2882{
1c5d22f7 2883 uint32_t old_write_domain;
e47c68e9 2884
05394f39 2885 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
e47c68e9
EA		 2886 return;
2887
2888 i915_gem_clflush_object(obj);
40ce6575 2889 intel_gtt_chipset_flush();
05394f39
CW		 2890 old_write_domain = obj->base.write_domain;
2891 obj->base.write_domain = 0;
1c5d22f7
CW		 2892
2893 trace_i915_gem_object_change_domain(obj,
05394f39 2894 obj->base.read_domains,
1c5d22f7 2895 old_write_domain);
e47c68e9
EA		 2896}
2897
2ef7eeaa
EA		 2898/**
2899 * Moves a single object to the GTT read, and possibly write domain.
2900 *
2901 * This function returns when the move is complete, including waiting on
2902 * flushes to occur.
2903 */
79e53945 2904int
2021746e 2905i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2ef7eeaa 2906{
1c5d22f7 2907 uint32_t old_write_domain, old_read_domains;
e47c68e9 2908 int ret;
2ef7eeaa 2909
02354392 2910 /* Not valid to be called on unbound objects. */
05394f39 2911 if (obj->gtt_space == NULL)
02354392
EA		 2912 return -EINVAL;
2913
88241785
CW		 2914 ret = i915_gem_object_flush_gpu_write_domain(obj);
2915 if (ret)
2916 return ret;
2917
87ca9c8a
CW		 2918 if (obj->pending_gpu_write || write) {
2919 ret = i915_gem_object_wait_rendering(obj, true);
2920 if (ret)
2921 return ret;
2922 }
2dafb1e0 2923
7213342d 2924 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2925
05394f39
CW		 2926 old_write_domain = obj->base.write_domain;
2927 old_read_domains = obj->base.read_domains;
1c5d22f7 2928
e47c68e9
EA		 2929 /* It should now be out of any other write domains, and we can update
2930 * the domain values for our changes.
2931 */
05394f39
CW		 2932 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2933 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
e47c68e9 2934 if (write) {
05394f39
CW		 2935 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2936 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2937 obj->dirty = 1;
2ef7eeaa
EA		 2938 }
2939
1c5d22f7
CW		 2940 trace_i915_gem_object_change_domain(obj,
2941 old_read_domains,
2942 old_write_domain);
2943
e47c68e9
EA		 2944 return 0;
2945}
2946
b9241ea3
ZW		 2947/*
2948 * Prepare buffer for display plane. Use uninterruptible for possible flush
2949 * wait, as in modesetting process we're not supposed to be interrupted.
2950 */
2951int
05394f39 2952i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
919926ae 2953 struct intel_ring_buffer *pipelined)
b9241ea3 2954{
ba3d8d74 2955 uint32_t old_read_domains;
b9241ea3
ZW		 2956 int ret;
2957
2958 /* Not valid to be called on unbound objects. */
05394f39 2959 if (obj->gtt_space == NULL)
b9241ea3
ZW		 2960 return -EINVAL;
2961
88241785
CW		 2962 ret = i915_gem_object_flush_gpu_write_domain(obj);
2963 if (ret)
2964 return ret;
2965
b9241ea3 2966
ced270fa 2967 /* Currently, we are always called from an non-interruptible context. */
0be73284 2968 if (pipelined != obj->ring) {
ced270fa
CW		 2969 ret = i915_gem_object_wait_rendering(obj, false);
2970 if (ret)
b9241ea3
ZW		 2971 return ret;
2972 }
2973
b118c1e3
CW		 2974 i915_gem_object_flush_cpu_write_domain(obj);
2975
05394f39
CW		 2976 old_read_domains = obj->base.read_domains;
2977 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2978
2979 trace_i915_gem_object_change_domain(obj,
2980 old_read_domains,
05394f39 2981 obj->base.write_domain);
b9241ea3
ZW		 2982
2983 return 0;
2984}
2985
85345517
CW		 2986int
2987i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2988 bool interruptible)
2989{
88241785
CW		 2990 int ret;
2991
85345517
CW		 2992 if (!obj->active)
2993 return 0;
2994
88241785
CW		 2995 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2996 ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
2997 0, obj->base.write_domain);
2998 if (ret)
2999 return ret;
3000 }
85345517 3001
05394f39 3002 return i915_gem_object_wait_rendering(obj, interruptible);
85345517
CW		 3003}
3004
e47c68e9
EA		 3005/**
3006 * Moves a single object to the CPU read, and possibly write domain.
3007 *
3008 * This function returns when the move is complete, including waiting on
3009 * flushes to occur.
3010 */
3011static int
919926ae 3012i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
e47c68e9 3013{
1c5d22f7 3014 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 3015 int ret;
3016
88241785
CW		 3017 ret = i915_gem_object_flush_gpu_write_domain(obj);
3018 if (ret)
3019 return ret;
3020
de18a29e
DV		 3021 ret = i915_gem_object_wait_rendering(obj, true);
3022 if (ret)
e47c68e9 3023 return ret;
2ef7eeaa 3024
e47c68e9 3025 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 3026
e47c68e9
EA		 3027 /* If we have a partially-valid cache of the object in the CPU,
3028 * finish invalidating it and free the per-page flags.
2ef7eeaa 3029 */
e47c68e9 3030 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 3031
05394f39
CW		 3032 old_write_domain = obj->base.write_domain;
3033 old_read_domains = obj->base.read_domains;
1c5d22f7 3034
e47c68e9 3035 /* Flush the CPU cache if it's still invalid. */
05394f39 3036 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 3037 i915_gem_clflush_object(obj);
2ef7eeaa 3038
05394f39 3039 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 3040 }
3041
3042 /* It should now be out of any other write domains, and we can update
3043 * the domain values for our changes.
3044 */
05394f39 3045 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9
EA		 3046
3047 /* If we're writing through the CPU, then the GPU read domains will
3048 * need to be invalidated at next use.
3049 */
3050 if (write) {
05394f39
CW		 3051 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3052 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
e47c68e9 3053 }
2ef7eeaa 3054
1c5d22f7
CW		 3055 trace_i915_gem_object_change_domain(obj,
3056 old_read_domains,
3057 old_write_domain);
3058
2ef7eeaa
EA		 3059 return 0;
3060}
3061
673a394b 3062/**
e47c68e9 3063 * Moves the object from a partially CPU read to a full one.
673a394b 3064 *
e47c68e9
EA		 3065 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3066 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3067 */
e47c68e9 3068static void
05394f39 3069i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
673a394b 3070{
05394f39 3071 if (!obj->page_cpu_valid)
e47c68e9
EA		 3072 return;
3073
3074 /* If we're partially in the CPU read domain, finish moving it in.
3075 */
05394f39 3076 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3077 int i;
3078
05394f39
CW		 3079 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3080 if (obj->page_cpu_valid[i])
e47c68e9 3081 continue;
05394f39 3082 drm_clflush_pages(obj->pages + i, 1);
e47c68e9 3083 }
e47c68e9
EA		 3084 }
3085
3086 /* Free the page_cpu_valid mappings which are now stale, whether
3087 * or not we've got I915_GEM_DOMAIN_CPU.
3088 */
05394f39
CW		 3089 kfree(obj->page_cpu_valid);
3090 obj->page_cpu_valid = NULL;
e47c68e9
EA		 3091}
3092
3093/**
3094 * Set the CPU read domain on a range of the object.
3095 *
3096 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3097 * not entirely valid. The page_cpu_valid member of the object flags which
3098 * pages have been flushed, and will be respected by
3099 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3100 * of the whole object.
3101 *
3102 * This function returns when the move is complete, including waiting on
3103 * flushes to occur.
3104 */
3105static int
05394f39 3106i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
e47c68e9
EA		 3107 uint64_t offset, uint64_t size)
3108{
1c5d22f7 3109 uint32_t old_read_domains;
e47c68e9 3110 int i, ret;
673a394b 3111
05394f39 3112 if (offset == 0 && size == obj->base.size)
e47c68e9 3113 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3114
88241785
CW		 3115 ret = i915_gem_object_flush_gpu_write_domain(obj);
3116 if (ret)
3117 return ret;
3118
de18a29e
DV		 3119 ret = i915_gem_object_wait_rendering(obj, true);
3120 if (ret)
6a47baa6 3121 return ret;
de18a29e 3122
e47c68e9
EA		 3123 i915_gem_object_flush_gtt_write_domain(obj);
3124
3125 /* If we're already fully in the CPU read domain, we're done. */
05394f39
CW		 3126 if (obj->page_cpu_valid == NULL &&
3127 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
e47c68e9 3128 return 0;
673a394b 3129
e47c68e9
EA		 3130 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3131 * newly adding I915_GEM_DOMAIN_CPU
3132 */
05394f39
CW		 3133 if (obj->page_cpu_valid == NULL) {
3134 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3135 GFP_KERNEL);
3136 if (obj->page_cpu_valid == NULL)
e47c68e9 3137 return -ENOMEM;
05394f39
CW		 3138 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3139 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
673a394b
EA		 3140
3141 /* Flush the cache on any pages that are still invalid from the CPU's
3142 * perspective.
3143 */
e47c68e9
EA		 3144 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3145 i++) {
05394f39 3146 if (obj->page_cpu_valid[i])
673a394b
EA		 3147 continue;
3148
05394f39 3149 drm_clflush_pages(obj->pages + i, 1);
673a394b 3150
05394f39 3151 obj->page_cpu_valid[i] = 1;
673a394b
EA		 3152 }
3153
e47c68e9
EA		 3154 /* It should now be out of any other write domains, and we can update
3155 * the domain values for our changes.
3156 */
05394f39 3157 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
e47c68e9 3158
05394f39
CW		 3159 old_read_domains = obj->base.read_domains;
3160 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
e47c68e9 3161
1c5d22f7
CW		 3162 trace_i915_gem_object_change_domain(obj,
3163 old_read_domains,
05394f39 3164 obj->base.write_domain);
1c5d22f7 3165
673a394b
EA		 3166 return 0;
3167}
3168
673a394b
EA		 3169/* Throttle our rendering by waiting until the ring has completed our requests
3170 * emitted over 20 msec ago.
3171 *
b962442e
EA		 3172 * Note that if we were to use the current jiffies each time around the loop,
3173 * we wouldn't escape the function with any frames outstanding if the time to
3174 * render a frame was over 20ms.
3175 *
673a394b
EA		 3176 * This should get us reasonable parallelism between CPU and GPU but also
3177 * relatively low latency when blocking on a particular request to finish.
3178 */
40a5f0de 3179static int
f787a5f5 3180i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
40a5f0de 3181{
f787a5f5
CW		 3182 struct drm_i915_private *dev_priv = dev->dev_private;
3183 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3184 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3185 struct drm_i915_gem_request *request;
3186 struct intel_ring_buffer *ring = NULL;
3187 u32 seqno = 0;
3188 int ret;
93533c29 3189
1c25595f 3190 spin_lock(&file_priv->mm.lock);
f787a5f5 3191 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3192 if (time_after_eq(request->emitted_jiffies, recent_enough))
3193 break;
40a5f0de 3194
f787a5f5
CW		 3195 ring = request->ring;
3196 seqno = request->seqno;
b962442e 3197 }
1c25595f 3198 spin_unlock(&file_priv->mm.lock);
40a5f0de 3199
f787a5f5
CW		 3200 if (seqno == 0)
3201 return 0;
2bc43b5c 3202
f787a5f5 3203 ret = 0;
78501eac 3204 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
f787a5f5
CW		 3205 /* And wait for the seqno passing without holding any locks and
3206 * causing extra latency for others. This is safe as the irq
3207 * generation is designed to be run atomically and so is
3208 * lockless.
3209 */
b13c2b96
CW		 3210 if (ring->irq_get(ring)) {
3211 ret = wait_event_interruptible(ring->irq_queue,
3212 i915_seqno_passed(ring->get_seqno(ring), seqno)
3213 || atomic_read(&dev_priv->mm.wedged));
3214 ring->irq_put(ring);
40a5f0de 3215
b13c2b96
CW		 3216 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3217 ret = -EIO;
3218 }
40a5f0de
EA		 3219 }
3220
f787a5f5
CW		 3221 if (ret == 0)
3222 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
40a5f0de
EA		 3223
3224 return ret;
3225}
3226
673a394b 3227int
05394f39
CW		 3228i915_gem_object_pin(struct drm_i915_gem_object *obj,
3229 uint32_t alignment,
75e9e915 3230 bool map_and_fenceable)
673a394b 3231{
05394f39 3232 struct drm_device *dev = obj->base.dev;
f13d3f73 3233 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 3234 int ret;
3235
05394f39 3236 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
23bc5982 3237 WARN_ON(i915_verify_lists(dev));
ac0c6b5a 3238
05394f39
CW		 3239 if (obj->gtt_space != NULL) {
3240 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3241 (map_and_fenceable && !obj->map_and_fenceable)) {
3242 WARN(obj->pin_count,
ae7d49d8 3243 "bo is already pinned with incorrect alignment:"
75e9e915
DV		 3244 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3245 " obj->map_and_fenceable=%d\n",
05394f39 3246 obj->gtt_offset, alignment,
75e9e915 3247 map_and_fenceable,
05394f39 3248 obj->map_and_fenceable);
ac0c6b5a
CW		 3249 ret = i915_gem_object_unbind(obj);
3250 if (ret)
3251 return ret;
3252 }
3253 }
3254
05394f39 3255 if (obj->gtt_space == NULL) {
a00b10c3 3256 ret = i915_gem_object_bind_to_gtt(obj, alignment,
75e9e915 3257 map_and_fenceable);
9731129c 3258 if (ret)
673a394b 3259 return ret;
22c344e9 3260 }
76446cac 3261
05394f39 3262 if (obj->pin_count++ == 0) {
05394f39
CW		 3263 if (!obj->active)
3264 list_move_tail(&obj->mm_list,
f13d3f73 3265 &dev_priv->mm.pinned_list);
673a394b 3266 }
6299f992 3267 obj->pin_mappable |= map_and_fenceable;
673a394b 3268
23bc5982 3269 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 3270 return 0;
3271}
3272
3273void
05394f39 3274i915_gem_object_unpin(struct drm_i915_gem_object *obj)
673a394b 3275{
05394f39 3276 struct drm_device *dev = obj->base.dev;
673a394b 3277 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 3278
23bc5982 3279 WARN_ON(i915_verify_lists(dev));
05394f39
CW		 3280 BUG_ON(obj->pin_count == 0);
3281 BUG_ON(obj->gtt_space == NULL);
673a394b 3282
05394f39
CW		 3283 if (--obj->pin_count == 0) {
3284 if (!obj->active)
3285 list_move_tail(&obj->mm_list,
673a394b 3286 &dev_priv->mm.inactive_list);
6299f992 3287 obj->pin_mappable = false;
673a394b 3288 }
23bc5982 3289 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 3290}
3291
3292int
3293i915_gem_pin_ioctl(struct drm_device *dev, void *data,
05394f39 3294 struct drm_file *file)
673a394b
EA		 3295{
3296 struct drm_i915_gem_pin *args = data;
05394f39 3297 struct drm_i915_gem_object *obj;
673a394b
EA		 3298 int ret;
3299
1d7cfea1
CW		 3300 ret = i915_mutex_lock_interruptible(dev);
3301 if (ret)
3302 return ret;
673a394b 3303
05394f39 3304 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 3305 if (obj == NULL) {
1d7cfea1
CW		 3306 ret = -ENOENT;
3307 goto unlock;
673a394b 3308 }
673a394b 3309
05394f39 3310 if (obj->madv != I915_MADV_WILLNEED) {
bb6baf76 3311 DRM_ERROR("Attempting to pin a purgeable buffer\n");
1d7cfea1
CW		 3312 ret = -EINVAL;
3313 goto out;
3ef94daa
CW		 3314 }
3315
05394f39 3316 if (obj->pin_filp != NULL && obj->pin_filp != file) {
79e53945
JB		 3317 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3318 args->handle);
1d7cfea1
CW		 3319 ret = -EINVAL;
3320 goto out;
79e53945
JB		 3321 }
3322
05394f39
CW		 3323 obj->user_pin_count++;
3324 obj->pin_filp = file;
3325 if (obj->user_pin_count == 1) {
75e9e915 3326 ret = i915_gem_object_pin(obj, args->alignment, true);
1d7cfea1
CW		 3327 if (ret)
3328 goto out;
673a394b
EA		 3329 }
3330
3331 /* XXX - flush the CPU caches for pinned objects
3332 * as the X server doesn't manage domains yet
3333 */
e47c68e9 3334 i915_gem_object_flush_cpu_write_domain(obj);
05394f39 3335 args->offset = obj->gtt_offset;
1d7cfea1 3336out:
05394f39 3337 drm_gem_object_unreference(&obj->base);
1d7cfea1 3338unlock:
673a394b 3339 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3340 return ret;
673a394b
EA		 3341}
3342
3343int
3344i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
05394f39 3345 struct drm_file *file)
673a394b
EA		 3346{
3347 struct drm_i915_gem_pin *args = data;
05394f39 3348 struct drm_i915_gem_object *obj;
76c1dec1 3349 int ret;
673a394b 3350
1d7cfea1
CW		 3351 ret = i915_mutex_lock_interruptible(dev);
3352 if (ret)
3353 return ret;
673a394b 3354
05394f39 3355 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 3356 if (obj == NULL) {
1d7cfea1
CW		 3357 ret = -ENOENT;
3358 goto unlock;
673a394b 3359 }
76c1dec1 3360
05394f39 3361 if (obj->pin_filp != file) {
79e53945
JB		 3362 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3363 args->handle);
1d7cfea1
CW		 3364 ret = -EINVAL;
3365 goto out;
79e53945 3366 }
05394f39
CW		 3367 obj->user_pin_count--;
3368 if (obj->user_pin_count == 0) {
3369 obj->pin_filp = NULL;
79e53945
JB		 3370 i915_gem_object_unpin(obj);
3371 }
673a394b 3372
1d7cfea1 3373out:
05394f39 3374 drm_gem_object_unreference(&obj->base);
1d7cfea1 3375unlock:
673a394b 3376 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3377 return ret;
673a394b
EA		 3378}
3379
3380int
3381i915_gem_busy_ioctl(struct drm_device *dev, void *data,
05394f39 3382 struct drm_file *file)
673a394b
EA		 3383{
3384 struct drm_i915_gem_busy *args = data;
05394f39 3385 struct drm_i915_gem_object *obj;
30dbf0c0
CW		 3386 int ret;
3387
76c1dec1 3388 ret = i915_mutex_lock_interruptible(dev);
1d7cfea1 3389 if (ret)
76c1dec1 3390 return ret;
673a394b 3391
05394f39 3392 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
673a394b 3393 if (obj == NULL) {
1d7cfea1
CW		 3394 ret = -ENOENT;
3395 goto unlock;
673a394b 3396 }
d1b851fc 3397
0be555b6
CW		 3398 /* Count all active objects as busy, even if they are currently not used
3399 * by the gpu. Users of this interface expect objects to eventually
3400 * become non-busy without any further actions, therefore emit any
3401 * necessary flushes here.
c4de0a5d 3402 */
05394f39 3403 args->busy = obj->active;
0be555b6
CW		 3404 if (args->busy) {
3405 /* Unconditionally flush objects, even when the gpu still uses this
3406 * object. Userspace calling this function indicates that it wants to
3407 * use this buffer rather sooner than later, so issuing the required
3408 * flush earlier is beneficial.
3409 */
1a1c6976 3410 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
88241785
CW		 3411 ret = i915_gem_flush_ring(dev, obj->ring,
3412 0, obj->base.write_domain);
1a1c6976
CW		 3413 } else if (obj->ring->outstanding_lazy_request ==
3414 obj->last_rendering_seqno) {
3415 struct drm_i915_gem_request *request;
3416
7a194876
CW		 3417 /* This ring is not being cleared by active usage,
3418 * so emit a request to do so.
3419 */
1a1c6976
CW		 3420 request = kzalloc(sizeof(*request), GFP_KERNEL);
3421 if (request)
3422 ret = i915_add_request(dev,
3423 NULL, request,
3424 obj->ring);
3425 else
7a194876
CW		 3426 ret = -ENOMEM;
3427 }
0be555b6
CW		 3428
3429 /* Update the active list for the hardware's current position.
3430 * Otherwise this only updates on a delayed timer or when irqs
3431 * are actually unmasked, and our working set ends up being
3432 * larger than required.
3433 */
05394f39 3434 i915_gem_retire_requests_ring(dev, obj->ring);
0be555b6 3435
05394f39 3436 args->busy = obj->active;
0be555b6 3437 }
673a394b 3438
05394f39 3439 drm_gem_object_unreference(&obj->base);
1d7cfea1 3440unlock:
673a394b 3441 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3442 return ret;
673a394b
EA		 3443}
3444
3445int
3446i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3447 struct drm_file *file_priv)
3448{
3449 return i915_gem_ring_throttle(dev, file_priv);
3450}
3451
3ef94daa
CW		 3452int
3453i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3454 struct drm_file *file_priv)
3455{
3456 struct drm_i915_gem_madvise *args = data;
05394f39 3457 struct drm_i915_gem_object *obj;
76c1dec1 3458 int ret;
3ef94daa
CW		 3459
3460 switch (args->madv) {
3461 case I915_MADV_DONTNEED:
3462 case I915_MADV_WILLNEED:
3463 break;
3464 default:
3465 return -EINVAL;
3466 }
3467
1d7cfea1
CW		 3468 ret = i915_mutex_lock_interruptible(dev);
3469 if (ret)
3470 return ret;
3471
05394f39 3472 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3ef94daa 3473 if (obj == NULL) {
1d7cfea1
CW		 3474 ret = -ENOENT;
3475 goto unlock;
3ef94daa 3476 }
3ef94daa 3477
05394f39 3478 if (obj->pin_count) {
1d7cfea1
CW		 3479 ret = -EINVAL;
3480 goto out;
3ef94daa
CW		 3481 }
3482
05394f39
CW		 3483 if (obj->madv != __I915_MADV_PURGED)
3484 obj->madv = args->madv;
3ef94daa 3485
2d7ef395 3486 /* if the object is no longer bound, discard its backing storage */
05394f39
CW		 3487 if (i915_gem_object_is_purgeable(obj) &&
3488 obj->gtt_space == NULL)
2d7ef395
CW		 3489 i915_gem_object_truncate(obj);
3490
05394f39 3491 args->retained = obj->madv != __I915_MADV_PURGED;
bb6baf76 3492
1d7cfea1 3493out:
05394f39 3494 drm_gem_object_unreference(&obj->base);
1d7cfea1 3495unlock:
3ef94daa 3496 mutex_unlock(&dev->struct_mutex);
1d7cfea1 3497 return ret;
3ef94daa
CW		 3498}
3499
05394f39
CW		 3500struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3501 size_t size)
ac52bc56 3502{
73aa808f 3503 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 3504 struct drm_i915_gem_object *obj;
ac52bc56 3505
c397b908
DV		 3506 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3507 if (obj == NULL)
3508 return NULL;
673a394b 3509
c397b908
DV		 3510 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3511 kfree(obj);
3512 return NULL;
3513 }
673a394b 3514
73aa808f
CW		 3515 i915_gem_info_add_obj(dev_priv, size);
3516
c397b908
DV		 3517 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3518 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 3519
c397b908 3520 obj->agp_type = AGP_USER_MEMORY;
62b8b215 3521 obj->base.driver_private = NULL;
c397b908 3522 obj->fence_reg = I915_FENCE_REG_NONE;
69dc4987 3523 INIT_LIST_HEAD(&obj->mm_list);
93a37f20 3524 INIT_LIST_HEAD(&obj->gtt_list);
69dc4987 3525 INIT_LIST_HEAD(&obj->ring_list);
432e58ed 3526 INIT_LIST_HEAD(&obj->exec_list);
c397b908 3527 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 3528 obj->madv = I915_MADV_WILLNEED;
75e9e915
DV		 3529 /* Avoid an unnecessary call to unbind on the first bind. */
3530 obj->map_and_fenceable = true;
de151cf6 3531
05394f39 3532 return obj;
c397b908
DV		 3533}
3534
3535int i915_gem_init_object(struct drm_gem_object *obj)
3536{
3537 BUG();
de151cf6 3538
673a394b
EA		 3539 return 0;
3540}
3541
05394f39 3542static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
673a394b 3543{
05394f39 3544 struct drm_device *dev = obj->base.dev;
be72615b 3545 drm_i915_private_t *dev_priv = dev->dev_private;
be72615b 3546 int ret;
673a394b 3547
be72615b
CW		 3548 ret = i915_gem_object_unbind(obj);
3549 if (ret == -ERESTARTSYS) {
05394f39 3550 list_move(&obj->mm_list,
be72615b
CW		 3551 &dev_priv->mm.deferred_free_list);
3552 return;
3553 }
673a394b 3554
05394f39 3555 if (obj->base.map_list.map)
7e616158 3556 i915_gem_free_mmap_offset(obj);
de151cf6 3557
05394f39
CW		 3558 drm_gem_object_release(&obj->base);
3559 i915_gem_info_remove_obj(dev_priv, obj->base.size);
c397b908 3560
05394f39
CW		 3561 kfree(obj->page_cpu_valid);
3562 kfree(obj->bit_17);
3563 kfree(obj);
673a394b
EA		 3564}
3565
05394f39 3566void i915_gem_free_object(struct drm_gem_object *gem_obj)
be72615b 3567{
05394f39
CW		 3568 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3569 struct drm_device *dev = obj->base.dev;
be72615b
CW		 3570
3571 trace_i915_gem_object_destroy(obj);
3572
05394f39 3573 while (obj->pin_count > 0)
be72615b
CW		 3574 i915_gem_object_unpin(obj);
3575
05394f39 3576 if (obj->phys_obj)
be72615b
CW		 3577 i915_gem_detach_phys_object(dev, obj);
3578
3579 i915_gem_free_object_tail(obj);
3580}
3581
29105ccc
CW		 3582int
3583i915_gem_idle(struct drm_device *dev)
3584{
3585 drm_i915_private_t *dev_priv = dev->dev_private;
3586 int ret;
28dfe52a 3587
29105ccc 3588 mutex_lock(&dev->struct_mutex);
1c5d22f7 3589
87acb0a5 3590 if (dev_priv->mm.suspended) {
29105ccc
CW		 3591 mutex_unlock(&dev->struct_mutex);
3592 return 0;
28dfe52a
EA		 3593 }
3594
29105ccc 3595 ret = i915_gpu_idle(dev);
6dbe2772
KP		 3596 if (ret) {
3597 mutex_unlock(&dev->struct_mutex);
673a394b 3598 return ret;
6dbe2772 3599 }
673a394b 3600
29105ccc
CW		 3601 /* Under UMS, be paranoid and evict. */
3602 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
5eac3ab4 3603 ret = i915_gem_evict_inactive(dev, false);
29105ccc
CW		 3604 if (ret) {
3605 mutex_unlock(&dev->struct_mutex);
3606 return ret;
3607 }
3608 }
3609
312817a3
CW		 3610 i915_gem_reset_fences(dev);
3611
29105ccc
CW		 3612 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3613 * We need to replace this with a semaphore, or something.
3614 * And not confound mm.suspended!
3615 */
3616 dev_priv->mm.suspended = 1;
bc0c7f14 3617 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 3618
3619 i915_kernel_lost_context(dev);
6dbe2772 3620 i915_gem_cleanup_ringbuffer(dev);
29105ccc 3621
6dbe2772
KP		 3622 mutex_unlock(&dev->struct_mutex);
3623
29105ccc
CW		 3624 /* Cancel the retire work handler, which should be idle now. */
3625 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3626
673a394b
EA		 3627 return 0;
3628}
3629
8187a2b7
ZN		 3630int
3631i915_gem_init_ringbuffer(struct drm_device *dev)
3632{
3633 drm_i915_private_t *dev_priv = dev->dev_private;
3634 int ret;
68f95ba9 3635
5c1143bb 3636 ret = intel_init_render_ring_buffer(dev);
68f95ba9 3637 if (ret)
b6913e4b 3638 return ret;
68f95ba9
CW		 3639
3640 if (HAS_BSD(dev)) {
5c1143bb 3641 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 3642 if (ret)
3643 goto cleanup_render_ring;
d1b851fc 3644 }
68f95ba9 3645
549f7365
CW		 3646 if (HAS_BLT(dev)) {
3647 ret = intel_init_blt_ring_buffer(dev);
3648 if (ret)
3649 goto cleanup_bsd_ring;
3650 }
3651
6f392d54
CW		 3652 dev_priv->next_seqno = 1;
3653
68f95ba9
CW		 3654 return 0;
3655
549f7365 3656cleanup_bsd_ring:
1ec14ad3 3657 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
68f95ba9 3658cleanup_render_ring:
1ec14ad3 3659 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
8187a2b7
ZN		 3660 return ret;
3661}
3662
3663void
3664i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3665{
3666 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 3667 int i;
8187a2b7 3668
1ec14ad3
CW		 3669 for (i = 0; i < I915_NUM_RINGS; i++)
3670 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
8187a2b7
ZN		 3671}
3672
673a394b
EA		 3673int
3674i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3675 struct drm_file *file_priv)
3676{
3677 drm_i915_private_t *dev_priv = dev->dev_private;
1ec14ad3 3678 int ret, i;
673a394b 3679
79e53945
JB		 3680 if (drm_core_check_feature(dev, DRIVER_MODESET))
3681 return 0;
3682
ba1234d1 3683 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3684 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 3685 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 3686 }
3687
673a394b 3688 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 3689 dev_priv->mm.suspended = 0;
3690
3691 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 3692 if (ret != 0) {
3693 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 3694 return ret;
d816f6ac 3695 }
9bb2d6f9 3696
69dc4987 3697 BUG_ON(!list_empty(&dev_priv->mm.active_list));
673a394b
EA		 3698 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3699 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
1ec14ad3
CW		 3700 for (i = 0; i < I915_NUM_RINGS; i++) {
3701 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3702 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3703 }
673a394b 3704 mutex_unlock(&dev->struct_mutex);
dbb19d30 3705
5f35308b
CW		 3706 ret = drm_irq_install(dev);
3707 if (ret)
3708 goto cleanup_ringbuffer;
dbb19d30 3709
673a394b 3710 return 0;
5f35308b
CW		 3711
3712cleanup_ringbuffer:
3713 mutex_lock(&dev->struct_mutex);
3714 i915_gem_cleanup_ringbuffer(dev);
3715 dev_priv->mm.suspended = 1;
3716 mutex_unlock(&dev->struct_mutex);
3717
3718 return ret;
673a394b
EA		 3719}
3720
3721int
3722i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3723 struct drm_file *file_priv)
3724{
79e53945
JB		 3725 if (drm_core_check_feature(dev, DRIVER_MODESET))
3726 return 0;
3727
dbb19d30 3728 drm_irq_uninstall(dev);
e6890f6f 3729 return i915_gem_idle(dev);
673a394b
EA		 3730}
3731
3732void
3733i915_gem_lastclose(struct drm_device *dev)
3734{
3735 int ret;
673a394b 3736
e806b495
EA		 3737 if (drm_core_check_feature(dev, DRIVER_MODESET))
3738 return;
3739
6dbe2772
KP		 3740 ret = i915_gem_idle(dev);
3741 if (ret)
3742 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 3743}
3744
64193406
CW		 3745static void
3746init_ring_lists(struct intel_ring_buffer *ring)
3747{
3748 INIT_LIST_HEAD(&ring->active_list);
3749 INIT_LIST_HEAD(&ring->request_list);
3750 INIT_LIST_HEAD(&ring->gpu_write_list);
3751}
3752
673a394b
EA		 3753void
3754i915_gem_load(struct drm_device *dev)
3755{
b5aa8a0f 3756 int i;
673a394b
EA		 3757 drm_i915_private_t *dev_priv = dev->dev_private;
3758
69dc4987 3759 INIT_LIST_HEAD(&dev_priv->mm.active_list);
673a394b
EA		 3760 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3761 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 3762 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 3763 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 3764 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
93a37f20 3765 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
1ec14ad3
CW		 3766 for (i = 0; i < I915_NUM_RINGS; i++)
3767 init_ring_lists(&dev_priv->ring[i]);
007cc8ac
DV		 3768 for (i = 0; i < 16; i++)
3769 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 3770 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3771 i915_gem_retire_work_handler);
30dbf0c0 3772 init_completion(&dev_priv->error_completion);
31169714 3773
94400120
DA		 3774 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3775 if (IS_GEN3(dev)) {
3776 u32 tmp = I915_READ(MI_ARB_STATE);
3777 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
3778 /* arb state is a masked write, so set bit + bit in mask */
3779 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
3780 I915_WRITE(MI_ARB_STATE, tmp);
3781 }
3782 }
3783
72bfa19c
CW		 3784 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3785
de151cf6 3786 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 3787 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3788 dev_priv->fence_reg_start = 3;
de151cf6 3789
a6c45cf0 3790 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 3791 dev_priv->num_fence_regs = 16;
3792 else
3793 dev_priv->num_fence_regs = 8;
3794
b5aa8a0f 3795 /* Initialize fence registers to zero */
a6c45cf0
CW		 3796 switch (INTEL_INFO(dev)->gen) {
3797 case 6:
3798 for (i = 0; i < 16; i++)
3799 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
3800 break;
3801 case 5:
3802 case 4:
b5aa8a0f
GH		 3803 for (i = 0; i < 16; i++)
3804 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 3805 break;
3806 case 3:
b5aa8a0f
GH		 3807 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3808 for (i = 0; i < 8; i++)
3809 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 3810 case 2:
3811 for (i = 0; i < 8; i++)
3812 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
3813 break;
b5aa8a0f 3814 }
673a394b 3815 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 3816 init_waitqueue_head(&dev_priv->pending_flip_queue);
17250b71
CW		 3817
3818 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3819 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3820 register_shrinker(&dev_priv->mm.inactive_shrinker);
673a394b 3821}
71acb5eb
DA		 3822
3823/*
3824 * Create a physically contiguous memory object for this object
3825 * e.g. for cursor + overlay regs
3826 */
995b6762
CW		 3827static int i915_gem_init_phys_object(struct drm_device *dev,
3828 int id, int size, int align)
71acb5eb
DA		 3829{
3830 drm_i915_private_t *dev_priv = dev->dev_private;
3831 struct drm_i915_gem_phys_object *phys_obj;
3832 int ret;
3833
3834 if (dev_priv->mm.phys_objs[id - 1] || !size)
3835 return 0;
3836
9a298b2a 3837 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 3838 if (!phys_obj)
3839 return -ENOMEM;
3840
3841 phys_obj->id = id;
3842
6eeefaf3 3843 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 3844 if (!phys_obj->handle) {
3845 ret = -ENOMEM;
3846 goto kfree_obj;
3847 }
3848#ifdef CONFIG_X86
3849 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3850#endif
3851
3852 dev_priv->mm.phys_objs[id - 1] = phys_obj;
3853
3854 return 0;
3855kfree_obj:
9a298b2a 3856 kfree(phys_obj);
71acb5eb
DA		 3857 return ret;
3858}
3859
995b6762 3860static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 3861{
3862 drm_i915_private_t *dev_priv = dev->dev_private;
3863 struct drm_i915_gem_phys_object *phys_obj;
3864
3865 if (!dev_priv->mm.phys_objs[id - 1])
3866 return;
3867
3868 phys_obj = dev_priv->mm.phys_objs[id - 1];
3869 if (phys_obj->cur_obj) {
3870 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3871 }
3872
3873#ifdef CONFIG_X86
3874 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3875#endif
3876 drm_pci_free(dev, phys_obj->handle);
3877 kfree(phys_obj);
3878 dev_priv->mm.phys_objs[id - 1] = NULL;
3879}
3880
3881void i915_gem_free_all_phys_object(struct drm_device *dev)
3882{
3883 int i;
3884
260883c8 3885 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 3886 i915_gem_free_phys_object(dev, i);
3887}
3888
3889void i915_gem_detach_phys_object(struct drm_device *dev,
05394f39 3890 struct drm_i915_gem_object *obj)
71acb5eb 3891{
05394f39 3892 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
e5281ccd 3893 char *vaddr;
71acb5eb 3894 int i;
71acb5eb
DA		 3895 int page_count;
3896
05394f39 3897 if (!obj->phys_obj)
71acb5eb 3898 return;
05394f39 3899 vaddr = obj->phys_obj->handle->vaddr;
71acb5eb 3900
05394f39 3901 page_count = obj->base.size / PAGE_SIZE;
71acb5eb 3902 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 3903 struct page *page = read_cache_page_gfp(mapping, i,
3904 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3905 if (!IS_ERR(page)) {
3906 char *dst = kmap_atomic(page);
3907 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3908 kunmap_atomic(dst);
3909
3910 drm_clflush_pages(&page, 1);
3911
3912 set_page_dirty(page);
3913 mark_page_accessed(page);
3914 page_cache_release(page);
3915 }
71acb5eb 3916 }
40ce6575 3917 intel_gtt_chipset_flush();
d78b47b9 3918
05394f39
CW		 3919 obj->phys_obj->cur_obj = NULL;
3920 obj->phys_obj = NULL;
71acb5eb
DA		 3921}
3922
3923int
3924i915_gem_attach_phys_object(struct drm_device *dev,
05394f39 3925 struct drm_i915_gem_object *obj,
6eeefaf3
CW		 3926 int id,
3927 int align)
71acb5eb 3928{
05394f39 3929 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
71acb5eb 3930 drm_i915_private_t *dev_priv = dev->dev_private;
71acb5eb
DA		 3931 int ret = 0;
3932 int page_count;
3933 int i;
3934
3935 if (id > I915_MAX_PHYS_OBJECT)
3936 return -EINVAL;
3937
05394f39
CW		 3938 if (obj->phys_obj) {
3939 if (obj->phys_obj->id == id)
71acb5eb
DA		 3940 return 0;
3941 i915_gem_detach_phys_object(dev, obj);
3942 }
3943
71acb5eb
DA		 3944 /* create a new object */
3945 if (!dev_priv->mm.phys_objs[id - 1]) {
3946 ret = i915_gem_init_phys_object(dev, id,
05394f39 3947 obj->base.size, align);
71acb5eb 3948 if (ret) {
05394f39
CW		 3949 DRM_ERROR("failed to init phys object %d size: %zu\n",
3950 id, obj->base.size);
e5281ccd 3951 return ret;
71acb5eb
DA		 3952 }
3953 }
3954
3955 /* bind to the object */
05394f39
CW		 3956 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3957 obj->phys_obj->cur_obj = obj;
71acb5eb 3958
05394f39 3959 page_count = obj->base.size / PAGE_SIZE;
71acb5eb
DA		 3960
3961 for (i = 0; i < page_count; i++) {
e5281ccd
CW		 3962 struct page *page;
3963 char *dst, *src;
3964
3965 page = read_cache_page_gfp(mapping, i,
3966 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3967 if (IS_ERR(page))
3968 return PTR_ERR(page);
71acb5eb 3969
ff75b9bc 3970 src = kmap_atomic(page);
05394f39 3971 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
71acb5eb 3972 memcpy(dst, src, PAGE_SIZE);
3e4d3af5 3973 kunmap_atomic(src);
71acb5eb 3974
e5281ccd
CW		 3975 mark_page_accessed(page);
3976 page_cache_release(page);
3977 }
d78b47b9 3978
71acb5eb 3979 return 0;
71acb5eb
DA		 3980}
3981
3982static int
05394f39
CW		 3983i915_gem_phys_pwrite(struct drm_device *dev,
3984 struct drm_i915_gem_object *obj,
71acb5eb
DA		 3985 struct drm_i915_gem_pwrite *args,
3986 struct drm_file *file_priv)
3987{
05394f39 3988 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
b47b30cc 3989 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
71acb5eb 3990
b47b30cc
CW		 3991 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
3992 unsigned long unwritten;
3993
3994 /* The physical object once assigned is fixed for the lifetime
3995 * of the obj, so we can safely drop the lock and continue
3996 * to access vaddr.
3997 */
3998 mutex_unlock(&dev->struct_mutex);
3999 unwritten = copy_from_user(vaddr, user_data, args->size);
4000 mutex_lock(&dev->struct_mutex);
4001 if (unwritten)
4002 return -EFAULT;
4003 }
71acb5eb 4004
40ce6575 4005 intel_gtt_chipset_flush();
71acb5eb
DA		 4006 return 0;
4007}
b962442e 4008
f787a5f5 4009void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4010{
f787a5f5 4011 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4012
4013 /* Clean up our request list when the client is going away, so that
4014 * later retire_requests won't dereference our soon-to-be-gone
4015 * file_priv.
4016 */
1c25595f 4017 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4018 while (!list_empty(&file_priv->mm.request_list)) {
4019 struct drm_i915_gem_request *request;
4020
4021 request = list_first_entry(&file_priv->mm.request_list,
4022 struct drm_i915_gem_request,
4023 client_list);
4024 list_del(&request->client_list);
4025 request->file_priv = NULL;
4026 }
1c25595f 4027 spin_unlock(&file_priv->mm.lock);
b962442e 4028}
31169714 4029
1637ef41
CW		 4030static int
4031i915_gpu_is_active(struct drm_device *dev)
4032{
4033 drm_i915_private_t *dev_priv = dev->dev_private;
4034 int lists_empty;
4035
1637ef41 4036 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
17250b71 4037 list_empty(&dev_priv->mm.active_list);
1637ef41
CW		 4038
4039 return !lists_empty;
4040}
4041
31169714 4042static int
17250b71
CW		 4043i915_gem_inactive_shrink(struct shrinker *shrinker,
4044 int nr_to_scan,
4045 gfp_t gfp_mask)
31169714 4046{
17250b71
CW		 4047 struct drm_i915_private *dev_priv =
4048 container_of(shrinker,
4049 struct drm_i915_private,
4050 mm.inactive_shrinker);
4051 struct drm_device *dev = dev_priv->dev;
4052 struct drm_i915_gem_object *obj, *next;
4053 int cnt;
4054
4055 if (!mutex_trylock(&dev->struct_mutex))
bbe2e11a 4056 return 0;
31169714
CW		 4057
4058 /* "fast-path" to count number of available objects */
4059 if (nr_to_scan == 0) {
17250b71
CW		 4060 cnt = 0;
4061 list_for_each_entry(obj,
4062 &dev_priv->mm.inactive_list,
4063 mm_list)
4064 cnt++;
4065 mutex_unlock(&dev->struct_mutex);
4066 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714
CW		 4067 }
4068
1637ef41 4069rescan:
31169714 4070 /* first scan for clean buffers */
17250b71 4071 i915_gem_retire_requests(dev);
31169714 4072
17250b71
CW		 4073 list_for_each_entry_safe(obj, next,
4074 &dev_priv->mm.inactive_list,
4075 mm_list) {
4076 if (i915_gem_object_is_purgeable(obj)) {
2021746e
CW		 4077 if (i915_gem_object_unbind(obj) == 0 &&
4078 --nr_to_scan == 0)
17250b71 4079 break;
31169714 4080 }
31169714
CW		 4081 }
4082
4083 /* second pass, evict/count anything still on the inactive list */
17250b71
CW		 4084 cnt = 0;
4085 list_for_each_entry_safe(obj, next,
4086 &dev_priv->mm.inactive_list,
4087 mm_list) {
2021746e
CW		 4088 if (nr_to_scan &&
4089 i915_gem_object_unbind(obj) == 0)
17250b71 4090 nr_to_scan--;
2021746e 4091 else
17250b71
CW		 4092 cnt++;
4093 }
4094
4095 if (nr_to_scan && i915_gpu_is_active(dev)) {
1637ef41
CW		 4096 /*
4097 * We are desperate for pages, so as a last resort, wait
4098 * for the GPU to finish and discard whatever we can.
4099 * This has a dramatic impact to reduce the number of
4100 * OOM-killer events whilst running the GPU aggressively.
4101 */
17250b71 4102 if (i915_gpu_idle(dev) == 0)
1637ef41
CW		 4103 goto rescan;
4104 }
17250b71
CW		 4105 mutex_unlock(&dev->struct_mutex);
4106 return cnt / 100 * sysctl_vfs_cache_pressure;
31169714 4107}
Linux 6.x block layer and io_uring tree(s)