drivers/gpu/drm/i915/gem/i915_gem_shmem.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2016 Intel Corporation
   5  */
   6
   7 #include <linux/pagevec.h>
   8 #include <linux/shmem_fs.h>
   9 #include <linux/swap.h>
  10
  11 #include <drm/drm_cache.h>
  12
  13 #include "gem/i915_gem_region.h"
  14 #include "i915_drv.h"
  15 #include "i915_gem_object.h"
  16 #include "i915_gem_tiling.h"
  17 #include "i915_gemfs.h"
  18 #include "i915_scatterlist.h"
  19 #include "i915_trace.h"
  20
  21 /*
  22  * Move folios to appropriate lru and release the batch, decrementing the
  23  * ref count of those folios.
  24  */
  25 static void check_release_folio_batch(struct folio_batch *fbatch)
  26 {
  27         check_move_unevictable_folios(fbatch);
  28         __folio_batch_release(fbatch);
  29         cond_resched();
  30 }
  31
  32 void shmem_sg_free_table(struct sg_table *st, struct address_space *mapping,
  33                          bool dirty, bool backup)
  34 {
  35         struct sgt_iter sgt_iter;
  36         struct folio_batch fbatch;
  37         struct folio *last = NULL;
  38         struct page *page;
  39
  40         mapping_clear_unevictable(mapping);
  41
  42         folio_batch_init(&fbatch);
  43         for_each_sgt_page(page, sgt_iter, st) {
  44                 struct folio *folio = page_folio(page);
  45
  46                 if (folio == last)
  47                         continue;
  48                 last = folio;
  49                 if (dirty)
  50                         folio_mark_dirty(folio);
  51                 if (backup)
  52                         folio_mark_accessed(folio);
  53
  54                 if (!folio_batch_add(&fbatch, folio))
  55                         check_release_folio_batch(&fbatch);
  56         }
  57         if (fbatch.nr)
  58                 check_release_folio_batch(&fbatch);
  59
  60         sg_free_table(st);
  61 }
  62
  63 int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,
  64                          size_t size, struct intel_memory_region *mr,
  65                          struct address_space *mapping,
  66                          unsigned int max_segment)
  67 {
  68         unsigned int page_count; /* restricted by sg_alloc_table */
  69         unsigned long i;
  70         struct scatterlist *sg;
  71         unsigned long next_pfn = 0;     /* suppress gcc warning */
  72         gfp_t noreclaim;
  73         int ret;
  74
  75         if (overflows_type(size / PAGE_SIZE, page_count))
  76                 return -E2BIG;
  77
  78         page_count = size / PAGE_SIZE;
  79         /*
  80          * If there's no chance of allocating enough pages for the whole
  81          * object, bail early.
  82          */
  83         if (size > resource_size(&mr->region))
  84                 return -ENOMEM;
  85
  86         if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN))
  87                 return -ENOMEM;
  88
  89         /*
  90          * Get the list of pages out of our struct file.  They'll be pinned
  91          * at this point until we release them.
  92          *
  93          * Fail silently without starting the shrinker
  94          */
  95         mapping_set_unevictable(mapping);
  96         noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
  97         noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
  98
  99         sg = st->sgl;
 100         st->nents = 0;
 101         for (i = 0; i < page_count; i++) {
 102                 struct folio *folio;
 103                 unsigned long nr_pages;
 104                 const unsigned int shrink[] = {
 105                         I915_SHRINK_BOUND | I915_SHRINK_UNBOUND,
 106                         0,
 107                 }, *s = shrink;
 108                 gfp_t gfp = noreclaim;
 109
 110                 do {
 111                         cond_resched();
 112                         folio = shmem_read_folio_gfp(mapping, i, gfp);
 113                         if (!IS_ERR(folio))
 114                                 break;
 115
 116                         if (!*s) {
 117                                 ret = PTR_ERR(folio);
 118                                 goto err_sg;
 119                         }
 120
 121                         i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);
 122
 123                         /*
 124                          * We've tried hard to allocate the memory by reaping
 125                          * our own buffer, now let the real VM do its job and
 126                          * go down in flames if truly OOM.
 127                          *
 128                          * However, since graphics tend to be disposable,
 129                          * defer the oom here by reporting the ENOMEM back
 130                          * to userspace.
 131                          */
 132                         if (!*s) {
 133                                 /* reclaim and warn, but no oom */
 134                                 gfp = mapping_gfp_mask(mapping);
 135
 136                                 /*
 137                                  * Our bo are always dirty and so we require
 138                                  * kswapd to reclaim our pages (direct reclaim
 139                                  * does not effectively begin pageout of our
 140                                  * buffers on its own). However, direct reclaim
 141                                  * only waits for kswapd when under allocation
 142                                  * congestion. So as a result __GFP_RECLAIM is
 143                                  * unreliable and fails to actually reclaim our
 144                                  * dirty pages -- unless you try over and over
 145                                  * again with !__GFP_NORETRY. However, we still
 146                                  * want to fail this allocation rather than
 147                                  * trigger the out-of-memory killer and for
 148                                  * this we want __GFP_RETRY_MAYFAIL.
 149                                  */
 150                                 gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
 151                         }
 152                 } while (1);
 153
 154                 nr_pages = min_t(unsigned long,
 155                                 folio_nr_pages(folio), page_count - i);
 156                 if (!i ||
 157                     sg->length >= max_segment ||
 158                     folio_pfn(folio) != next_pfn) {
 159                         if (i)
 160                                 sg = sg_next(sg);
 161
 162                         st->nents++;
 163                         sg_set_folio(sg, folio, nr_pages * PAGE_SIZE, 0);
 164                 } else {
 165                         /* XXX: could overflow? */
 166                         sg->length += nr_pages * PAGE_SIZE;
 167                 }
 168                 next_pfn = folio_pfn(folio) + nr_pages;
 169                 i += nr_pages - 1;
 170
 171                 /* Check that the i965g/gm workaround works. */
 172                 GEM_BUG_ON(gfp & __GFP_DMA32 && next_pfn >= 0x00100000UL);
 173         }
 174         if (sg) /* loop terminated early; short sg table */
 175                 sg_mark_end(sg);
 176
 177         /* Trim unused sg entries to avoid wasting memory. */
 178         i915_sg_trim(st);
 179
 180         return 0;
 181 err_sg:
 182         sg_mark_end(sg);
 183         if (sg != st->sgl) {
 184                 shmem_sg_free_table(st, mapping, false, false);
 185         } else {
 186                 mapping_clear_unevictable(mapping);
 187                 sg_free_table(st);
 188         }
 189
 190         /*
 191          * shmemfs first checks if there is enough memory to allocate the page
 192          * and reports ENOSPC should there be insufficient, along with the usual
 193          * ENOMEM for a genuine allocation failure.
 194          *
 195          * We use ENOSPC in our driver to mean that we have run out of aperture
 196          * space and so want to translate the error from shmemfs back to our
 197          * usual understanding of ENOMEM.
 198          */
 199         if (ret == -ENOSPC)
 200                 ret = -ENOMEM;
 201
 202         return ret;
 203 }
 204
 205 static int shmem_get_pages(struct drm_i915_gem_object *obj)
 206 {
 207         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 208         struct intel_memory_region *mem = obj->mm.region;
 209         struct address_space *mapping = obj->base.filp->f_mapping;
 210         unsigned int max_segment = i915_sg_segment_size(i915->drm.dev);
 211         struct sg_table *st;
 212         struct sgt_iter sgt_iter;
 213         struct page *page;
 214         int ret;
 215
 216         /*
 217          * Assert that the object is not currently in any GPU domain. As it
 218          * wasn't in the GTT, there shouldn't be any way it could have been in
 219          * a GPU cache
 220          */
 221         GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
 222         GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
 223
 224 rebuild_st:
 225         st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN);
 226         if (!st)
 227                 return -ENOMEM;
 228
 229         ret = shmem_sg_alloc_table(i915, st, obj->base.size, mem, mapping,
 230                                    max_segment);
 231         if (ret)
 232                 goto err_st;
 233
 234         ret = i915_gem_gtt_prepare_pages(obj, st);
 235         if (ret) {
 236                 /*
 237                  * DMA remapping failed? One possible cause is that
 238                  * it could not reserve enough large entries, asking
 239                  * for PAGE_SIZE chunks instead may be helpful.
 240                  */
 241                 if (max_segment > PAGE_SIZE) {
 242                         for_each_sgt_page(page, sgt_iter, st)
 243                                 put_page(page);
 244                         sg_free_table(st);
 245                         kfree(st);
 246
 247                         max_segment = PAGE_SIZE;
 248                         goto rebuild_st;
 249                 } else {
 250                         dev_warn(i915->drm.dev,
 251                                  "Failed to DMA remap %zu pages\n",
 252                                  obj->base.size >> PAGE_SHIFT);
 253                         goto err_pages;
 254                 }
 255         }
 256
 257         if (i915_gem_object_needs_bit17_swizzle(obj))
 258                 i915_gem_object_do_bit_17_swizzle(obj, st);
 259
 260         if (i915_gem_object_can_bypass_llc(obj))
 261                 obj->cache_dirty = true;
 262
 263         __i915_gem_object_set_pages(obj, st);
 264
 265         return 0;
 266
 267 err_pages:
 268         shmem_sg_free_table(st, mapping, false, false);
 269         /*
 270          * shmemfs first checks if there is enough memory to allocate the page
 271          * and reports ENOSPC should there be insufficient, along with the usual
 272          * ENOMEM for a genuine allocation failure.
 273          *
 274          * We use ENOSPC in our driver to mean that we have run out of aperture
 275          * space and so want to translate the error from shmemfs back to our
 276          * usual understanding of ENOMEM.
 277          */
 278 err_st:
 279         if (ret == -ENOSPC)
 280                 ret = -ENOMEM;
 281
 282         kfree(st);
 283
 284         return ret;
 285 }
 286
 287 static int
 288 shmem_truncate(struct drm_i915_gem_object *obj)
 289 {
 290         /*
 291          * Our goal here is to return as much of the memory as
 292          * is possible back to the system as we are called from OOM.
 293          * To do this we must instruct the shmfs to drop all of its
 294          * backing pages, *now*.
 295          */
 296         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
 297         obj->mm.madv = __I915_MADV_PURGED;
 298         obj->mm.pages = ERR_PTR(-EFAULT);
 299
 300         return 0;
 301 }
 302
 303 void __shmem_writeback(size_t size, struct address_space *mapping)
 304 {
 305         struct writeback_control wbc = {
 306                 .sync_mode = WB_SYNC_NONE,
 307                 .nr_to_write = SWAP_CLUSTER_MAX,
 308                 .range_start = 0,
 309                 .range_end = LLONG_MAX,
 310                 .for_reclaim = 1,
 311         };
 312         unsigned long i;
 313
 314         /*
 315          * Leave mmapings intact (GTT will have been revoked on unbinding,
 316          * leaving only CPU mmapings around) and add those pages to the LRU
 317          * instead of invoking writeback so they are aged and paged out
 318          * as normal.
 319          */
 320
 321         /* Begin writeback on each dirty page */
 322         for (i = 0; i < size >> PAGE_SHIFT; i++) {
 323                 struct page *page;
 324
 325                 page = find_lock_page(mapping, i);
 326                 if (!page)
 327                         continue;
 328
 329                 if (!page_mapped(page) && clear_page_dirty_for_io(page)) {
 330                         int ret;
 331
 332                         SetPageReclaim(page);
 333                         ret = mapping->a_ops->writepage(page, &wbc);
 334                         if (!PageWriteback(page))
 335                                 ClearPageReclaim(page);
 336                         if (!ret)
 337                                 goto put;
 338                 }
 339                 unlock_page(page);
 340 put:
 341                 put_page(page);
 342         }
 343 }
 344
 345 static void
 346 shmem_writeback(struct drm_i915_gem_object *obj)
 347 {
 348         __shmem_writeback(obj->base.size, obj->base.filp->f_mapping);
 349 }
 350
 351 static int shmem_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
 352 {
 353         switch (obj->mm.madv) {
 354         case I915_MADV_DONTNEED:
 355                 return i915_gem_object_truncate(obj);
 356         case __I915_MADV_PURGED:
 357                 return 0;
 358         }
 359
 360         if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK)
 361                 shmem_writeback(obj);
 362
 363         return 0;
 364 }
 365
 366 void
 367 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 368                                 struct sg_table *pages,
 369                                 bool needs_clflush)
 370 {
 371         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 372
 373         GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 374
 375         if (obj->mm.madv == I915_MADV_DONTNEED)
 376                 obj->mm.dirty = false;
 377
 378         if (needs_clflush &&
 379             (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
 380             !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
 381                 drm_clflush_sg(pages);
 382
 383         __start_cpu_write(obj);
 384         /*
 385          * On non-LLC igfx platforms, force the flush-on-acquire if this is ever
 386          * swapped-in. Our async flush path is not trust worthy enough yet(and
 387          * happens in the wrong order), and with some tricks it's conceivable
 388          * for userspace to change the cache-level to I915_CACHE_NONE after the
 389          * pages are swapped-in, and since execbuf binds the object before doing
 390          * the async flush, we have a race window.
 391          */
 392         if (!HAS_LLC(i915) && !IS_DGFX(i915))
 393                 obj->cache_dirty = true;
 394 }
 395
 396 void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
 397 {
 398         __i915_gem_object_release_shmem(obj, pages, true);
 399
 400         i915_gem_gtt_finish_pages(obj, pages);
 401
 402         if (i915_gem_object_needs_bit17_swizzle(obj))
 403                 i915_gem_object_save_bit_17_swizzle(obj, pages);
 404
 405         shmem_sg_free_table(pages, file_inode(obj->base.filp)->i_mapping,
 406                             obj->mm.dirty, obj->mm.madv == I915_MADV_WILLNEED);
 407         kfree(pages);
 408         obj->mm.dirty = false;
 409 }
 410
 411 static void
 412 shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
 413 {
 414         if (likely(i915_gem_object_has_struct_page(obj)))
 415                 i915_gem_object_put_pages_shmem(obj, pages);
 416         else
 417                 i915_gem_object_put_pages_phys(obj, pages);
 418 }
 419
 420 static int
 421 shmem_pwrite(struct drm_i915_gem_object *obj,
 422              const struct drm_i915_gem_pwrite *arg)
 423 {
 424         struct address_space *mapping = obj->base.filp->f_mapping;
 425         const struct address_space_operations *aops = mapping->a_ops;
 426         char __user *user_data = u64_to_user_ptr(arg->data_ptr);
 427         u64 remain, offset;
 428         unsigned int pg;
 429
 430         /* Caller already validated user args */
 431         GEM_BUG_ON(!access_ok(user_data, arg->size));
 432
 433         if (!i915_gem_object_has_struct_page(obj))
 434                 return i915_gem_object_pwrite_phys(obj, arg);
 435
 436         /*
 437          * Before we instantiate/pin the backing store for our use, we
 438          * can prepopulate the shmemfs filp efficiently using a write into
 439          * the pagecache. We avoid the penalty of instantiating all the
 440          * pages, important if the user is just writing to a few and never
 441          * uses the object on the GPU, and using a direct write into shmemfs
 442          * allows it to avoid the cost of retrieving a page (either swapin
 443          * or clearing-before-use) before it is overwritten.
 444          */
 445         if (i915_gem_object_has_pages(obj))
 446                 return -ENODEV;
 447
 448         if (obj->mm.madv != I915_MADV_WILLNEED)
 449                 return -EFAULT;
 450
 451         /*
 452          * Before the pages are instantiated the object is treated as being
 453          * in the CPU domain. The pages will be clflushed as required before
 454          * use, and we can freely write into the pages directly. If userspace
 455          * races pwrite with any other operation; corruption will ensue -
 456          * that is userspace's prerogative!
 457          */
 458
 459         remain = arg->size;
 460         offset = arg->offset;
 461         pg = offset_in_page(offset);
 462
 463         do {
 464                 unsigned int len, unwritten;
 465                 struct page *page;
 466                 void *data, *vaddr;
 467                 int err;
 468                 char __maybe_unused c;
 469
 470                 len = PAGE_SIZE - pg;
 471                 if (len > remain)
 472                         len = remain;
 473
 474                 /* Prefault the user page to reduce potential recursion */
 475                 err = __get_user(c, user_data);
 476                 if (err)
 477                         return err;
 478
 479                 err = __get_user(c, user_data + len - 1);
 480                 if (err)
 481                         return err;
 482
 483                 err = aops->write_begin(obj->base.filp, mapping, offset, len,
 484                                         &page, &data);
 485                 if (err < 0)
 486                         return err;
 487
 488                 vaddr = kmap_atomic(page);
 489                 unwritten = __copy_from_user_inatomic(vaddr + pg,
 490                                                       user_data,
 491                                                       len);
 492                 kunmap_atomic(vaddr);
 493
 494                 err = aops->write_end(obj->base.filp, mapping, offset, len,
 495                                       len - unwritten, page, data);
 496                 if (err < 0)
 497                         return err;
 498
 499                 /* We don't handle -EFAULT, leave it to the caller to check */
 500                 if (unwritten)
 501                         return -ENODEV;
 502
 503                 remain -= len;
 504                 user_data += len;
 505                 offset += len;
 506                 pg = 0;
 507         } while (remain);
 508
 509         return 0;
 510 }
 511
 512 static int
 513 shmem_pread(struct drm_i915_gem_object *obj,
 514             const struct drm_i915_gem_pread *arg)
 515 {
 516         if (!i915_gem_object_has_struct_page(obj))
 517                 return i915_gem_object_pread_phys(obj, arg);
 518
 519         return -ENODEV;
 520 }
 521
 522 static void shmem_release(struct drm_i915_gem_object *obj)
 523 {
 524         if (i915_gem_object_has_struct_page(obj))
 525                 i915_gem_object_release_memory_region(obj);
 526
 527         fput(obj->base.filp);
 528 }
 529
 530 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
 531         .name = "i915_gem_object_shmem",
 532         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 533
 534         .get_pages = shmem_get_pages,
 535         .put_pages = shmem_put_pages,
 536         .truncate = shmem_truncate,
 537         .shrink = shmem_shrink,
 538
 539         .pwrite = shmem_pwrite,
 540         .pread = shmem_pread,
 541
 542         .release = shmem_release,
 543 };
 544
 545 static int __create_shmem(struct drm_i915_private *i915,
 546                           struct drm_gem_object *obj,
 547                           resource_size_t size)
 548 {
 549         unsigned long flags = VM_NORESERVE;
 550         struct file *filp;
 551
 552         drm_gem_private_object_init(&i915->drm, obj, size);
 553
 554         /* XXX: The __shmem_file_setup() function returns -EINVAL if size is
 555          * greater than MAX_LFS_FILESIZE.
 556          * To handle the same error as other code that returns -E2BIG when
 557          * the size is too large, we add a code that returns -E2BIG when the
 558          * size is larger than the size that can be handled.
 559          * If BITS_PER_LONG is 32, size > MAX_LFS_FILESIZE is always false,
 560          * so we only needs to check when BITS_PER_LONG is 64.
 561          * If BITS_PER_LONG is 32, E2BIG checks are processed when
 562          * i915_gem_object_size_2big() is called before init_object() callback
 563          * is called.
 564          */
 565         if (BITS_PER_LONG == 64 && size > MAX_LFS_FILESIZE)
 566                 return -E2BIG;
 567
 568         if (i915->mm.gemfs)
 569                 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
 570                                                  flags);
 571         else
 572                 filp = shmem_file_setup("i915", size, flags);
 573         if (IS_ERR(filp))
 574                 return PTR_ERR(filp);
 575
 576         obj->filp = filp;
 577         return 0;
 578 }
 579
 580 static int shmem_object_init(struct intel_memory_region *mem,
 581                              struct drm_i915_gem_object *obj,
 582                              resource_size_t offset,
 583                              resource_size_t size,
 584                              resource_size_t page_size,
 585                              unsigned int flags)
 586 {
 587         static struct lock_class_key lock_class;
 588         struct drm_i915_private *i915 = mem->i915;
 589         struct address_space *mapping;
 590         unsigned int cache_level;
 591         gfp_t mask;
 592         int ret;
 593
 594         ret = __create_shmem(i915, &obj->base, size);
 595         if (ret)
 596                 return ret;
 597
 598         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
 599         if (IS_I965GM(i915) || IS_I965G(i915)) {
 600                 /* 965gm cannot relocate objects above 4GiB. */
 601                 mask &= ~__GFP_HIGHMEM;
 602                 mask |= __GFP_DMA32;
 603         }
 604
 605         mapping = obj->base.filp->f_mapping;
 606         mapping_set_gfp_mask(mapping, mask);
 607         GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
 608
 609         i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class, flags);
 610         obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
 611         obj->write_domain = I915_GEM_DOMAIN_CPU;
 612         obj->read_domains = I915_GEM_DOMAIN_CPU;
 613
 614         /*
 615          * MTL doesn't snoop CPU cache by default for GPU access (namely
 616          * 1-way coherency). However some UMD's are currently depending on
 617          * that. Make 1-way coherent the default setting for MTL. A follow
 618          * up patch will extend the GEM_CREATE uAPI to allow UMD's specify
 619          * caching mode at BO creation time
 620          */
 621         if (HAS_LLC(i915) || (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)))
 622                 /* On some devices, we can have the GPU use the LLC (the CPU
 623                  * cache) for about a 10% performance improvement
 624                  * compared to uncached.  Graphics requests other than
 625                  * display scanout are coherent with the CPU in
 626                  * accessing this cache.  This means in this mode we
 627                  * don't need to clflush on the CPU side, and on the
 628                  * GPU side we only need to flush internal caches to
 629                  * get data visible to the CPU.
 630                  *
 631                  * However, we maintain the display planes as UC, and so
 632                  * need to rebind when first used as such.
 633                  */
 634                 cache_level = I915_CACHE_LLC;
 635         else
 636                 cache_level = I915_CACHE_NONE;
 637
 638         i915_gem_object_set_cache_coherency(obj, cache_level);
 639
 640         i915_gem_object_init_memory_region(obj, mem);
 641
 642         return 0;
 643 }
 644
 645 struct drm_i915_gem_object *
 646 i915_gem_object_create_shmem(struct drm_i915_private *i915,
 647                              resource_size_t size)
 648 {
 649         return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
 650                                              size, 0, 0);
 651 }
 652
 653 /* Allocate a new GEM object and fill it with the supplied data */
 654 struct drm_i915_gem_object *
 655 i915_gem_object_create_shmem_from_data(struct drm_i915_private *dev_priv,
 656                                        const void *data, resource_size_t size)
 657 {
 658         struct drm_i915_gem_object *obj;
 659         struct file *file;
 660         const struct address_space_operations *aops;
 661         resource_size_t offset;
 662         int err;
 663
 664         GEM_WARN_ON(IS_DGFX(dev_priv));
 665         obj = i915_gem_object_create_shmem(dev_priv, round_up(size, PAGE_SIZE));
 666         if (IS_ERR(obj))
 667                 return obj;
 668
 669         GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
 670
 671         file = obj->base.filp;
 672         aops = file->f_mapping->a_ops;
 673         offset = 0;
 674         do {
 675                 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
 676                 struct page *page;
 677                 void *pgdata, *vaddr;
 678
 679                 err = aops->write_begin(file, file->f_mapping, offset, len,
 680                                         &page, &pgdata);
 681                 if (err < 0)
 682                         goto fail;
 683
 684                 vaddr = kmap(page);
 685                 memcpy(vaddr, data, len);
 686                 kunmap(page);
 687
 688                 err = aops->write_end(file, file->f_mapping, offset, len, len,
 689                                       page, pgdata);
 690                 if (err < 0)
 691                         goto fail;
 692
 693                 size -= len;
 694                 data += len;
 695                 offset += len;
 696         } while (size);
 697
 698         return obj;
 699
 700 fail:
 701         i915_gem_object_put(obj);
 702         return ERR_PTR(err);
 703 }
 704
 705 static int init_shmem(struct intel_memory_region *mem)
 706 {
 707         i915_gemfs_init(mem->i915);
 708         intel_memory_region_set_name(mem, "system");
 709
 710         return 0; /* We have fallback to the kernel mnt if gemfs init failed. */
 711 }
 712
 713 static int release_shmem(struct intel_memory_region *mem)
 714 {
 715         i915_gemfs_fini(mem->i915);
 716         return 0;
 717 }
 718
 719 static const struct intel_memory_region_ops shmem_region_ops = {
 720         .init = init_shmem,
 721         .release = release_shmem,
 722         .init_object = shmem_object_init,
 723 };
 724
 725 struct intel_memory_region *i915_gem_shmem_setup(struct drm_i915_private *i915,
 726                                                  u16 type, u16 instance)
 727 {
 728         return intel_memory_region_create(i915, 0,
 729                                           totalram_pages() << PAGE_SHIFT,
 730                                           PAGE_SIZE, 0, 0,
 731                                           type, instance,
 732                                           &shmem_region_ops);
 733 }
 734
 735 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj)
 736 {
 737         return obj->ops == &i915_gem_shmem_ops;
 738 }