mm: free device private pages have zero refcount

Since 27674ef6c73f ("mm: remove the extra ZONE_DEVICE struct page
refcount") device private pages have no longer had an extra reference
count when the page is in use.  However before handing them back to the
owning device driver we add an extra reference count such that free pages
have a reference count of one.

This makes it difficult to tell if a page is free or not because both free
and in use pages will have a non-zero refcount.  Instead we should return
pages to the drivers page allocator with a zero reference count.  Kernel
code can then safely use kernel functions such as get_page_unless_zero().

Link: https://lkml.kernel.org/r/cf70cf6f8c0bdb8aaebdbfb0d790aea4c683c3c6.1664366292.git-series.apopple@nvidia.com
Signed-off-by: Alistair Popple <apopple@nvidia.com>
Acked-by: Felix Kuehling <Felix.Kuehling@amd.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Alex Deucher <alexander.deucher@amd.com>
Cc: Christian König <christian.koenig@amd.com>
Cc: Ben Skeggs <bskeggs@redhat.com>
Cc: Lyude Paul <lyude@redhat.com>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Alex Sierra <alex.sierra@amd.com>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: "Huang, Ying" <ying.huang@intel.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c
index 965c9e9e500bceae02b9bbefbb7bbffe0e05058a..e2f11f9c3f2aa691bd4b1a130d06d38d989ecc5c 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv_uvmem.c
+++ b/arch/powerpc/kvm/book3s_hv_uvmem.c
@@ -718,7 +718,7 @@ static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm)
 
        dpage = pfn_to_page(uvmem_pfn);
        dpage->zone_device_data = pvt;
-       lock_page(dpage);
+       zone_device_page_init(dpage);
        return dpage;
 out_clear:
        spin_lock(&kvmppc_uvmem_bitmap_lock);

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 776448bd9fe4abfa6c6a4d16b667b9eb451199f0..97a684568ae01e9e2f8301a1a1ca2a5313f42975 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -223,7 +223,7 @@ svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
        page = pfn_to_page(pfn);
        svm_range_bo_ref(prange->svm_bo);
        page->zone_device_data = prange->svm_bo;
-       lock_page(page);
+       zone_device_page_init(page);
 }
 
 static void

diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c
index 16356611b5b9500bc2563b704bb0f8120a5dba2b..b092988266a6af0b2d828d824167d5a5284db8ed 100644 (file)
--- a/drivers/gpu/drm/nouveau/nouveau_dmem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c
@@ -326,7 +326,7 @@ nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
                        return NULL;
        }
 
-       lock_page(page);
+       zone_device_page_init(page);
        return page;
 }
 

diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index c3b4cc84877b5611576b44b04ed2bdaea7e0646f..7fcaf3180a5b6268f05451c35083148a4cf2fc02 100644 (file)
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -187,6 +187,7 @@ static inline bool folio_is_device_coherent(const struct folio *folio)
 }
 
 #ifdef CONFIG_ZONE_DEVICE
+void zone_device_page_init(struct page *page);
 void *memremap_pages(struct dev_pagemap *pgmap, int nid);
 void memunmap_pages(struct dev_pagemap *pgmap);
 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);

diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index e566166b55712656e3bb63438c3e80cbf0fa08f2..bc2b9499116532a18e936ace205c25eafa5b8da6 100644 (file)
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -627,8 +627,8 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
                        goto error;
        }
 
+       zone_device_page_init(dpage);
        dpage->zone_device_data = rpage;
-       lock_page(dpage);
        return dpage;
 
 error:

diff --git a/mm/memremap.c b/mm/memremap.c
index 25029a474d30bc86abe42bf5424027417943e670..1c2c038f34109953665fc834d49dd9a1eec92459 100644 (file)
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -505,8 +505,17 @@ void free_zone_device_page(struct page *page)
        /*
         * Reset the page count to 1 to prepare for handing out the page again.
         */
+       if (page->pgmap->type != MEMORY_DEVICE_PRIVATE &&
+           page->pgmap->type != MEMORY_DEVICE_COHERENT)
+               set_page_count(page, 1);
+}
+
+void zone_device_page_init(struct page *page)
+{
        set_page_count(page, 1);
+       lock_page(page);
 }
+EXPORT_SYMBOL_GPL(zone_device_page_init);
 
 #ifdef CONFIG_FS_DAX
 bool __put_devmap_managed_page_refs(struct page *page, int refs)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 12b6184cbbed6d00107c9d509012250c5c31c80c..059f6946832fa9279f39391950dcbec0e433a929 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6819,6 +6819,14 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
                set_pageblock_migratetype(page, MIGRATE_MOVABLE);
                cond_resched();
        }
+
+       /*
+        * ZONE_DEVICE pages are released directly to the driver page allocator
+        * which will set the page count to 1 when allocating the page.
+        */
+       if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
+           pgmap->type == MEMORY_DEVICE_COHERENT)
+               set_page_count(page, 0);
 }
 
 /*