iommu: Allow passing custom allocators to pgtable drivers

This will be useful for GPU drivers who want to keep page tables in a
pool so they can:

- keep freed page tables in a free pool and speed-up upcoming page
  table allocations
- batch page table allocation instead of allocating one page at a time
- pre-reserve pages for page tables needed for map/unmap operations,
  to ensure map/unmap operations don't try to allocate memory in paths
  they're allowed to block or fail

It might also be valuable for other aspects of GPU and similar
use-cases, like fine-grained memory accounting and resource limiting.

We will extend the Arm LPAE format to support custom allocators in a
separate commit.

Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Reviewed-by: Steven Price <steven.price@arm.com>
Reviewed-by: Robin Murphy <robin.murphy@arm.com>
Link: https://lore.kernel.org/r/20231124142434.1577550-2-boris.brezillon@collabora.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index b843fcd365d286668273667401b331a27fc04d23..8841c1487f00481f92759e499ad85a70423b7428 100644 (file)
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -34,6 +34,26 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
 #endif
 };
 
+static int check_custom_allocator(enum io_pgtable_fmt fmt,
+                                 struct io_pgtable_cfg *cfg)
+{
+       /* No custom allocator, no need to check the format. */
+       if (!cfg->alloc && !cfg->free)
+               return 0;
+
+       /* When passing a custom allocator, both the alloc and free
+        * functions should be provided.
+        */
+       if (!cfg->alloc || !cfg->free)
+               return -EINVAL;
+
+       /* Make sure the format supports custom allocators. */
+       if (io_pgtable_init_table[fmt]->caps & IO_PGTABLE_CAP_CUSTOM_ALLOCATOR)
+               return 0;
+
+       return -EINVAL;
+}
+
 struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
                                            struct io_pgtable_cfg *cfg,
                                            void *cookie)
@@ -44,6 +64,9 @@ struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
        if (fmt >= IO_PGTABLE_NUM_FMTS)
                return NULL;
 
+       if (check_custom_allocator(fmt, cfg))
+               return NULL;
+
        fns = io_pgtable_init_table[fmt];
        if (!fns)
                return NULL;

diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index 25142a0e2fc2c51d4c7807a1fb87cc21b16a163b..86cf1f7ae389a40180b86dd6850102f6fe04c188 100644 (file)
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -100,6 +100,30 @@ struct io_pgtable_cfg {
        const struct iommu_flush_ops    *tlb;
        struct device                   *iommu_dev;
 
+       /**
+        * @alloc: Custom page allocator.
+        *
+        * Optional hook used to allocate page tables. If this function is NULL,
+        * @free must be NULL too.
+        *
+        * Memory returned should be zeroed and suitable for dma_map_single() and
+        * virt_to_phys().
+        *
+        * Not all formats support custom page allocators. Before considering
+        * passing a non-NULL value, make sure the chosen page format supports
+        * this feature.
+        */
+       void *(*alloc)(void *cookie, size_t size, gfp_t gfp);
+
+       /**
+        * @free: Custom page de-allocator.
+        *
+        * Optional hook used to free page tables allocated with the @alloc
+        * hook. Must be non-NULL if @alloc is not NULL, must be NULL
+        * otherwise.
+        */
+       void (*free)(void *cookie, void *pages, size_t size);
+
        /* Low-level data specific to the table format */
        union {
                struct {
@@ -241,16 +265,26 @@ io_pgtable_tlb_add_page(struct io_pgtable *iop,
                iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie);
 }
 
+/**
+ * enum io_pgtable_caps - IO page table backend capabilities.
+ */
+enum io_pgtable_caps {
+       /** @IO_PGTABLE_CAP_CUSTOM_ALLOCATOR: Backend accepts custom page table allocators. */
+       IO_PGTABLE_CAP_CUSTOM_ALLOCATOR = BIT(0),
+};
+
 /**
  * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
  *                              particular format.
  *
  * @alloc: Allocate a set of page tables described by cfg.
  * @free:  Free the page tables associated with iop.
+ * @caps:  Combination of @io_pgtable_caps flags encoding the backend capabilities.
  */
 struct io_pgtable_init_fns {
        struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie);
        void (*free)(struct io_pgtable *iop);
+       u32 caps;
 };
 
 extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;