#include <linux/module.h>
#include <linux/string.h>
+#include <net/checksum.h>
#include <asm/byteorder.h>
+ #include <asm/checksum.h>
static inline unsigned short from64to16(unsigned long x)
{
#include <linux/compiler.h>
#include <linux/export.h>
#include <linux/preempt.h>
+ #include <asm/fpu.h>
#include <asm/thread_info.h>
+#include <asm/fpu.h>
#if defined(CONFIG_ALPHA_EV6) || defined(CONFIG_ALPHA_EV67)
#define STT(reg,val) asm volatile ("ftoit $f"#reg",%0" : "=r"(val));
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
-
-static inline pmd_t pmd_mkinvalid(pmd_t pmd)
-{
- pmd = set_pmd_bit(pmd, __pgprot(PMD_PRESENT_INVALID));
- pmd = clear_pmd_bit(pmd, __pgprot(PMD_SECT_VALID));
-
- return pmd;
-}
+#define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd)))
+#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
+#define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd))
+#define pmd_mkuffd_wp(pmd) pte_pmd(pte_mkuffd_wp(pmd_pte(pmd)))
+#define pmd_clear_uffd_wp(pmd) pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd)))
+#define pmd_swp_uffd_wp(pmd) pte_swp_uffd_wp(pmd_pte(pmd))
+#define pmd_swp_mkuffd_wp(pmd) pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd)))
+#define pmd_swp_clear_uffd_wp(pmd) \
+ pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd)))
+#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
- #define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
-
#define pmd_write(pmd) pte_write(pmd_pte(pmd))
#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
#include <linux/highmem.h>
#include <linux/suspend.h>
#include <linux/dma-direct.h>
+#include <linux/execmem.h>
+ #include <linux/vmalloc.h>
#include <asm/swiotlb.h>
#include <asm/machdep.h>
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2015 MediaTek Inc.
+ */
+
+#include <linux/dma-buf.h>
++#include <linux/vmalloc.h>
+
+#include <drm/drm.h>
+#include <drm/drm_device.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_prime.h>
+
+#include "mtk_drm_drv.h"
+#include "mtk_gem.h"
+
+static int mtk_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+
+static const struct vm_operations_struct vm_ops = {
+ .open = drm_gem_vm_open,
+ .close = drm_gem_vm_close,
+};
+
+static const struct drm_gem_object_funcs mtk_gem_object_funcs = {
+ .free = mtk_gem_free_object,
+ .get_sg_table = mtk_gem_prime_get_sg_table,
+ .vmap = mtk_gem_prime_vmap,
+ .vunmap = mtk_gem_prime_vunmap,
+ .mmap = mtk_gem_object_mmap,
+ .vm_ops = &vm_ops,
+};
+
+static struct mtk_gem_obj *mtk_gem_init(struct drm_device *dev,
+ unsigned long size)
+{
+ struct mtk_gem_obj *mtk_gem_obj;
+ int ret;
+
+ size = round_up(size, PAGE_SIZE);
+
+ if (size == 0)
+ return ERR_PTR(-EINVAL);
+
+ mtk_gem_obj = kzalloc(sizeof(*mtk_gem_obj), GFP_KERNEL);
+ if (!mtk_gem_obj)
+ return ERR_PTR(-ENOMEM);
+
+ mtk_gem_obj->base.funcs = &mtk_gem_object_funcs;
+
+ ret = drm_gem_object_init(dev, &mtk_gem_obj->base, size);
+ if (ret < 0) {
+ DRM_ERROR("failed to initialize gem object\n");
+ kfree(mtk_gem_obj);
+ return ERR_PTR(ret);
+ }
+
+ return mtk_gem_obj;
+}
+
+struct mtk_gem_obj *mtk_gem_create(struct drm_device *dev,
+ size_t size, bool alloc_kmap)
+{
+ struct mtk_drm_private *priv = dev->dev_private;
+ struct mtk_gem_obj *mtk_gem;
+ struct drm_gem_object *obj;
+ int ret;
+
+ mtk_gem = mtk_gem_init(dev, size);
+ if (IS_ERR(mtk_gem))
+ return ERR_CAST(mtk_gem);
+
+ obj = &mtk_gem->base;
+
+ mtk_gem->dma_attrs = DMA_ATTR_WRITE_COMBINE;
+
+ if (!alloc_kmap)
+ mtk_gem->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;
+
+ mtk_gem->cookie = dma_alloc_attrs(priv->dma_dev, obj->size,
+ &mtk_gem->dma_addr, GFP_KERNEL,
+ mtk_gem->dma_attrs);
+ if (!mtk_gem->cookie) {
+ DRM_ERROR("failed to allocate %zx byte dma buffer", obj->size);
+ ret = -ENOMEM;
+ goto err_gem_free;
+ }
+
+ if (alloc_kmap)
+ mtk_gem->kvaddr = mtk_gem->cookie;
+
+ DRM_DEBUG_DRIVER("cookie = %p dma_addr = %pad size = %zu\n",
+ mtk_gem->cookie, &mtk_gem->dma_addr,
+ size);
+
+ return mtk_gem;
+
+err_gem_free:
+ drm_gem_object_release(obj);
+ kfree(mtk_gem);
+ return ERR_PTR(ret);
+}
+
+void mtk_gem_free_object(struct drm_gem_object *obj)
+{
+ struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+ struct mtk_drm_private *priv = obj->dev->dev_private;
+
+ if (mtk_gem->sg)
+ drm_prime_gem_destroy(obj, mtk_gem->sg);
+ else
+ dma_free_attrs(priv->dma_dev, obj->size, mtk_gem->cookie,
+ mtk_gem->dma_addr, mtk_gem->dma_attrs);
+
+ /* release file pointer to gem object. */
+ drm_gem_object_release(obj);
+
+ kfree(mtk_gem);
+}
+
+int mtk_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
+ struct drm_mode_create_dumb *args)
+{
+ struct mtk_gem_obj *mtk_gem;
+ int ret;
+
+ args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+
+ /*
+ * Multiply 2 variables of different types,
+ * for example: args->size = args->spacing * args->height;
+ * may cause coverity issue with unintentional overflow.
+ */
+ args->size = args->pitch;
+ args->size *= args->height;
+
+ mtk_gem = mtk_gem_create(dev, args->size, false);
+ if (IS_ERR(mtk_gem))
+ return PTR_ERR(mtk_gem);
+
+ /*
+ * allocate a id of idr table where the obj is registered
+ * and handle has the id what user can see.
+ */
+ ret = drm_gem_handle_create(file_priv, &mtk_gem->base, &args->handle);
+ if (ret)
+ goto err_handle_create;
+
+ /* drop reference from allocate - handle holds it now. */
+ drm_gem_object_put(&mtk_gem->base);
+
+ return 0;
+
+err_handle_create:
+ mtk_gem_free_object(&mtk_gem->base);
+ return ret;
+}
+
+static int mtk_gem_object_mmap(struct drm_gem_object *obj,
+ struct vm_area_struct *vma)
+
+{
+ int ret;
+ struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+ struct mtk_drm_private *priv = obj->dev->dev_private;
+
+ /*
+ * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the
+ * whole buffer from the start.
+ */
+ vma->vm_pgoff = 0;
+
+ /*
+ * dma_alloc_attrs() allocated a struct page table for mtk_gem, so clear
+ * VM_PFNMAP flag that was set by drm_gem_mmap_obj()/drm_gem_mmap().
+ */
+ vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP);
+ vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+ vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+
+ ret = dma_mmap_attrs(priv->dma_dev, vma, mtk_gem->cookie,
+ mtk_gem->dma_addr, obj->size, mtk_gem->dma_attrs);
+
+ return ret;
+}
+
+/*
+ * Allocate a sg_table for this GEM object.
+ * Note: Both the table's contents, and the sg_table itself must be freed by
+ * the caller.
+ * Returns a pointer to the newly allocated sg_table, or an ERR_PTR() error.
+ */
+struct sg_table *mtk_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+ struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+ struct mtk_drm_private *priv = obj->dev->dev_private;
+ struct sg_table *sgt;
+ int ret;
+
+ sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+ if (!sgt)
+ return ERR_PTR(-ENOMEM);
+
+ ret = dma_get_sgtable_attrs(priv->dma_dev, sgt, mtk_gem->cookie,
+ mtk_gem->dma_addr, obj->size,
+ mtk_gem->dma_attrs);
+ if (ret) {
+ DRM_ERROR("failed to allocate sgt, %d\n", ret);
+ kfree(sgt);
+ return ERR_PTR(ret);
+ }
+
+ return sgt;
+}
+
+struct drm_gem_object *mtk_gem_prime_import_sg_table(struct drm_device *dev,
+ struct dma_buf_attachment *attach, struct sg_table *sg)
+{
+ struct mtk_gem_obj *mtk_gem;
+
+ /* check if the entries in the sg_table are contiguous */
+ if (drm_prime_get_contiguous_size(sg) < attach->dmabuf->size) {
+ DRM_ERROR("sg_table is not contiguous");
+ return ERR_PTR(-EINVAL);
+ }
+
+ mtk_gem = mtk_gem_init(dev, attach->dmabuf->size);
+ if (IS_ERR(mtk_gem))
+ return ERR_CAST(mtk_gem);
+
+ mtk_gem->dma_addr = sg_dma_address(sg->sgl);
+ mtk_gem->sg = sg;
+
+ return &mtk_gem->base;
+}
+
+int mtk_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
+{
+ struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+ struct sg_table *sgt = NULL;
+ unsigned int npages;
+
+ if (mtk_gem->kvaddr)
+ goto out;
+
+ sgt = mtk_gem_prime_get_sg_table(obj);
+ if (IS_ERR(sgt))
+ return PTR_ERR(sgt);
+
+ npages = obj->size >> PAGE_SHIFT;
+ mtk_gem->pages = kcalloc(npages, sizeof(*mtk_gem->pages), GFP_KERNEL);
+ if (!mtk_gem->pages) {
+ sg_free_table(sgt);
+ kfree(sgt);
+ return -ENOMEM;
+ }
+
+ drm_prime_sg_to_page_array(sgt, mtk_gem->pages, npages);
+
+ mtk_gem->kvaddr = vmap(mtk_gem->pages, npages, VM_MAP,
+ pgprot_writecombine(PAGE_KERNEL));
+ if (!mtk_gem->kvaddr) {
+ sg_free_table(sgt);
+ kfree(sgt);
+ kfree(mtk_gem->pages);
+ return -ENOMEM;
+ }
+ sg_free_table(sgt);
+ kfree(sgt);
+
+out:
+ iosys_map_set_vaddr(map, mtk_gem->kvaddr);
+
+ return 0;
+}
+
+void mtk_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
+{
+ struct mtk_gem_obj *mtk_gem = to_mtk_gem_obj(obj);
+ void *vaddr = map->vaddr;
+
+ if (!mtk_gem->pages)
+ return;
+
+ vunmap(vaddr);
+ mtk_gem->kvaddr = NULL;
+ kfree(mtk_gem->pages);
+}
if (__compiletime_lessthan(p_size, size))
__read_overflow();
if (p_size < size)
- fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL);
+ fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size,
+ __real_kmemdup(p, 0, gfp));
return __real_kmemdup(p, size, gfp);
}
+ #define kmemdup(...) alloc_hooks(kmemdup_noprof(__VA_ARGS__))
/**
* strcpy - Copy a string into another string buffer
* @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc).
*/
- static inline __alloc_size(1) void *kzalloc(size_t size, gfp_t flags)
+ static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
{
- return kmalloc(size, flags | __GFP_ZERO);
+ return kmalloc_noprof(size, flags | __GFP_ZERO);
}
+ #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
+ #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
- /**
- * kzalloc_node - allocate zeroed memory from a particular memory node.
- * @size: how many bytes of memory are required.
- * @flags: the type of memory to allocate (see kmalloc).
- * @node: memory node from which to allocate
- */
- static inline __alloc_size(1) void *kzalloc_node(size_t size, gfp_t flags, int node)
- {
- return kmalloc_node(size, flags | __GFP_ZERO, node);
- }
+ extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_size(1);
+ #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))
- extern void *kvmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1);
- static inline __alloc_size(1) void *kvmalloc(size_t size, gfp_t flags)
- {
- return kvmalloc_node(size, flags, NUMA_NO_NODE);
- }
- static inline __alloc_size(1) void *kvzalloc_node(size_t size, gfp_t flags, int node)
- {
- return kvmalloc_node(size, flags | __GFP_ZERO, node);
- }
- static inline __alloc_size(1) void *kvzalloc(size_t size, gfp_t flags)
- {
- return kvmalloc(size, flags | __GFP_ZERO);
- }
+ #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE)
+ #define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE)
-#define kvzalloc(_size, _flags) kvmalloc(_size, _flags|__GFP_ZERO)
++#define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO)
+
-#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, _flags|__GFP_ZERO, _node)
++#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
-static inline __alloc_size(1, 2) void *kvmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
+static inline __alloc_size(1, 2) void *
- kvmalloc_array_node(size_t n, size_t size, gfp_t flags, int node)
++kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
{
size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL;
- return kvmalloc_node(bytes, flags, node);
- return kvmalloc_node_noprof(bytes, flags, NUMA_NO_NODE);
++ return kvmalloc_node_noprof(bytes, flags, node);
}
- static inline __alloc_size(1, 2) void *
- kvmalloc_array(size_t n, size_t size, gfp_t flags)
- {
- return kvmalloc_array_node(n, size, flags, NUMA_NO_NODE);
- }
-
- static inline __alloc_size(1, 2) void *
- kvcalloc_node(size_t n, size_t size, gfp_t flags, int node)
- {
- return kvmalloc_array_node(n, size, flags | __GFP_ZERO, node);
- }
++#define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
++#define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node)
++#define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
+
- static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags)
- {
- return kvmalloc_array(n, size, flags | __GFP_ZERO);
- }
+ #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))
-#define kvcalloc(_n, _size, _flags) kvmalloc_array(_n, _size, _flags|__GFP_ZERO)
-#define kvcalloc_noprof(_n, _size, _flags) kvmalloc_array_noprof(_n, _size, _flags|__GFP_ZERO)
++#define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))
++#define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__))
- extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
+ extern void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
__realloc_size(3);
+ #define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__))
+
extern void kvfree(const void *addr);
-DEFINE_FREE(kvfree, void *, if (_T) kvfree(_T))
+DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T))
extern void kvfree_sensitive(const void *addr, size_t len);
extern char *kstrdup(const char *s, gfp_t gfp) __malloc;
extern const char *kstrdup_const(const char *s, gfp_t gfp);
extern char *kstrndup(const char *s, size_t len, gfp_t gfp);
- extern void *kmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2);
+ extern void *kmemdup_noprof(const void *src, size_t len, gfp_t gfp) __realloc_size(2);
+ #define kmemdup(...) alloc_hooks(kmemdup_noprof(__VA_ARGS__))
+
extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2);
extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp);
-extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp);
+extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp)
+ __realloc_size(2, 3);
/* lib/argv_split.c */
extern char **argv_split(gfp_t gfp, const char *str, int *argcp);
--- /dev/null
- return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/io_uring.h>
+#include <linux/io_uring_types.h>
+#include <asm/shmparam.h>
+
+#include "memmap.h"
+#include "kbuf.h"
+
+static void *io_mem_alloc_compound(struct page **pages, int nr_pages,
+ size_t size, gfp_t gfp)
+{
+ struct page *page;
+ int i, order;
+
+ order = get_order(size);
+ if (order > MAX_PAGE_ORDER)
+ return ERR_PTR(-ENOMEM);
+ else if (order)
+ gfp |= __GFP_COMP;
+
+ page = alloc_pages(gfp, order);
+ if (!page)
+ return ERR_PTR(-ENOMEM);
+
+ for (i = 0; i < nr_pages; i++)
+ pages[i] = page + i;
+
+ return page_address(page);
+}
+
+static void *io_mem_alloc_single(struct page **pages, int nr_pages, size_t size,
+ gfp_t gfp)
+{
+ void *ret;
+ int i;
+
+ for (i = 0; i < nr_pages; i++) {
+ pages[i] = alloc_page(gfp);
+ if (!pages[i])
+ goto err;
+ }
+
+ ret = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
+ if (ret)
+ return ret;
+err:
+ while (i--)
+ put_page(pages[i]);
+ return ERR_PTR(-ENOMEM);
+}
+
+void *io_pages_map(struct page ***out_pages, unsigned short *npages,
+ size_t size)
+{
+ gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
+ struct page **pages;
+ int nr_pages;
+ void *ret;
+
+ nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ pages = kvmalloc_array(nr_pages, sizeof(struct page *), gfp);
+ if (!pages)
+ return ERR_PTR(-ENOMEM);
+
+ ret = io_mem_alloc_compound(pages, nr_pages, size, gfp);
+ if (!IS_ERR(ret))
+ goto done;
+
+ ret = io_mem_alloc_single(pages, nr_pages, size, gfp);
+ if (!IS_ERR(ret)) {
+done:
+ *out_pages = pages;
+ *npages = nr_pages;
+ return ret;
+ }
+
+ kvfree(pages);
+ *out_pages = NULL;
+ *npages = 0;
+ return ret;
+}
+
+void io_pages_unmap(void *ptr, struct page ***pages, unsigned short *npages,
+ bool put_pages)
+{
+ bool do_vunmap = false;
+
+ if (!ptr)
+ return;
+
+ if (put_pages && *npages) {
+ struct page **to_free = *pages;
+ int i;
+
+ /*
+ * Only did vmap for the non-compound multiple page case.
+ * For the compound page, we just need to put the head.
+ */
+ if (PageCompound(to_free[0]))
+ *npages = 1;
+ else if (*npages > 1)
+ do_vunmap = true;
+ for (i = 0; i < *npages; i++)
+ put_page(to_free[i]);
+ }
+ if (do_vunmap)
+ vunmap(ptr);
+ kvfree(*pages);
+ *pages = NULL;
+ *npages = 0;
+}
+
+void io_pages_free(struct page ***pages, int npages)
+{
+ struct page **page_array = *pages;
+
+ if (!page_array)
+ return;
+
+ unpin_user_pages(page_array, npages);
+ kvfree(page_array);
+ *pages = NULL;
+}
+
+struct page **io_pin_pages(unsigned long uaddr, unsigned long len, int *npages)
+{
+ unsigned long start, end, nr_pages;
+ struct page **pages;
+ int ret;
+
+ end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ start = uaddr >> PAGE_SHIFT;
+ nr_pages = end - start;
+ if (WARN_ON_ONCE(!nr_pages))
+ return ERR_PTR(-EINVAL);
+
+ pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
+ if (!pages)
+ return ERR_PTR(-ENOMEM);
+
+ ret = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
+ pages);
+ /* success, mapped all pages */
+ if (ret == nr_pages) {
+ *npages = nr_pages;
+ return pages;
+ }
+
+ /* partial map, or didn't map anything */
+ if (ret >= 0) {
+ /* if we did partial map, release any pages we did get */
+ if (ret)
+ unpin_user_pages(pages, ret);
+ ret = -EFAULT;
+ }
+ kvfree(pages);
+ return ERR_PTR(ret);
+}
+
+void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
+ unsigned long uaddr, size_t size)
+{
+ struct page **page_array;
+ unsigned int nr_pages;
+ void *page_addr;
+
+ *npages = 0;
+
+ if (uaddr & (PAGE_SIZE - 1) || !size)
+ return ERR_PTR(-EINVAL);
+
+ nr_pages = 0;
+ page_array = io_pin_pages(uaddr, size, &nr_pages);
+ if (IS_ERR(page_array))
+ return page_array;
+
+ page_addr = vmap(page_array, nr_pages, VM_MAP, PAGE_KERNEL);
+ if (page_addr) {
+ *pages = page_array;
+ *npages = nr_pages;
+ return page_addr;
+ }
+
+ io_pages_free(&page_array, nr_pages);
+ return ERR_PTR(-ENOMEM);
+}
+
+static void *io_uring_validate_mmap_request(struct file *file, loff_t pgoff,
+ size_t sz)
+{
+ struct io_ring_ctx *ctx = file->private_data;
+ loff_t offset = pgoff << PAGE_SHIFT;
+
+ switch ((pgoff << PAGE_SHIFT) & IORING_OFF_MMAP_MASK) {
+ case IORING_OFF_SQ_RING:
+ case IORING_OFF_CQ_RING:
+ /* Don't allow mmap if the ring was setup without it */
+ if (ctx->flags & IORING_SETUP_NO_MMAP)
+ return ERR_PTR(-EINVAL);
+ return ctx->rings;
+ case IORING_OFF_SQES:
+ /* Don't allow mmap if the ring was setup without it */
+ if (ctx->flags & IORING_SETUP_NO_MMAP)
+ return ERR_PTR(-EINVAL);
+ return ctx->sq_sqes;
+ case IORING_OFF_PBUF_RING: {
+ struct io_buffer_list *bl;
+ unsigned int bgid;
+ void *ptr;
+
+ bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
+ bl = io_pbuf_get_bl(ctx, bgid);
+ if (IS_ERR(bl))
+ return bl;
+ ptr = bl->buf_ring;
+ io_put_bl(ctx, bl);
+ return ptr;
+ }
+ }
+
+ return ERR_PTR(-EINVAL);
+}
+
+int io_uring_mmap_pages(struct io_ring_ctx *ctx, struct vm_area_struct *vma,
+ struct page **pages, int npages)
+{
+ unsigned long nr_pages = npages;
+
+ vm_flags_set(vma, VM_DONTEXPAND);
+ return vm_insert_pages(vma, vma->vm_start, pages, &nr_pages);
+}
+
+#ifdef CONFIG_MMU
+
+__cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct io_ring_ctx *ctx = file->private_data;
+ size_t sz = vma->vm_end - vma->vm_start;
+ long offset = vma->vm_pgoff << PAGE_SHIFT;
+ void *ptr;
+
+ ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
+ if (IS_ERR(ptr))
+ return PTR_ERR(ptr);
+
+ switch (offset & IORING_OFF_MMAP_MASK) {
+ case IORING_OFF_SQ_RING:
+ case IORING_OFF_CQ_RING:
+ return io_uring_mmap_pages(ctx, vma, ctx->ring_pages,
+ ctx->n_ring_pages);
+ case IORING_OFF_SQES:
+ return io_uring_mmap_pages(ctx, vma, ctx->sqe_pages,
+ ctx->n_sqe_pages);
+ case IORING_OFF_PBUF_RING:
+ return io_pbuf_mmap(file, vma);
+ }
+
+ return -EINVAL;
+}
+
+unsigned long io_uring_get_unmapped_area(struct file *filp, unsigned long addr,
+ unsigned long len, unsigned long pgoff,
+ unsigned long flags)
+{
+ void *ptr;
+
+ /*
+ * Do not allow to map to user-provided address to avoid breaking the
+ * aliasing rules. Userspace is not able to guess the offset address of
+ * kernel kmalloc()ed memory area.
+ */
+ if (addr)
+ return -EINVAL;
+
+ ptr = io_uring_validate_mmap_request(filp, pgoff, len);
+ if (IS_ERR(ptr))
+ return -ENOMEM;
+
+ /*
+ * Some architectures have strong cache aliasing requirements.
+ * For such architectures we need a coherent mapping which aliases
+ * kernel memory *and* userspace memory. To achieve that:
+ * - use a NULL file pointer to reference physical memory, and
+ * - use the kernel virtual address of the shared io_uring context
+ * (instead of the userspace-provided address, which has to be 0UL
+ * anyway).
+ * - use the same pgoff which the get_unmapped_area() uses to
+ * calculate the page colouring.
+ * For architectures without such aliasing requirements, the
+ * architecture will return any suitable mapping because addr is 0.
+ */
+ filp = NULL;
+ flags |= MAP_SHARED;
+ pgoff = 0; /* has been translated to ptr above */
+#ifdef SHM_COLOUR
+ addr = (uintptr_t) ptr;
+ pgoff = addr >> PAGE_SHIFT;
+#else
+ addr = 0UL;
+#endif
++ return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags);
+}
+
+#else /* !CONFIG_MMU */
+
+int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ return is_nommu_shared_mapping(vma->vm_flags) ? 0 : -EINVAL;
+}
+
+unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
+{
+ return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
+}
+
+unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr,
+ unsigned long len, unsigned long pgoff,
+ unsigned long flags)
+{
+ void *ptr;
+
+ ptr = io_uring_validate_mmap_request(file, pgoff, len);
+ if (IS_ERR(ptr))
+ return PTR_ERR(ptr);
+
+ return (unsigned long) ptr;
+}
+
+#endif /* !CONFIG_MMU */
#include <linux/dynamic_debug.h>
#include <linux/audit.h>
#include <linux/cfi.h>
+ #include <linux/codetag.h>
#include <linux/debugfs.h>
+#include <linux/execmem.h>
#include <uapi/linux/module.h>
#include "internal.h"
{
}
-static bool mod_mem_use_vmalloc(enum mod_mem_type type)
+static int module_memory_alloc(struct module *mod, enum mod_mem_type type)
{
- return IS_ENABLED(CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC) &&
- mod_mem_type_is_core_data(type);
-}
+ unsigned int size = PAGE_ALIGN(mod->mem[type].size);
+ enum execmem_type execmem_type;
+ void *ptr;
-static void *module_memory_alloc(unsigned int size, enum mod_mem_type type)
-{
- if (mod_mem_use_vmalloc(type))
- return vzalloc(size);
- return module_alloc(size);
+ mod->mem[type].size = size;
+
+ if (mod_mem_type_is_data(type))
+ execmem_type = EXECMEM_MODULE_DATA;
+ else
+ execmem_type = EXECMEM_MODULE_TEXT;
+
+ ptr = execmem_alloc(execmem_type, size);
+ if (!ptr)
+ return -ENOMEM;
+
+ /*
+ * The pointer to these blocks of memory are stored on the module
+ * structure and we keep that around so long as the module is
+ * around. We only free that memory when we unload the module.
+ * Just mark them as not being a leak then. The .init* ELF
+ * sections *do* get freed after boot so we *could* treat them
+ * slightly differently with kmemleak_ignore() and only grey
+ * them out as they work as typical memory allocations which
+ * *do* eventually get freed, but let's just keep things simple
+ * and avoid *any* false positives.
+ */
+ kmemleak_not_leak(ptr);
+
+ memset(ptr, 0, size);
+ mod->mem[type].base = ptr;
+
+ return 0;
}
- static void module_memory_free(struct module *mod, enum mod_mem_type type)
-static void module_memory_free(void *ptr, enum mod_mem_type type,
++static void module_memory_free(struct module *mod, enum mod_mem_type type,
+ bool unload_codetags)
{
- if (mod_mem_use_vmalloc(type))
- vfree(ptr);
- else
- module_memfree(ptr);
+ void *ptr = mod->mem[type].base;
+
+ if (!unload_codetags && mod_mem_type_is_core_data(type))
+ return;
+
+ execmem_free(ptr);
}
- static void free_mod_mem(struct module *mod)
+ static void free_mod_mem(struct module *mod, bool unload_codetags)
{
for_each_mod_mem_type(type) {
struct module_memory *mod_mem = &mod->mem[type];
/* Free lock-classes; relies on the preceding sync_rcu(). */
lockdep_free_key_range(mod_mem->base, mod_mem->size);
if (mod_mem->size)
- module_memory_free(mod, type);
- module_memory_free(mod_mem->base, type,
- unload_codetags);
++ module_memory_free(mod, type, unload_codetags);
}
/* MOD_DATA hosts mod, so free it at last */
lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
- module_memory_free(mod, MOD_DATA);
- module_memory_free(mod->mem[MOD_DATA].base, MOD_DATA, unload_codetags);
++ module_memory_free(mod, MOD_DATA, unload_codetags);
}
/* Free a module, remove from lists, etc. */
return 0;
out_enomem:
for (t--; t >= 0; t--)
- module_memory_free(mod, t);
- module_memory_free(mod->mem[t].base, t, true);
++ module_memory_free(mod, t, true);
return ret;
}
selfdir = $(realpath $(dir $(filter %/lib.mk,$(MAKEFILE_LIST))))
top_srcdir = $(selfdir)/../../..
+# msg: emit succinct information message describing current building step
+# $1 - generic step name (e.g., CC, LINK, etc);
+# $2 - optional "flavor" specifier; if provided, will be emitted as [flavor];
+# $3 - target (assumed to be file); only file name will be emitted;
+# $4 - optional extra arg, emitted as-is, if provided.
+ifeq ($(V),1)
+Q =
+msg =
+else
+Q = @
+msg = @printf ' %-8s%s %s%s\n' "$(1)" "$(if $(2), [$(2)])" "$(notdir $(3))" "$(if $(4), $(4))";
+MAKEFLAGS += --no-print-directory
+endif
+
ifeq ($(KHDR_INCLUDES),)
-KHDR_INCLUDES := -isystem $(top_srcdir)/usr/include
+KHDR_INCLUDES := -D_GNU_SOURCE -isystem $(top_srcdir)/usr/include
endif
+ # In order to use newer items that haven't yet been added to the user's system
+ # header files, add $(TOOLS_INCLUDES) to the compiler invocation in each
+ # each selftest.
+ # You may need to add files to that location, or to refresh an existing file. In
+ # order to do that, run "make headers" from $(top_srcdir), then copy the
+ # header file that you want from $(top_srcdir)/usr/include/... , to the matching
+ # subdir in $(TOOLS_INCLUDE).
+ TOOLS_INCLUDES := -isystem $(top_srcdir)/tools/include/uapi
+
# The following are built by lib.mk common compile rules.
# TEST_CUSTOM_PROGS should be used by tests that require
# custom build rule and prevent common build rule use.
projects / linux-2.6-block.git / commitdiff