xe_sync.o \
xe_trace.o \
xe_ttm_gtt_mgr.o \
+ xe_ttm_stolen_mgr.o \
xe_ttm_vram_mgr.o \
xe_tuning.o \
xe_uc.o \
#include "xe_preempt_fence.h"
#include "xe_res_cursor.h"
#include "xe_trace.h"
+#include "xe_ttm_stolen_mgr.h"
#include "xe_vm.h"
static const struct ttm_place sys_placement_flags = {
bool mem_type_is_vram(u32 mem_type)
{
- return mem_type >= XE_PL_VRAM0;
+ return mem_type >= XE_PL_VRAM0 && mem_type != XE_PL_STOLEN;
+}
+
+static bool resource_is_stolen_vram(struct xe_device *xe, struct ttm_resource *res)
+{
+ return res->mem_type == XE_PL_STOLEN && IS_DGFX(xe);
}
static bool resource_is_vram(struct ttm_resource *res)
bool xe_bo_is_vram(struct xe_bo *bo)
{
- return resource_is_vram(bo->ttm.resource);
+ return resource_is_vram(bo->ttm.resource) ||
+ resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);
+}
+
+bool xe_bo_is_stolen(struct xe_bo *bo)
+{
+ return bo->ttm.resource->mem_type == XE_PL_STOLEN;
}
static bool xe_bo_is_user(struct xe_bo *bo)
static struct xe_gt *
mem_type_to_gt(struct xe_device *xe, u32 mem_type)
{
- XE_BUG_ON(!mem_type_is_vram(mem_type));
+ XE_BUG_ON(mem_type != XE_PL_STOLEN && !mem_type_is_vram(mem_type));
- return xe_device_get_gt(xe, mem_type - XE_PL_VRAM0);
+ return xe_device_get_gt(xe, mem_type == XE_PL_STOLEN ? 0 : (mem_type - XE_PL_VRAM0));
}
static void try_add_system(struct xe_bo *bo, struct ttm_place *places,
}
}
+static void try_add_stolen(struct xe_device *xe, struct xe_bo *bo,
+ struct ttm_place *places, u32 bo_flags, u32 *c)
+{
+ if (bo_flags & XE_BO_CREATE_STOLEN_BIT) {
+ places[*c] = (struct ttm_place) {
+ .mem_type = XE_PL_STOLEN,
+ .flags = bo_flags & (XE_BO_CREATE_PINNED_BIT |
+ XE_BO_CREATE_GGTT_BIT) ?
+ TTM_PL_FLAG_CONTIGUOUS : 0,
+ };
+ *c += 1;
+ }
+}
+
static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo,
u32 bo_flags)
{
try_add_vram1(xe, bo, places, bo_flags, &c);
try_add_system(bo, places, bo_flags, &c);
}
+ try_add_stolen(xe, bo, places, bo_flags, &c);
if (!c)
return -EINVAL;
switch (tbo->resource->mem_type) {
case XE_PL_VRAM0:
case XE_PL_VRAM1:
+ case XE_PL_STOLEN:
case XE_PL_TT:
default:
/* for now kick out to system */
#if !defined(CONFIG_X86)
mem->bus.caching = ttm_write_combined;
#endif
- break;
+ return 0;
+ case XE_PL_STOLEN:
+ return xe_ttm_stolen_io_mem_reserve(xe, mem);
default:
return -EINVAL;
}
- return 0;
}
static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
}
-static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
+static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,
unsigned long page_offset)
{
- struct xe_device *xe = ttm_to_xe_device(bo->bdev);
- struct xe_gt *gt = mem_type_to_gt(xe, bo->resource->mem_type);
+ struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev);
+ struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
+ struct xe_gt *gt = mem_type_to_gt(xe, ttm_bo->resource->mem_type);
struct xe_res_cursor cursor;
- xe_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);
+ if (ttm_bo->resource->mem_type == XE_PL_STOLEN)
+ return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;
+
+ xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);
return (gt->mem.vram.io_start + cursor.start) >> PAGE_SHIFT;
}
return bo;
}
- if (flags & (XE_BO_CREATE_VRAM0_BIT | XE_BO_CREATE_VRAM1_BIT) &&
+ if (flags & (XE_BO_CREATE_VRAM0_BIT | XE_BO_CREATE_VRAM1_BIT |
+ XE_BO_CREATE_STOLEN_BIT) &&
!(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) &&
xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) {
size = ALIGN(size, SZ_64K);
ctx.resv = resv;
}
- err = __xe_bo_placement_for_flags(xe, bo, bo->flags);
- if (WARN_ON(err))
- return ERR_PTR(err);
+ if (!(flags & XE_BO_FIXED_PLACEMENT_BIT)) {
+ err = __xe_bo_placement_for_flags(xe, bo, bo->flags);
+ if (WARN_ON(err))
+ return ERR_PTR(err);
+ }
/* Defer populating type_sg bos */
placement = (type == ttm_bo_type_sg ||
return bo;
}
-struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt,
- struct xe_vm *vm, size_t size,
- enum ttm_bo_type type, u32 flags)
+static int __xe_bo_fixed_placement(struct xe_device *xe,
+ struct xe_bo *bo,
+ u32 flags,
+ u64 start, u64 end, u64 size)
{
- struct xe_bo *bo;
+ struct ttm_place *place = bo->placements;
+
+ if (flags & (XE_BO_CREATE_USER_BIT|XE_BO_CREATE_SYSTEM_BIT))
+ return -EINVAL;
+
+ place->flags = TTM_PL_FLAG_CONTIGUOUS;
+ place->fpfn = start >> PAGE_SHIFT;
+ place->lpfn = end >> PAGE_SHIFT;
+
+ switch (flags & (XE_BO_CREATE_STOLEN_BIT |
+ XE_BO_CREATE_VRAM0_BIT |XE_BO_CREATE_VRAM1_BIT)) {
+ case XE_BO_CREATE_VRAM0_BIT:
+ place->mem_type = XE_PL_VRAM0;
+ break;
+ case XE_BO_CREATE_VRAM1_BIT:
+ place->mem_type = XE_PL_VRAM1;
+ break;
+ case XE_BO_CREATE_STOLEN_BIT:
+ place->mem_type = XE_PL_STOLEN;
+ break;
+
+ default:
+ /* 0 or multiple of the above set */
+ return -EINVAL;
+ }
+
+ bo->placement = (struct ttm_placement) {
+ .num_placement = 1,
+ .placement = place,
+ .num_busy_placement = 1,
+ .busy_placement = place,
+ };
+
+ return 0;
+}
+
+struct xe_bo *
+xe_bo_create_locked_range(struct xe_device *xe,
+ struct xe_gt *gt, struct xe_vm *vm,
+ size_t size, u64 start, u64 end,
+ enum ttm_bo_type type, u32 flags)
+{
+ struct xe_bo *bo = NULL;
int err;
if (vm)
xe_vm_assert_held(vm);
- bo = __xe_bo_create_locked(xe, NULL, gt, vm ? &vm->resv : NULL, size,
+
+ if (start || end != ~0ULL) {
+ bo = xe_bo_alloc();
+ if (IS_ERR(bo))
+ return bo;
+
+ flags |= XE_BO_FIXED_PLACEMENT_BIT;
+ err = __xe_bo_fixed_placement(xe, bo, flags, start, end, size);
+ if (err) {
+ xe_bo_free(bo);
+ return ERR_PTR(err);
+ }
+ }
+
+ bo = __xe_bo_create_locked(xe, bo, gt, vm ? &vm->resv : NULL, size,
type, flags);
if (IS_ERR(bo))
return bo;
xe_vm_get(vm);
bo->vm = vm;
- if (flags & XE_BO_CREATE_GGTT_BIT) {
+ if (bo->flags & XE_BO_CREATE_GGTT_BIT) {
+ if (!gt && flags & XE_BO_CREATE_STOLEN_BIT)
+ gt = xe_device_get_gt(xe, 0);
+
XE_BUG_ON(!gt);
err = xe_ggtt_insert_bo(gt->mem.ggtt, bo);
return ERR_PTR(err);
}
+struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt,
+ struct xe_vm *vm, size_t size,
+ enum ttm_bo_type type, u32 flags)
+{
+ return xe_bo_create_locked_range(xe, gt, vm, size, 0, ~0ULL, type, flags);
+}
+
struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt,
struct xe_vm *vm, size_t size,
enum ttm_bo_type type, u32 flags)
return bo;
}
-struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt,
- struct xe_vm *vm, size_t size,
- enum ttm_bo_type type, u32 flags)
+struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt,
+ struct xe_vm *vm,
+ size_t size, u64 offset,
+ enum ttm_bo_type type, u32 flags)
{
- struct xe_bo *bo = xe_bo_create_locked(xe, gt, vm, size, type, flags);
+ struct xe_bo *bo;
int err;
+ u64 start = offset == ~0ull ? 0 : offset;
+ u64 end = offset == ~0ull ? offset : start + size;
+
+ if (flags & XE_BO_CREATE_STOLEN_BIT &&
+ xe_ttm_stolen_inaccessible(xe))
+ flags |= XE_BO_CREATE_GGTT_BIT;
+ bo = xe_bo_create_locked_range(xe, gt, vm, size, start, end, type, flags);
if (IS_ERR(bo))
return bo;
return ERR_PTR(err);
}
+struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt,
+ struct xe_vm *vm, size_t size,
+ enum ttm_bo_type type, u32 flags)
+{
+ return xe_bo_create_pin_map_at(xe, gt, vm, size, ~0ull, type, flags);
+}
+
struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt,
const void *data, size_t size,
enum ttm_bo_type type, u32 flags)
struct xe_device *xe = xe_bo_device(bo);
struct xe_gt *gt = mem_type_to_gt(xe, bo->ttm.resource->mem_type);
+ if (bo->ttm.resource->mem_type == XE_PL_STOLEN)
+ return xe_ttm_stolen_gpu_offset(xe);
+
return gt->mem.vram.io_start - xe->mem.vram.io_start;
}
bool lmem;
XE_BUG_ON(!(place->flags & TTM_PL_FLAG_CONTIGUOUS));
- XE_BUG_ON(!mem_type_is_vram(place->mem_type));
+ XE_BUG_ON(!mem_type_is_vram(place->mem_type) && place->mem_type != XE_PL_STOLEN);
place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE, &lmem) -
vram_region_io_offset(bo)) >> PAGE_SHIFT;
*is_lmem = xe_bo_is_vram(bo);
- if (!*is_lmem) {
+ if (!*is_lmem && !xe_bo_is_stolen(bo)) {
XE_BUG_ON(!bo->ttm.ttm);
xe_res_first_sg(xe_bo_get_sg(bo), page << PAGE_SHIFT,
#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */
-#define XE_BO_CREATE_USER_BIT BIT(1)
-#define XE_BO_CREATE_SYSTEM_BIT BIT(2)
+#define XE_BO_CREATE_USER_BIT BIT(0)
+#define XE_BO_CREATE_SYSTEM_BIT BIT(1)
+#define XE_BO_CREATE_STOLEN_BIT BIT(2)
#define XE_BO_CREATE_VRAM0_BIT BIT(3)
#define XE_BO_CREATE_VRAM1_BIT BIT(4)
#define XE_BO_CREATE_VRAM_IF_DGFX(gt) \
#define XE_BO_CREATE_PINNED_BIT BIT(7)
#define XE_BO_DEFER_BACKING BIT(8)
#define XE_BO_SCANOUT_BIT BIT(9)
+#define XE_BO_FIXED_PLACEMENT_BIT BIT(10)
/* this one is trigger internally only */
#define XE_BO_INTERNAL_TEST BIT(30)
#define XE_BO_INTERNAL_64K BIT(31)
#define XE_PL_TT TTM_PL_TT
#define XE_PL_VRAM0 TTM_PL_VRAM
#define XE_PL_VRAM1 (XE_PL_VRAM0 + 1)
+#define XE_PL_STOLEN (TTM_NUM_MEM_TYPES - 1)
#define XE_BO_PROPS_INVALID (-1)
struct xe_gt *gt, struct dma_resv *resv,
size_t size, enum ttm_bo_type type,
u32 flags);
+struct xe_bo *
+xe_bo_create_locked_range(struct xe_device *xe,
+ struct xe_gt *gt, struct xe_vm *vm,
+ size_t size, u64 start, u64 end,
+ enum ttm_bo_type type, u32 flags);
struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt,
struct xe_vm *vm, size_t size,
enum ttm_bo_type type, u32 flags);
struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt,
struct xe_vm *vm, size_t size,
enum ttm_bo_type type, u32 flags);
+struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt,
+ struct xe_vm *vm, size_t size, u64 offset,
+ enum ttm_bo_type type, u32 flags);
struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt,
const void *data, size_t size,
enum ttm_bo_type type, u32 flags);
bool mem_type_is_vram(u32 mem_type);
bool xe_bo_is_vram(struct xe_bo *bo);
+bool xe_bo_is_stolen(struct xe_bo *bo);
bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type);
man = ttm_manager_type(bdev, XE_PL_TT);
ttm_resource_manager_create_debugfs(man, root, "gtt_mm");
+ man = ttm_manager_type(bdev, XE_PL_STOLEN);
+ if (man)
+ ttm_resource_manager_create_debugfs(man, root, "stolen_mm");
+
for_each_gt(gt, xe, id)
xe_gt_debugfs_register(gt);
}
#include "xe_pcode.h"
#include "xe_pm.h"
#include "xe_query.h"
+#include "xe_ttm_stolen_mgr.h"
#include "xe_vm.h"
#include "xe_vm_madvise.h"
#include "xe_wait_user_fence.h"
goto err_irq_shutdown;
}
+ /* Allocate and map stolen after potential VRAM resize */
+ xe_ttm_stolen_mgr_init(xe);
+
for_each_gt(gt, xe, id) {
err = xe_gt_init(gt);
if (err)
return true;
}
+int xe_mmio_total_vram_size(struct xe_device *xe, u64 *vram_size, u64 *flat_ccs_base)
+{
+ struct xe_gt *gt = xe_device_get_gt(xe, 0);
+ struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+ int err;
+ u32 reg;
+
+ if (!xe->info.has_flat_ccs) {
+ *vram_size = pci_resource_len(pdev, GEN12_LMEM_BAR);
+ if (flat_ccs_base)
+ *flat_ccs_base = *vram_size;
+ return 0;
+ }
+
+ err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
+ if (err)
+ return err;
+
+ reg = xe_gt_mcr_unicast_read_any(gt, XEHP_TILE0_ADDR_RANGE);
+ *vram_size = (u64)REG_FIELD_GET(GENMASK(14, 8), reg) * SZ_1G;
+ if (flat_ccs_base) {
+ reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR);
+ *flat_ccs_base = (u64)REG_FIELD_GET(GENMASK(31, 8), reg) * SZ_64K;
+ }
+
+ if (flat_ccs_base)
+ drm_info(&xe->drm, "lmem_size: 0x%llx flat_ccs_base: 0x%llx\n",
+ *vram_size, *flat_ccs_base);
+
+ return xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
+}
+
int xe_mmio_probe_vram(struct xe_device *xe)
{
struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
u64 original_size;
u64 current_size;
u64 flat_ccs_base;
- int resize_result;
+ int resize_result, err;
if (!IS_DGFX(xe)) {
xe->mem.vram.mapping = 0;
original_size = pci_resource_len(pdev, GEN12_LMEM_BAR);
- if (xe->info.has_flat_ccs) {
- int err;
- u32 reg;
-
- err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
- if (err)
- return err;
- reg = xe_gt_mcr_unicast_read_any(gt, XEHP_TILE0_ADDR_RANGE);
- lmem_size = (u64)REG_FIELD_GET(GENMASK(14, 8), reg) * SZ_1G;
- reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR);
- flat_ccs_base = (u64)REG_FIELD_GET(GENMASK(31, 8), reg) * SZ_64K;
-
- drm_info(&xe->drm, "lmem_size: 0x%llx flat_ccs_base: 0x%llx\n",
- lmem_size, flat_ccs_base);
-
- err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
- if (err)
- return err;
- } else {
- flat_ccs_base = lmem_size;
- }
+ err = xe_mmio_total_vram_size(xe, &lmem_size, &flat_ccs_base);
+ if (err)
+ return err;
resize_result = xe_resize_lmem_bar(xe, lmem_size);
current_size = pci_resource_len(pdev, GEN12_LMEM_BAR);
}
int xe_mmio_probe_vram(struct xe_device *xe);
+int xe_mmio_total_vram_size(struct xe_device *xe, u64 *vram_size, u64 *flat_ccs_base);
#endif
#include "xe_pt_walk.h"
#include "xe_vm.h"
#include "xe_res_cursor.h"
+#include "xe_ttm_stolen_mgr.h"
struct xe_pt_dir {
struct xe_pt pt;
else
xe_walk.cache = XE_CACHE_WB;
}
+ if (xe_bo_is_stolen(bo))
+ xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo));
xe_bo_assert_held(bo);
if (xe_vma_is_userptr(vma))
xe_res_first_sg(vma->userptr.sg, 0, vma->end - vma->start + 1,
&curs);
- else if (xe_bo_is_vram(bo))
+ else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
xe_res_first(bo->ttm.resource, vma->bo_offset,
vma->end - vma->start + 1, &curs);
else
#include <drm/ttm/ttm_tt.h>
#include "xe_bo.h"
+#include "xe_device.h"
#include "xe_macros.h"
#include "xe_ttm_vram_mgr.h"
-/* state back for walking over vram_mgr and gtt_mgr allocations */
+/* state back for walking over vram_mgr, stolen_mgr, and gtt_mgr allocations */
struct xe_res_cursor {
u64 start;
u64 size;
void *node;
u32 mem_type;
struct scatterlist *sgl;
+ struct drm_buddy *mm;
};
+static struct drm_buddy *xe_res_get_buddy(struct ttm_resource *res)
+{
+ struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
+
+ if (res->mem_type != XE_PL_STOLEN) {
+ return &xe_device_get_gt(xe, res->mem_type - XE_PL_VRAM0)->mem.vram_mgr->mm;
+ } else {
+ struct ttm_resource_manager *mgr =
+ ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+
+ return &to_xe_ttm_vram_mgr(mgr)->mm;
+ }
+}
+
/**
* xe_res_first - initialize a xe_res_cursor
*
u64 start, u64 size,
struct xe_res_cursor *cur)
{
- struct drm_buddy_block *block;
- struct list_head *head, *next;
-
cur->sgl = NULL;
if (!res)
goto fallback;
cur->mem_type = res->mem_type;
switch (cur->mem_type) {
+ case XE_PL_STOLEN:
case XE_PL_VRAM0:
- case XE_PL_VRAM1:
+ case XE_PL_VRAM1: {
+ struct drm_buddy_block *block;
+ struct list_head *head, *next;
+ struct drm_buddy *mm = xe_res_get_buddy(res);
+
head = &to_xe_ttm_vram_mgr_resource(res)->blocks;
block = list_first_entry_or_null(head,
if (!block)
goto fallback;
- while (start >= xe_ttm_vram_mgr_block_size(block)) {
- start -= xe_ttm_vram_mgr_block_size(block);
+ while (start >= drm_buddy_block_size(mm, block)) {
+ start -= drm_buddy_block_size(mm, block);
next = block->link.next;
if (next != head)
link);
}
- cur->start = xe_ttm_vram_mgr_block_start(block) + start;
- cur->size = min(xe_ttm_vram_mgr_block_size(block) - start,
+ cur->mm = mm;
+ cur->start = drm_buddy_block_offset(block) + start;
+ cur->size = min(drm_buddy_block_size(mm, block) - start,
size);
cur->remaining = size;
cur->node = block;
break;
+ }
default:
goto fallback;
}
}
switch (cur->mem_type) {
+ case XE_PL_STOLEN:
case XE_PL_VRAM0:
case XE_PL_VRAM1:
start = size - cur->size;
block = list_entry(next, struct drm_buddy_block, link);
- while (start >= xe_ttm_vram_mgr_block_size(block)) {
- start -= xe_ttm_vram_mgr_block_size(block);
+ while (start >= drm_buddy_block_size(cur->mm, block)) {
+ start -= drm_buddy_block_size(cur->mm, block);
next = block->link.next;
block = list_entry(next, struct drm_buddy_block, link);
}
- cur->start = xe_ttm_vram_mgr_block_start(block) + start;
- cur->size = min(xe_ttm_vram_mgr_block_size(block) - start,
+ cur->start = drm_buddy_block_offset(block) + start;
+ cur->size = min(drm_buddy_block_size(cur->mm, block) - start,
cur->remaining);
cur->node = block;
break;
--- /dev/null
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2021-2022 Intel Corporation
+ * Copyright (C) 2021-2002 Red Hat
+ */
+
+#include <drm/drm_managed.h>
+#include <drm/drm_mm.h>
+
+#include <drm/ttm/ttm_device.h>
+#include <drm/ttm/ttm_range_manager.h>
+#include <drm/ttm/ttm_placement.h>
+
+#include "../i915/i915_reg.h"
+
+#include "xe_bo.h"
+#include "xe_device.h"
+#include "xe_gt.h"
+#include "xe_mmio.h"
+#include "xe_res_cursor.h"
+#include "xe_ttm_stolen_mgr.h"
+#include "xe_ttm_vram_mgr.h"
+
+bool xe_ttm_stolen_inaccessible(struct xe_device *xe)
+{
+ return !IS_DGFX(xe) && GRAPHICS_VERx100(xe) < 1270;
+}
+
+struct xe_ttm_stolen_mgr {
+ struct xe_ttm_vram_mgr base;
+
+ /* PCI base offset */
+ resource_size_t io_base;
+ /* GPU base offset */
+ resource_size_t stolen_base;
+
+ void *__iomem mapping;
+};
+
+static inline struct xe_ttm_stolen_mgr *
+to_stolen_mgr(struct ttm_resource_manager *man)
+{
+ return container_of(man, struct xe_ttm_stolen_mgr, base.manager);
+}
+
+static s64 detect_bar2_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
+{
+ struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+ struct xe_gt *gt = to_gt(xe);
+ u64 vram_size, stolen_size;
+ int err;
+
+ err = xe_mmio_total_vram_size(xe, &vram_size, NULL);
+ if (err) {
+ drm_info(&xe->drm, "Querying total vram size failed\n");
+ return 0;
+ }
+
+ /* Use DSM base address instead for stolen memory */
+ mgr->stolen_base = xe_mmio_read64(gt, GEN12_DSMBASE.reg) & GEN12_BDSM_MASK;
+ if (drm_WARN_ON(&xe->drm, vram_size < mgr->stolen_base))
+ return 0;
+
+ stolen_size = vram_size - mgr->stolen_base;
+ if (mgr->stolen_base + stolen_size <= pci_resource_len(pdev, 2))
+ mgr->io_base = pci_resource_start(pdev, 2) + mgr->stolen_base;
+
+ return stolen_size;
+}
+
+static u32 detect_bar2_integrated(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
+{
+ struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+ u32 stolen_size;
+ u32 ggc, gms;
+
+ ggc = xe_mmio_read32(to_gt(xe), GGC.reg);
+
+ /* check GGMS, should be fixed 0x3 (8MB) */
+ if (drm_WARN_ON(&xe->drm, (ggc & GGMS_MASK) != GGMS_MASK))
+ return 0;
+
+ mgr->stolen_base = mgr->io_base = pci_resource_start(pdev, 2) + SZ_8M;
+
+ /* return valid GMS value, -EIO if invalid */
+ gms = REG_FIELD_GET(GMS_MASK, ggc);
+ switch (gms) {
+ case 0x0 ... 0x04:
+ stolen_size = gms * 32 * SZ_1M;
+ break;
+ case 0xf0 ... 0xfe:
+ stolen_size = (gms - 0xf0 + 1) * 4 * SZ_1M;
+ break;
+ default:
+ return 0;
+ }
+
+ if (drm_WARN_ON(&xe->drm, stolen_size + SZ_8M > pci_resource_len(pdev, 2)))
+ return 0;
+
+ return stolen_size;
+}
+
+extern struct resource intel_graphics_stolen_res;
+
+static u64 detect_stolen(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
+{
+#ifdef CONFIG_X86
+ /* Map into GGTT */
+ mgr->io_base = pci_resource_start(to_pci_dev(xe->drm.dev), 2);
+
+ /* Stolen memory is x86 only */
+ mgr->stolen_base = intel_graphics_stolen_res.start;
+ return resource_size(&intel_graphics_stolen_res);
+#else
+ return 0;
+#endif
+}
+
+void xe_ttm_stolen_mgr_init(struct xe_device *xe)
+{
+ struct xe_ttm_stolen_mgr *mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL);
+ struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+ u64 stolen_size, pgsize;
+ int err;
+
+ if (IS_DGFX(xe))
+ stolen_size = detect_bar2_dgfx(xe, mgr);
+ else if (!xe_ttm_stolen_inaccessible(xe))
+ stolen_size = detect_bar2_integrated(xe, mgr);
+ else
+ stolen_size = detect_stolen(xe, mgr);
+
+ if (!stolen_size) {
+ drm_dbg_kms(&xe->drm, "No stolen memory support\n");
+ return;
+ }
+
+ pgsize = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K;
+ if (pgsize < PAGE_SIZE)
+ pgsize = PAGE_SIZE;
+
+
+ err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size, pgsize);
+ if (err) {
+ drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err);
+ return;
+ }
+
+ drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n",
+ stolen_size);
+
+ if (!xe_ttm_stolen_inaccessible(xe))
+ mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, stolen_size);
+}
+
+u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset)
+{
+ struct xe_device *xe = xe_bo_device(bo);
+ struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+ struct xe_ttm_stolen_mgr *mgr = to_stolen_mgr(ttm_mgr);
+ struct xe_res_cursor cur;
+
+ if (!mgr->io_base)
+ return 0;
+
+ if (!IS_DGFX(xe) && xe_ttm_stolen_inaccessible(xe))
+ return mgr->io_base + xe_bo_ggtt_addr(bo) + offset;
+
+ xe_res_first(bo->ttm.resource, offset, 4096, &cur);
+ return mgr->io_base + cur.start;
+}
+
+static int __xe_ttm_stolen_io_mem_reserve_bar2(struct xe_device *xe,
+ struct xe_ttm_stolen_mgr *mgr,
+ struct ttm_resource *mem)
+{
+ struct xe_res_cursor cur;
+
+ if (!mgr->io_base)
+ return -EIO;
+
+ xe_res_first(mem, 0, 4096, &cur);
+ mem->bus.offset = cur.start;
+
+ drm_WARN_ON(&xe->drm, !(mem->placement & TTM_PL_FLAG_CONTIGUOUS));
+ WARN_ON_ONCE(1);
+
+ if (mem->placement & TTM_PL_FLAG_CONTIGUOUS && mgr->mapping)
+ mem->bus.addr = (u8 *)mgr->mapping + mem->bus.offset;
+
+ mem->bus.offset += mgr->io_base;
+ mem->bus.is_iomem = true;
+ mem->bus.caching = ttm_write_combined;
+
+ return 0;
+}
+
+static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe,
+ struct xe_ttm_stolen_mgr *mgr,
+ struct ttm_resource *mem)
+{
+#ifdef CONFIG_X86
+ struct xe_bo *bo = ttm_to_xe_bo(mem->bo);
+
+ /* XXX: Require BO to be mapped to GGTT? */
+ if (drm_WARN_ON(&xe->drm, !(bo->flags & XE_BO_CREATE_GGTT_BIT)))
+ return -EIO;
+
+ /* GGTT is always contiguously mapped */
+ mem->bus.offset = xe_bo_ggtt_addr(bo) + mgr->io_base;
+
+ mem->bus.is_iomem = true;
+ mem->bus.caching = ttm_write_combined;
+
+ return 0;
+#else
+ /* How is it even possible to get here without gen12 stolen? */
+ drm_WARN_ON(&xe->drm, 1);
+ return -EIO;
+#endif
+}
+
+int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem)
+{
+ struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+ struct xe_ttm_stolen_mgr *mgr = ttm_mgr ? to_stolen_mgr(ttm_mgr) : NULL;
+
+ if (!mgr || !mgr->io_base)
+ return -EIO;
+
+ if (!xe_ttm_stolen_inaccessible(xe))
+ return __xe_ttm_stolen_io_mem_reserve_bar2(xe, mgr, mem);
+ else
+ return __xe_ttm_stolen_io_mem_reserve_stolen(xe, mgr, mem);
+}
+
+u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe)
+{
+ struct xe_ttm_stolen_mgr *mgr =
+ to_stolen_mgr(ttm_manager_type(&xe->ttm, XE_PL_STOLEN));
+
+ return mgr->stolen_base;
+}
--- /dev/null
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef _XE_TTM_STOLEN_MGR_H_
+#define _XE_TTM_STOLEN_MGR_H_
+
+#include <linux/types.h>
+
+struct ttm_resource;
+struct xe_bo;
+struct xe_device;
+
+void xe_ttm_stolen_mgr_init(struct xe_device *xe);
+int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem);
+bool xe_ttm_stolen_inaccessible(struct xe_device *xe);
+u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset);
+u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe);
+
+#endif
#include "xe_res_cursor.h"
#include "xe_ttm_vram_mgr.h"
-static inline struct xe_ttm_vram_mgr *
-to_vram_mgr(struct ttm_resource_manager *man)
-{
- return container_of(man, struct xe_ttm_vram_mgr, manager);
-}
-
-static inline struct xe_gt *
-mgr_to_gt(struct xe_ttm_vram_mgr *mgr)
-{
- return mgr->gt;
-}
-
static inline struct drm_buddy_block *
xe_ttm_vram_mgr_first_block(struct list_head *list)
{
return list_first_entry_or_null(list, struct drm_buddy_block, link);
}
-static inline bool xe_is_vram_mgr_blocks_contiguous(struct list_head *head)
+static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm,
+ struct list_head *head)
{
struct drm_buddy_block *block;
u64 start, size;
return false;
while (head != block->link.next) {
- start = xe_ttm_vram_mgr_block_start(block);
- size = xe_ttm_vram_mgr_block_size(block);
+ start = drm_buddy_block_offset(block);
+ size = drm_buddy_block_size(mm, block);
block = list_entry(block->link.next, struct drm_buddy_block,
link);
- if (start + size != xe_ttm_vram_mgr_block_start(block))
+ if (start + size != drm_buddy_block_offset(block))
return false;
}
struct ttm_resource **res)
{
u64 max_bytes, cur_size, min_block_size;
- struct xe_ttm_vram_mgr *mgr = to_vram_mgr(man);
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
struct xe_ttm_vram_mgr_resource *vres;
u64 size, remaining_size, lpfn, fpfn;
struct drm_buddy *mm = &mgr->mm;
int r;
lpfn = (u64)place->lpfn << PAGE_SHIFT;
- if (!lpfn)
+ if (!lpfn || lpfn > man->size)
lpfn = man->size;
fpfn = (u64)place->fpfn << PAGE_SHIFT;
- max_bytes = mgr->gt->mem.vram.size;
+ max_bytes = mgr->manager.size;
if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
pages_per_block = ~0ul;
} else {
* Compute the original_size value by subtracting the
* last block size with (aligned size - original size)
*/
- original_size = xe_ttm_vram_mgr_block_size(block) -
+ original_size = drm_buddy_block_size(mm, block) -
(size - cur_size);
}
list_for_each_entry(block, &vres->blocks, link) {
unsigned long start;
- start = xe_ttm_vram_mgr_block_start(block) +
- xe_ttm_vram_mgr_block_size(block);
+ start = drm_buddy_block_offset(block) +
+ drm_buddy_block_size(mm, block);
start >>= PAGE_SHIFT;
if (start > PFN_UP(vres->base.size))
vres->base.start = max(vres->base.start, start);
}
- if (xe_is_vram_mgr_blocks_contiguous(&vres->blocks))
+ if (xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks))
vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
*res = &vres->base;
{
struct xe_ttm_vram_mgr_resource *vres =
to_xe_ttm_vram_mgr_resource(res);
- struct xe_ttm_vram_mgr *mgr = to_vram_mgr(man);
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
struct drm_buddy *mm = &mgr->mm;
mutex_lock(&mgr->lock);
static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man,
struct drm_printer *printer)
{
- struct xe_ttm_vram_mgr *mgr = to_vram_mgr(man);
+ struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
struct drm_buddy *mm = &mgr->mm;
mutex_lock(&mgr->lock);
.debug = xe_ttm_vram_mgr_debug
};
-static void ttm_vram_mgr_fini(struct drm_device *drm, void *arg)
+static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
{
+ struct xe_device *xe = to_xe_device(dev);
struct xe_ttm_vram_mgr *mgr = arg;
- struct xe_device *xe = gt_to_xe(mgr->gt);
struct ttm_resource_manager *man = &mgr->manager;
- int err;
ttm_resource_manager_set_used(man, false);
- err = ttm_resource_manager_evict_all(&xe->ttm, man);
- if (err)
+ if (ttm_resource_manager_evict_all(&xe->ttm, man))
return;
drm_buddy_fini(&mgr->mm);
- ttm_resource_manager_cleanup(man);
- ttm_set_driver_manager(&xe->ttm, XE_PL_VRAM0 + mgr->gt->info.vram_id,
- NULL);
+ ttm_resource_manager_cleanup(&mgr->manager);
+
+ ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL);
}
-int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr)
+int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
+ u32 mem_type, u64 size, u64 default_page_size)
{
- struct xe_device *xe = gt_to_xe(gt);
struct ttm_resource_manager *man = &mgr->manager;
int err;
- XE_BUG_ON(xe_gt_is_media_type(gt));
-
- mgr->gt = gt;
man->func = &xe_ttm_vram_mgr_func;
+ mgr->mem_type = mem_type;
+ mutex_init(&mgr->lock);
+ mgr->default_page_size = default_page_size;
- ttm_resource_manager_init(man, &xe->ttm, gt->mem.vram.size);
- err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
- if (err)
- return err;
+ ttm_resource_manager_init(man, &xe->ttm, size);
+ err = drm_buddy_init(&mgr->mm, man->size, default_page_size);
- mutex_init(&mgr->lock);
- mgr->default_page_size = PAGE_SIZE;
+ ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager);
+ ttm_resource_manager_set_used(&mgr->manager, true);
+
+ return drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr);
+}
+
+int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr)
+{
+ struct xe_device *xe = gt_to_xe(gt);
- ttm_set_driver_manager(&xe->ttm, XE_PL_VRAM0 + gt->info.vram_id,
- &mgr->manager);
- ttm_resource_manager_set_used(man, true);
+ XE_BUG_ON(xe_gt_is_media_type(gt));
- err = drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr);
- if (err)
- return err;
+ mgr->gt = gt;
- return 0;
+ return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + gt->info.vram_id,
+ gt->mem.vram.size, PAGE_SIZE);
}
int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
struct xe_device;
struct xe_gt;
+int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
+ u32 mem_type, u64 size, u64 default_page_size);
int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr);
int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
struct ttm_resource *res,
void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir,
struct sg_table *sgt);
-static inline u64 xe_ttm_vram_mgr_block_start(struct drm_buddy_block *block)
-{
- return drm_buddy_block_offset(block);
-}
-
-static inline u64 xe_ttm_vram_mgr_block_size(struct drm_buddy_block *block)
-{
- return PAGE_SIZE << drm_buddy_block_order(block);
-}
-
static inline struct xe_ttm_vram_mgr_resource *
to_xe_ttm_vram_mgr_resource(struct ttm_resource *res)
{
return container_of(res, struct xe_ttm_vram_mgr_resource, base);
}
+static inline struct xe_ttm_vram_mgr *
+to_xe_ttm_vram_mgr(struct ttm_resource_manager *man)
+{
+ return container_of(man, struct xe_ttm_vram_mgr, manager);
+}
+
#endif
u64 default_page_size;
/** @lock: protects allocations of VRAM */
struct mutex lock;
+
+ u32 mem_type;
};
/**
projects / linux-2.6-block.git / commitdiff