From 98a54e88e87f7291d4bbc6ec646c498f64ae042f Mon Sep 17 00:00:00 2001 From: Le Ma Date: Tue, 17 May 2022 22:20:10 +0800 Subject: [PATCH] drm/amdgpu: add sysfs node for compute partition mode Add current/available compute partitin mode sysfs node. v2: make the sysfs node as IP independent one in amdgpu_gfx.c Signed-off-by: Le Ma Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 132 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 6 + drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h | 3 + drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 67 +++++++++++ drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c | 32 +++++ 6 files changed, 241 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index e94507a10e15..f432064a0535 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3634,6 +3634,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, mutex_init(&adev->srbm_mutex); mutex_init(&adev->gfx.pipe_reserve_mutex); mutex_init(&adev->gfx.gfx_off_mutex); + mutex_init(&adev->gfx.partition_mutex); mutex_init(&adev->grbm_idx_mutex); mutex_init(&adev->mn_lock); mutex_init(&adev->virt.vf_errors.lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index e5ff0bf9f23b..4a4d71ff9b95 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -1174,3 +1174,135 @@ bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id) return !(xcc_id % (adev->gfx.num_xcc_per_xcp ? adev->gfx.num_xcc_per_xcp : 1)); } + +static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev, + struct device_attribute *addr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + enum amdgpu_gfx_partition mode; + char *partition_mode; + + mode = adev->gfx.funcs->query_partition_mode(adev); + + switch (mode) { + case AMDGPU_SPX_PARTITION_MODE: + partition_mode = "SPX"; + break; + case AMDGPU_DPX_PARTITION_MODE: + partition_mode = "DPX"; + break; + case AMDGPU_TPX_PARTITION_MODE: + partition_mode = "TPX"; + break; + case AMDGPU_QPX_PARTITION_MODE: + partition_mode = "QPX"; + break; + case AMDGPU_CPX_PARTITION_MODE: + partition_mode = "CPX"; + break; + default: + partition_mode = "UNKNOWN"; + break; + } + + return sysfs_emit(buf, "%s\n", partition_mode); +} + +static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev, + struct device_attribute *addr, + const char *buf, size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + enum amdgpu_gfx_partition mode; + int ret; + + if (adev->gfx.num_xcd % 2 != 0) + return -EINVAL; + + if (!strncasecmp("SPX", buf, strlen("SPX"))) { + mode = AMDGPU_SPX_PARTITION_MODE; + } else if (!strncasecmp("DPX", buf, strlen("DPX"))) { + if (adev->gfx.num_xcd != 4 || adev->gfx.num_xcd != 8) + return -EINVAL; + mode = AMDGPU_DPX_PARTITION_MODE; + } else if (!strncasecmp("TPX", buf, strlen("TPX"))) { + if (adev->gfx.num_xcd != 6) + return -EINVAL; + mode = AMDGPU_TPX_PARTITION_MODE; + } else if (!strncasecmp("QPX", buf, strlen("QPX"))) { + if (adev->gfx.num_xcd != 8) + return -EINVAL; + mode = AMDGPU_QPX_PARTITION_MODE; + } else if (!strncasecmp("CPX", buf, strlen("CPX"))) { + mode = AMDGPU_CPX_PARTITION_MODE; + } else { + return -EINVAL; + } + + mutex_lock(&adev->gfx.partition_mutex); + + ret = adev->gfx.funcs->switch_partition_mode(adev, mode); + + mutex_unlock(&adev->gfx.partition_mutex); + + if (ret) + return ret; + + return count; +} + +static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev, + struct device_attribute *addr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + char *supported_partition; + + /* TBD */ + switch (adev->gfx.num_xcd) { + case 8: + supported_partition = "SPX, DPX, QPX, CPX"; + break; + case 6: + supported_partition = "SPX, TPX, CPX"; + break; + case 4: + supported_partition = "SPX, DPX, CPX"; + break; + /* this seems only existing in emulation phase */ + case 2: + supported_partition = "SPX, CPX"; + break; + default: + supported_partition = "Not supported"; + break; + } + + return sysfs_emit(buf, "%s\n", supported_partition); +} + +static DEVICE_ATTR(current_compute_partition, S_IRUGO | S_IWUSR, + amdgpu_gfx_get_current_compute_partition, + amdgpu_gfx_set_compute_partition); + +static DEVICE_ATTR(available_compute_partition, S_IRUGO, + amdgpu_gfx_get_available_compute_partition, NULL); + +int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev) +{ + int r; + + r = device_create_file(adev->dev, &dev_attr_current_compute_partition); + if (r) + return r; + + r = device_create_file(adev->dev, &dev_attr_available_compute_partition); + if (r) + return r; + + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index 2755f00ac19a..8df36527aee9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -259,6 +259,10 @@ struct amdgpu_gfx_funcs { void (*update_perfmon_mgcg)(struct amdgpu_device *adev, bool enable); int (*get_gfx_shadow_info)(struct amdgpu_device *adev, struct amdgpu_gfx_shadow_info *shadow_info); + enum amdgpu_gfx_partition + (*query_partition_mode)(struct amdgpu_device *adev); + int (*switch_partition_mode)(struct amdgpu_device *adev, + enum amdgpu_gfx_partition mode); }; struct sq_work { @@ -394,6 +398,7 @@ struct amdgpu_gfx { enum amdgpu_gfx_partition partition_mode; uint32_t num_xcd; uint32_t num_xcc_per_xcp; + struct mutex partition_mutex; }; #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev)) @@ -478,4 +483,5 @@ int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry); bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id); +int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h index c686ff4bcc39..6a636c34b717 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h @@ -95,6 +95,9 @@ struct amdgpu_nbio_funcs { void (*apply_l1_link_width_reconfig_wa)(struct amdgpu_device *adev); void (*clear_doorbell_interrupt)(struct amdgpu_device *adev); u32 (*get_rom_offset)(struct amdgpu_device *adev); + u32 (*get_compute_partition_mode)(struct amdgpu_device *adev); + void (*set_compute_partition_mode)(struct amdgpu_device *adev, + enum amdgpu_gfx_partition mode); }; struct amdgpu_nbio { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index 064cd02451c2..1d15db9423c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -662,6 +662,67 @@ static void gfx_v9_4_3_select_me_pipe_q(struct amdgpu_device *adev, soc15_grbm_select(adev, me, pipe, q, vm, 0); } +static enum amdgpu_gfx_partition +gfx_v9_4_3_query_compute_partition(struct amdgpu_device *adev) +{ + enum amdgpu_gfx_partition mode = AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE; + + if (adev->nbio.funcs->get_compute_partition_mode) + mode = adev->nbio.funcs->get_compute_partition_mode(adev); + + return mode; +} + +static int gfx_v9_4_3_switch_compute_partition(struct amdgpu_device *adev, + enum amdgpu_gfx_partition mode) +{ + u32 tmp = 0; + int num_xcc_per_partition, i; + + if (mode == adev->gfx.partition_mode) + return mode; + + switch (mode) { + case AMDGPU_SPX_PARTITION_MODE: + num_xcc_per_partition = adev->gfx.num_xcd; + break; + case AMDGPU_DPX_PARTITION_MODE: + num_xcc_per_partition = adev->gfx.num_xcd / 2; + break; + case AMDGPU_TPX_PARTITION_MODE: + num_xcc_per_partition = adev->gfx.num_xcd / 3; + break; + case AMDGPU_QPX_PARTITION_MODE: + num_xcc_per_partition = adev->gfx.num_xcd / 4; + break; + case AMDGPU_CPX_PARTITION_MODE: + num_xcc_per_partition = 1; + break; + default: + return -EINVAL; + } + + /* TODO: + * Stop user queues and threads, and make sure GPU is empty of work. + */ + + for (i = 0; i < adev->gfx.num_xcd; i++) { + tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP, + num_xcc_per_partition); + tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, VIRTUAL_XCC_ID, + i % num_xcc_per_partition); + WREG32_SOC15(GC, i, regCP_HYP_XCP_CTL, tmp); + } + + if (adev->nbio.funcs->set_compute_partition_mode) + adev->nbio.funcs->set_compute_partition_mode(adev, mode); + + adev->gfx.num_xcc_per_xcp = num_xcc_per_partition; + adev->gfx.partition_mode = mode; + + return 0; +} + static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = { .get_gpu_clock_counter = &gfx_v9_4_3_get_gpu_clock_counter, .select_se_sh = &gfx_v9_4_3_select_se_sh, @@ -669,6 +730,8 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = { .read_wave_sgprs = &gfx_v9_4_3_read_wave_sgprs, .read_wave_vgprs = &gfx_v9_4_3_read_wave_vgprs, .select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q, + .query_partition_mode = &gfx_v9_4_3_query_compute_partition, + .switch_partition_mode = &gfx_v9_4_3_switch_compute_partition, }; static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev) @@ -858,6 +921,10 @@ static int gfx_v9_4_3_sw_init(void *handle) if (r) return r; + r = amdgpu_gfx_sysfs_init(adev); + if (r) + return r; + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c index a331a59c49e3..bdb84a53f0e4 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c @@ -347,6 +347,36 @@ static void nbio_v7_9_enable_doorbell_interrupt(struct amdgpu_device *adev, DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1); } +static enum amdgpu_gfx_partition nbio_v7_9_get_compute_partition_mode(struct amdgpu_device *adev) +{ + u32 tmp; + + tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_CAP); + + if (REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_CAP, SPX_SUPPORT)) + return AMDGPU_SPX_PARTITION_MODE; + else if (REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_CAP, DPX_SUPPORT)) + return AMDGPU_DPX_PARTITION_MODE; + else if (REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_CAP, TPX_SUPPORT)) + return AMDGPU_TPX_PARTITION_MODE; + else if (REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_CAP, CPX_SUPPORT)) + return AMDGPU_CPX_PARTITION_MODE; + else + return AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE; +} + +static void nbio_v7_9_set_compute_partition_mode(struct amdgpu_device *adev, + enum amdgpu_gfx_partition mode) +{ + u32 tmp; + + tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_CAP); + tmp &= ~0x1f; + tmp |= 1 << mode; + + WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_CAP, tmp); +} + const struct amdgpu_nbio_funcs nbio_v7_9_funcs = { .get_hdp_flush_req_offset = nbio_v7_9_get_hdp_flush_req_offset, .get_hdp_flush_done_offset = nbio_v7_9_get_hdp_flush_done_offset, @@ -366,4 +396,6 @@ const struct amdgpu_nbio_funcs nbio_v7_9_funcs = { .get_clockgating_state = nbio_v7_9_get_clockgating_state, .ih_control = nbio_v7_9_ih_control, .remap_hdp_registers = nbio_v7_9_remap_hdp_registers, + .get_compute_partition_mode = nbio_v7_9_get_compute_partition_mode, + .set_compute_partition_mode = nbio_v7_9_set_compute_partition_mode, }; -- 2.25.1