drm/amdkfd: Update CU masking for GFX 9.4.3

The CU mask passed from user-space will change based on
different spatial partitioning mode. As a result, update
CU masking code for GFX9.4.3 to work for all partitioning
modes.

Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index 763966236658e01c012247835558ccb8daba3c17..447829c22295c6cf6c7570a592c0ef33ba0211c8 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -97,14 +97,16 @@ void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd,
 
 void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
                const uint32_t *cu_mask, uint32_t cu_mask_count,
-               uint32_t *se_mask)
+               uint32_t *se_mask, uint32_t inst)
 {
        struct kfd_cu_info cu_info;
        uint32_t cu_per_sh[KFD_MAX_NUM_SE][KFD_MAX_NUM_SH_PER_SE] = {0};
        bool wgp_mode_req = KFD_GC_VERSION(mm->dev) >= IP_VERSION(10, 0, 0);
        uint32_t en_mask = wgp_mode_req ? 0x3 : 0x1;
-       int i, se, sh, cu, cu_bitmap_sh_mul, inc = wgp_mode_req ? 2 : 1;
+       int i, se, sh, cu, cu_bitmap_sh_mul, cu_inc = wgp_mode_req ? 2 : 1;
        uint32_t cu_active_per_node;
+       int inc = cu_inc * NUM_XCC(mm->dev->xcc_mask);
+       int xcc_inst = inst + ffs(mm->dev->xcc_mask) - 1;
 
        amdgpu_amdkfd_get_cu_info(mm->dev->adev, &cu_info);
 
@@ -143,7 +145,8 @@ void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
        for (se = 0; se < cu_info.num_shader_engines; se++)
                for (sh = 0; sh < cu_info.num_shader_arrays_per_engine; sh++)
                        cu_per_sh[se][sh] = hweight32(
-                               cu_info.cu_bitmap[0][se % 4][sh + (se / 4) * cu_bitmap_sh_mul]);
+                               cu_info.cu_bitmap[xcc_inst][se % 4][sh + (se / 4) *
+                               cu_bitmap_sh_mul]);
 
        /* Symmetrically map cu_mask to all SEs & SHs:
         * se_mask programs up to 2 SH in the upper and lower 16 bits.
@@ -166,20 +169,33 @@ void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
         * cu_mask[0] bit8 -> se_mask[0] bit1 (SE0,SH0,CU1)
         * ...
         *
+        * For GFX 9.4.3, the following code only looks at a
+        * subset of the cu_mask corresponding to the inst parameter.
+        * If we have n XCCs under one GPU node
+        * cu_mask[0] bit0 -> XCC0 se_mask[0] bit0 (XCC0,SE0,SH0,CU0)
+        * cu_mask[0] bit1 -> XCC1 se_mask[0] bit0 (XCC1,SE0,SH0,CU0)
+        * ..
+        * cu_mask[0] bitn -> XCCn se_mask[0] bit0 (XCCn,SE0,SH0,CU0)
+        * cu_mask[0] bit n+1 -> XCC0 se_mask[1] bit0 (XCC0,SE1,SH0,CU0)
+        *
+        * For example, if there are 6 XCCs under 1 KFD node, this code
+        * running for each inst, will look at the bits as:
+        * inst, inst + 6, inst + 12...
+        *
         * First ensure all CUs are disabled, then enable user specified CUs.
         */
        for (i = 0; i < cu_info.num_shader_engines; i++)
                se_mask[i] = 0;
 
-       i = 0;
-       for (cu = 0; cu < 16; cu += inc) {
+       i = inst;
+       for (cu = 0; cu < 16; cu += cu_inc) {
                for (sh = 0; sh < cu_info.num_shader_arrays_per_engine; sh++) {
                        for (se = 0; se < cu_info.num_shader_engines; se++) {
                                if (cu_per_sh[se][sh] > cu) {
                                        if (cu_mask[i / 32] & (en_mask << (i % 32)))
                                                se_mask[se] |= en_mask << (cu + sh * 16);
                                        i += inc;
-                                       if (i == cu_mask_count)
+                                       if (i >= cu_mask_count)
                                                return;
                                }
                        }

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 23158db7da035d9820b8b14007826f75f69f3cc1..57bf5e513f4d16198a6dda43f27f422e4df5f44a 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -138,7 +138,7 @@ void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd,
 
 void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
                const uint32_t *cu_mask, uint32_t cu_mask_count,
-               uint32_t *se_mask);
+               uint32_t *se_mask, uint32_t inst);
 
 int kfd_hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd,
                uint32_t pipe_id, uint32_t queue_id,

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index ee1d32d957f2b1bc3757778506ed55a8264588e1..1a4a69943c714069ba5a2b314fabdd6849e74a6a 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -52,7 +52,7 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
                return;
 
        mqd_symmetrically_map_cu_mask(mm,
-               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask);
+               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, 0);
 
        m = get_mqd(mqd);
        m->compute_static_thread_mgmt_se0 = se_mask[0];

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 83699392c8089cd12d8bafa274c72e6e0d8bcca3..8b7fed91352696cf2b5cafab0680ad0737fa95ee 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -52,7 +52,7 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
                return;
 
        mqd_symmetrically_map_cu_mask(mm,
-               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask);
+               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, 0);
 
        m = get_mqd(mqd);
        m->compute_static_thread_mgmt_se0 = se_mask[0];

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index 0bbf0edbabd47ecb9cb551173862183a7c194b76..964b5d50a77e6cc3b23071f177e88d9ad4c7dee9 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -71,7 +71,7 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
        }
 
        mqd_symmetrically_map_cu_mask(mm,
-               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask);
+               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, 0);
 
        m->compute_static_thread_mgmt_se0 = se_mask[0];
        m->compute_static_thread_mgmt_se1 = se_mask[1];

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index e23d32f356077225f06b76008b8b75cc6ab22a87..42d881809dc70e230133674e4b12f6f68567837a 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -60,7 +60,7 @@ static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd)
 }
 
 static void update_cu_mask(struct mqd_manager *mm, void *mqd,
-                       struct mqd_update_info *minfo)
+                       struct mqd_update_info *minfo, uint32_t inst)
 {
        struct v9_mqd *m;
        uint32_t se_mask[KFD_MAX_NUM_SE] = {0};
@@ -69,27 +69,36 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
                return;
 
        mqd_symmetrically_map_cu_mask(mm,
-               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask);
+               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, inst);
 
        m = get_mqd(mqd);
+
        m->compute_static_thread_mgmt_se0 = se_mask[0];
        m->compute_static_thread_mgmt_se1 = se_mask[1];
        m->compute_static_thread_mgmt_se2 = se_mask[2];
        m->compute_static_thread_mgmt_se3 = se_mask[3];
-       m->compute_static_thread_mgmt_se4 = se_mask[4];
-       m->compute_static_thread_mgmt_se5 = se_mask[5];
-       m->compute_static_thread_mgmt_se6 = se_mask[6];
-       m->compute_static_thread_mgmt_se7 = se_mask[7];
-
-       pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n",
-               m->compute_static_thread_mgmt_se0,
-               m->compute_static_thread_mgmt_se1,
-               m->compute_static_thread_mgmt_se2,
-               m->compute_static_thread_mgmt_se3,
-               m->compute_static_thread_mgmt_se4,
-               m->compute_static_thread_mgmt_se5,
-               m->compute_static_thread_mgmt_se6,
-               m->compute_static_thread_mgmt_se7);
+       if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3)) {
+               m->compute_static_thread_mgmt_se4 = se_mask[4];
+               m->compute_static_thread_mgmt_se5 = se_mask[5];
+               m->compute_static_thread_mgmt_se6 = se_mask[6];
+               m->compute_static_thread_mgmt_se7 = se_mask[7];
+
+               pr_debug("update cu mask to %#x %#x %#x %#x %#x %#x %#x %#x\n",
+                       m->compute_static_thread_mgmt_se0,
+                       m->compute_static_thread_mgmt_se1,
+                       m->compute_static_thread_mgmt_se2,
+                       m->compute_static_thread_mgmt_se3,
+                       m->compute_static_thread_mgmt_se4,
+                       m->compute_static_thread_mgmt_se5,
+                       m->compute_static_thread_mgmt_se6,
+                       m->compute_static_thread_mgmt_se7);
+       } else {
+               pr_debug("inst: %u, update cu mask to %#x %#x %#x %#x\n",
+                       inst, m->compute_static_thread_mgmt_se0,
+                       m->compute_static_thread_mgmt_se1,
+                       m->compute_static_thread_mgmt_se2,
+                       m->compute_static_thread_mgmt_se3);
+       }
 }
 
 static void set_priority(struct v9_mqd *m, struct queue_properties *q)
@@ -290,7 +299,8 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
        if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
                m->cp_hqd_ctx_save_control = 0;
 
-       update_cu_mask(mm, mqd, minfo);
+       if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3))
+               update_cu_mask(mm, mqd, minfo, 0);
        set_priority(m, q);
 
        q->is_active = QUEUE_IS_ACTIVE(*q);
@@ -676,6 +686,8 @@ static void update_mqd_v9_4_3(struct mqd_manager *mm, void *mqd,
                m = get_mqd(mqd + size * xcc);
                update_mqd(mm, m, q, minfo);
 
+               update_cu_mask(mm, mqd, minfo, xcc);
+
                if (q->format == KFD_QUEUE_FORMAT_AQL) {
                        switch (xcc) {
                        case 0:

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 657c378229808fdfb2518203269fb81c744e53d0..3e1a574d4ea66981b9a5243211bd451def7aee94 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -55,7 +55,7 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd,
                return;
 
        mqd_symmetrically_map_cu_mask(mm,
-               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask);
+               minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, 0);
 
        m = get_mqd(mqd);
        m->compute_static_thread_mgmt_se0 = se_mask[0];