2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/module.h>
24 #include <linux/fdtable.h>
25 #include <linux/uaccess.h>
30 #include "radeon_kfd.h"
31 #include "radeon_ucode.h"
32 #include <linux/firmware.h>
33 #include "cik_structs.h"
35 #define CIK_PIPE_PER_MEC (4)
37 static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
38 TCP_WATCH0_ADDR_H, TCP_WATCH0_ADDR_L, TCP_WATCH0_CNTL,
39 TCP_WATCH1_ADDR_H, TCP_WATCH1_ADDR_L, TCP_WATCH1_CNTL,
40 TCP_WATCH2_ADDR_H, TCP_WATCH2_ADDR_L, TCP_WATCH2_CNTL,
41 TCP_WATCH3_ADDR_H, TCP_WATCH3_ADDR_L, TCP_WATCH3_CNTL
51 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
52 void **mem_obj, uint64_t *gpu_addr,
55 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
57 static uint64_t get_vmem_size(struct kgd_dev *kgd);
58 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
60 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
61 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
64 * Register access functions
67 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
68 uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
69 uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
71 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
74 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
75 uint32_t hpd_size, uint64_t hpd_gpu_addr);
76 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
77 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
78 uint32_t queue_id, uint32_t __user *wptr);
79 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
80 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
81 uint32_t pipe_id, uint32_t queue_id);
83 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
84 unsigned int timeout, uint32_t pipe_id,
86 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
87 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
88 unsigned int timeout);
89 static int kgd_address_watch_disable(struct kgd_dev *kgd);
90 static int kgd_address_watch_execute(struct kgd_dev *kgd,
91 unsigned int watch_point_id,
95 static int kgd_wave_control_execute(struct kgd_dev *kgd,
96 uint32_t gfx_index_val,
98 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
99 unsigned int watch_point_id,
100 unsigned int reg_offset);
102 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid);
103 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
105 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
107 static const struct kfd2kgd_calls kfd2kgd = {
108 .init_gtt_mem_allocation = alloc_gtt_mem,
109 .free_gtt_mem = free_gtt_mem,
110 .get_vmem_size = get_vmem_size,
111 .get_gpu_clock_counter = get_gpu_clock_counter,
112 .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
113 .program_sh_mem_settings = kgd_program_sh_mem_settings,
114 .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
115 .init_pipeline = kgd_init_pipeline,
116 .init_interrupts = kgd_init_interrupts,
117 .hqd_load = kgd_hqd_load,
118 .hqd_sdma_load = kgd_hqd_sdma_load,
119 .hqd_is_occupied = kgd_hqd_is_occupied,
120 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
121 .hqd_destroy = kgd_hqd_destroy,
122 .hqd_sdma_destroy = kgd_hqd_sdma_destroy,
123 .address_watch_disable = kgd_address_watch_disable,
124 .address_watch_execute = kgd_address_watch_execute,
125 .wave_control_execute = kgd_wave_control_execute,
126 .address_watch_get_offset = kgd_address_watch_get_offset,
127 .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid,
128 .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid,
129 .write_vmid_invalidate_request = write_vmid_invalidate_request,
130 .get_fw_version = get_fw_version
133 static const struct kgd2kfd_calls *kgd2kfd;
135 int radeon_kfd_init(void)
139 #if defined(CONFIG_HSA_AMD_MODULE)
140 int (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
142 kgd2kfd_init_p = symbol_request(kgd2kfd_init);
144 if (kgd2kfd_init_p == NULL)
147 ret = kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd);
149 symbol_put(kgd2kfd_init);
153 #elif defined(CONFIG_HSA_AMD)
154 ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd);
165 void radeon_kfd_fini(void)
169 symbol_put(kgd2kfd_init);
173 void radeon_kfd_device_probe(struct radeon_device *rdev)
176 rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev,
177 rdev->pdev, &kfd2kgd);
180 void radeon_kfd_device_init(struct radeon_device *rdev)
183 struct kgd2kfd_shared_resources gpu_resources = {
184 .compute_vmid_bitmap = 0xFF00,
186 .first_compute_pipe = 1,
187 .compute_pipe_count = 4 - 1,
190 radeon_doorbell_get_kfd_info(rdev,
191 &gpu_resources.doorbell_physical_address,
192 &gpu_resources.doorbell_aperture_size,
193 &gpu_resources.doorbell_start_offset);
195 kgd2kfd->device_init(rdev->kfd, &gpu_resources);
199 void radeon_kfd_device_fini(struct radeon_device *rdev)
202 kgd2kfd->device_exit(rdev->kfd);
207 void radeon_kfd_interrupt(struct radeon_device *rdev, const void *ih_ring_entry)
210 kgd2kfd->interrupt(rdev->kfd, ih_ring_entry);
213 void radeon_kfd_suspend(struct radeon_device *rdev)
216 kgd2kfd->suspend(rdev->kfd);
219 int radeon_kfd_resume(struct radeon_device *rdev)
224 r = kgd2kfd->resume(rdev->kfd);
229 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
230 void **mem_obj, uint64_t *gpu_addr,
233 struct radeon_device *rdev = (struct radeon_device *)kgd;
234 struct kgd_mem **mem = (struct kgd_mem **) mem_obj;
238 BUG_ON(gpu_addr == NULL);
239 BUG_ON(cpu_ptr == NULL);
241 *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL);
245 r = radeon_bo_create(rdev, size, PAGE_SIZE, true, RADEON_GEM_DOMAIN_GTT,
246 RADEON_GEM_GTT_WC, NULL, NULL, &(*mem)->bo);
249 "failed to allocate BO for amdkfd (%d)\n", r);
254 r = radeon_bo_reserve((*mem)->bo, true);
256 dev_err(rdev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
257 goto allocate_mem_reserve_bo_failed;
260 r = radeon_bo_pin((*mem)->bo, RADEON_GEM_DOMAIN_GTT,
263 dev_err(rdev->dev, "(%d) failed to pin bo for amdkfd\n", r);
264 goto allocate_mem_pin_bo_failed;
266 *gpu_addr = (*mem)->gpu_addr;
268 r = radeon_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr);
271 "(%d) failed to map bo to kernel for amdkfd\n", r);
272 goto allocate_mem_kmap_bo_failed;
274 *cpu_ptr = (*mem)->cpu_ptr;
276 radeon_bo_unreserve((*mem)->bo);
280 allocate_mem_kmap_bo_failed:
281 radeon_bo_unpin((*mem)->bo);
282 allocate_mem_pin_bo_failed:
283 radeon_bo_unreserve((*mem)->bo);
284 allocate_mem_reserve_bo_failed:
285 radeon_bo_unref(&(*mem)->bo);
290 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
292 struct kgd_mem *mem = (struct kgd_mem *) mem_obj;
296 radeon_bo_reserve(mem->bo, true);
297 radeon_bo_kunmap(mem->bo);
298 radeon_bo_unpin(mem->bo);
299 radeon_bo_unreserve(mem->bo);
300 radeon_bo_unref(&(mem->bo));
304 static uint64_t get_vmem_size(struct kgd_dev *kgd)
306 struct radeon_device *rdev = (struct radeon_device *)kgd;
310 return rdev->mc.real_vram_size;
313 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
315 struct radeon_device *rdev = (struct radeon_device *)kgd;
317 return rdev->asic->get_gpu_clock_counter(rdev);
320 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
322 struct radeon_device *rdev = (struct radeon_device *)kgd;
324 /* The sclk is in quantas of 10kHz */
325 return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
328 static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)
330 return (struct radeon_device *)kgd;
333 static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)
335 struct radeon_device *rdev = get_radeon_device(kgd);
337 writel(value, (void __iomem *)(rdev->rmmio + offset));
340 static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)
342 struct radeon_device *rdev = get_radeon_device(kgd);
344 return readl((void __iomem *)(rdev->rmmio + offset));
347 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,
348 uint32_t queue, uint32_t vmid)
350 struct radeon_device *rdev = get_radeon_device(kgd);
351 uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);
353 mutex_lock(&rdev->srbm_mutex);
354 write_register(kgd, SRBM_GFX_CNTL, value);
357 static void unlock_srbm(struct kgd_dev *kgd)
359 struct radeon_device *rdev = get_radeon_device(kgd);
361 write_register(kgd, SRBM_GFX_CNTL, 0);
362 mutex_unlock(&rdev->srbm_mutex);
365 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
368 uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;
369 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
371 lock_srbm(kgd, mec, pipe, queue_id, 0);
374 static void release_queue(struct kgd_dev *kgd)
379 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
380 uint32_t sh_mem_config,
381 uint32_t sh_mem_ape1_base,
382 uint32_t sh_mem_ape1_limit,
383 uint32_t sh_mem_bases)
385 lock_srbm(kgd, 0, 0, 0, vmid);
387 write_register(kgd, SH_MEM_CONFIG, sh_mem_config);
388 write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);
389 write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);
390 write_register(kgd, SH_MEM_BASES, sh_mem_bases);
395 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
399 * We have to assume that there is no outstanding mapping.
400 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0
401 * because a mapping is in progress or because a mapping finished and
403 * So the protocol is to always wait & clear.
405 uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
406 ATC_VMID_PASID_MAPPING_VALID_MASK;
408 write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),
411 while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &
414 write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);
416 /* Mapping vmid to pasid also for IH block */
417 write_register(kgd, IH_VMID_0_LUT + vmid * sizeof(uint32_t),
423 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
424 uint32_t hpd_size, uint64_t hpd_gpu_addr)
426 uint32_t mec = (pipe_id / CIK_PIPE_PER_MEC) + 1;
427 uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);
429 lock_srbm(kgd, mec, pipe, 0, 0);
430 write_register(kgd, CP_HPD_EOP_BASE_ADDR,
431 lower_32_bits(hpd_gpu_addr >> 8));
432 write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,
433 upper_32_bits(hpd_gpu_addr >> 8));
434 write_register(kgd, CP_HPD_EOP_VMID, 0);
435 write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);
441 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
446 mec = (pipe_id / CIK_PIPE_PER_MEC) + 1;
447 pipe = (pipe_id % CIK_PIPE_PER_MEC);
449 lock_srbm(kgd, mec, pipe, 0, 0);
451 write_register(kgd, CPC_INT_CNTL,
452 TIME_STAMP_INT_ENABLE | OPCODE_ERROR_INT_ENABLE);
459 static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
463 retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
464 m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
466 pr_debug("kfd: sdma base address: 0x%x\n", retval);
471 static inline struct cik_mqd *get_mqd(void *mqd)
473 return (struct cik_mqd *)mqd;
476 static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
478 return (struct cik_sdma_rlc_registers *)mqd;
481 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
482 uint32_t queue_id, uint32_t __user *wptr)
484 uint32_t wptr_shadow, is_wptr_shadow_valid;
489 is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
491 acquire_queue(kgd, pipe_id, queue_id);
492 write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);
493 write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);
494 write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);
496 write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);
497 write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);
498 write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);
500 write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);
501 write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);
502 write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);
504 write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);
506 write_register(kgd, CP_HQD_PERSISTENT_STATE,
507 m->cp_hqd_persistent_state);
508 write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);
509 write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);
511 write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,
512 m->cp_hqd_atomic0_preop_lo);
514 write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,
515 m->cp_hqd_atomic0_preop_hi);
517 write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,
518 m->cp_hqd_atomic1_preop_lo);
520 write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,
521 m->cp_hqd_atomic1_preop_hi);
523 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,
524 m->cp_hqd_pq_rptr_report_addr_lo);
526 write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
527 m->cp_hqd_pq_rptr_report_addr_hi);
529 write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);
531 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,
532 m->cp_hqd_pq_wptr_poll_addr_lo);
534 write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,
535 m->cp_hqd_pq_wptr_poll_addr_hi);
537 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,
538 m->cp_hqd_pq_doorbell_control);
540 write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);
542 write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);
544 write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);
545 write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);
547 write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);
549 if (is_wptr_shadow_valid)
550 write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);
552 write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);
558 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
560 struct cik_sdma_rlc_registers *m;
561 uint32_t sdma_base_addr;
563 m = get_sdma_mqd(mqd);
564 sdma_base_addr = get_sdma_base_addr(m);
567 sdma_base_addr + SDMA0_RLC0_VIRTUAL_ADDR,
568 m->sdma_rlc_virtual_addr);
571 sdma_base_addr + SDMA0_RLC0_RB_BASE,
572 m->sdma_rlc_rb_base);
575 sdma_base_addr + SDMA0_RLC0_RB_BASE_HI,
576 m->sdma_rlc_rb_base_hi);
579 sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_LO,
580 m->sdma_rlc_rb_rptr_addr_lo);
583 sdma_base_addr + SDMA0_RLC0_RB_RPTR_ADDR_HI,
584 m->sdma_rlc_rb_rptr_addr_hi);
587 sdma_base_addr + SDMA0_RLC0_DOORBELL,
588 m->sdma_rlc_doorbell);
591 sdma_base_addr + SDMA0_RLC0_RB_CNTL,
592 m->sdma_rlc_rb_cntl);
597 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
598 uint32_t pipe_id, uint32_t queue_id)
604 acquire_queue(kgd, pipe_id, queue_id);
605 act = read_register(kgd, CP_HQD_ACTIVE);
607 low = lower_32_bits(queue_address >> 8);
608 high = upper_32_bits(queue_address >> 8);
610 if (low == read_register(kgd, CP_HQD_PQ_BASE) &&
611 high == read_register(kgd, CP_HQD_PQ_BASE_HI))
618 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
620 struct cik_sdma_rlc_registers *m;
621 uint32_t sdma_base_addr;
622 uint32_t sdma_rlc_rb_cntl;
624 m = get_sdma_mqd(mqd);
625 sdma_base_addr = get_sdma_base_addr(m);
627 sdma_rlc_rb_cntl = read_register(kgd,
628 sdma_base_addr + SDMA0_RLC0_RB_CNTL);
630 if (sdma_rlc_rb_cntl & SDMA_RB_ENABLE)
636 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
637 unsigned int timeout, uint32_t pipe_id,
642 acquire_queue(kgd, pipe_id, queue_id);
643 write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);
645 write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);
648 temp = read_register(kgd, CP_HQD_ACTIVE);
652 pr_err("kfd: cp queue preemption time out (%dms)\n",
665 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
666 unsigned int timeout)
668 struct cik_sdma_rlc_registers *m;
669 uint32_t sdma_base_addr;
672 m = get_sdma_mqd(mqd);
673 sdma_base_addr = get_sdma_base_addr(m);
675 temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);
676 temp = temp & ~SDMA_RB_ENABLE;
677 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);
680 temp = read_register(kgd, sdma_base_addr +
681 SDMA0_RLC0_CONTEXT_STATUS);
682 if (temp & SDMA_RLC_IDLE)
690 write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);
691 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);
692 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);
693 write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);
698 static int kgd_address_watch_disable(struct kgd_dev *kgd)
700 union TCP_WATCH_CNTL_BITS cntl;
705 cntl.bitfields.valid = 0;
706 cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK;
707 cntl.bitfields.atc = 1;
709 /* Turning off this address until we set all the registers */
710 for (i = 0; i < MAX_WATCH_ADDRESSES; i++)
712 watchRegs[i * ADDRESS_WATCH_REG_MAX +
713 ADDRESS_WATCH_REG_CNTL],
719 static int kgd_address_watch_execute(struct kgd_dev *kgd,
720 unsigned int watch_point_id,
725 union TCP_WATCH_CNTL_BITS cntl;
727 cntl.u32All = cntl_val;
729 /* Turning off this watch point until we set all the registers */
730 cntl.bitfields.valid = 0;
732 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
733 ADDRESS_WATCH_REG_CNTL],
737 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
738 ADDRESS_WATCH_REG_ADDR_HI],
742 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
743 ADDRESS_WATCH_REG_ADDR_LO],
746 /* Enable the watch point */
747 cntl.bitfields.valid = 1;
750 watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX +
751 ADDRESS_WATCH_REG_CNTL],
757 static int kgd_wave_control_execute(struct kgd_dev *kgd,
758 uint32_t gfx_index_val,
761 struct radeon_device *rdev = get_radeon_device(kgd);
764 mutex_lock(&rdev->grbm_idx_mutex);
766 write_register(kgd, GRBM_GFX_INDEX, gfx_index_val);
767 write_register(kgd, SQ_CMD, sq_cmd);
769 /* Restore the GRBM_GFX_INDEX register */
771 data = INSTANCE_BROADCAST_WRITES | SH_BROADCAST_WRITES |
774 write_register(kgd, GRBM_GFX_INDEX, data);
776 mutex_unlock(&rdev->grbm_idx_mutex);
781 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
782 unsigned int watch_point_id,
783 unsigned int reg_offset)
785 return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
788 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid)
791 struct radeon_device *rdev = (struct radeon_device *) kgd;
793 reg = RREG32(ATC_VMID0_PASID_MAPPING + vmid*4);
794 return reg & ATC_VMID_PASID_MAPPING_VALID_MASK;
797 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
801 struct radeon_device *rdev = (struct radeon_device *) kgd;
803 reg = RREG32(ATC_VMID0_PASID_MAPPING + vmid*4);
804 return reg & ATC_VMID_PASID_MAPPING_PASID_MASK;
807 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
809 struct radeon_device *rdev = (struct radeon_device *) kgd;
811 return WREG32(VM_INVALIDATE_REQUEST, 1 << vmid);
814 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
816 struct radeon_device *rdev = (struct radeon_device *) kgd;
817 const union radeon_firmware_header *hdr;
819 BUG_ON(kgd == NULL || rdev->mec_fw == NULL);
823 hdr = (const union radeon_firmware_header *) rdev->pfp_fw->data;
827 hdr = (const union radeon_firmware_header *) rdev->me_fw->data;
831 hdr = (const union radeon_firmware_header *) rdev->ce_fw->data;
834 case KGD_ENGINE_MEC1:
835 hdr = (const union radeon_firmware_header *) rdev->mec_fw->data;
838 case KGD_ENGINE_MEC2:
839 hdr = (const union radeon_firmware_header *)
844 hdr = (const union radeon_firmware_header *) rdev->rlc_fw->data;
847 case KGD_ENGINE_SDMA1:
848 case KGD_ENGINE_SDMA2:
849 hdr = (const union radeon_firmware_header *)
860 /* Only 12 bit in use*/
861 return hdr->common.ucode_version;