drm/amdgpu: Allow vblank_disable_immediate.
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_device.c
CommitLineData
d38ceaf9
AD
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
0875dc9e 28#include <linux/kthread.h>
d38ceaf9
AD
29#include <linux/console.h>
30#include <linux/slab.h>
31#include <linux/debugfs.h>
32#include <drm/drmP.h>
33#include <drm/drm_crtc_helper.h>
34#include <drm/amdgpu_drm.h>
35#include <linux/vgaarb.h>
36#include <linux/vga_switcheroo.h>
37#include <linux/efi.h>
38#include "amdgpu.h"
f4b373f4 39#include "amdgpu_trace.h"
d38ceaf9
AD
40#include "amdgpu_i2c.h"
41#include "atom.h"
42#include "amdgpu_atombios.h"
a5bde2f9 43#include "amdgpu_atomfirmware.h"
d0dd7f0c 44#include "amd_pcie.h"
33f34802
KW
45#ifdef CONFIG_DRM_AMDGPU_SI
46#include "si.h"
47#endif
a2e73f56
AD
48#ifdef CONFIG_DRM_AMDGPU_CIK
49#include "cik.h"
50#endif
aaa36a97 51#include "vi.h"
460826e6 52#include "soc15.h"
d38ceaf9 53#include "bif/bif_4_1_d.h"
9accf2fd 54#include <linux/pci.h>
bec86378 55#include <linux/firmware.h>
d38ceaf9 56
e2a75f88 57MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
2d2e5e7e 58MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
e2a75f88 59
2dc80b00
S
60#define AMDGPU_RESUME_MS 2000
61
d38ceaf9
AD
62static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
63static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
4f0955fc 64static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
d38ceaf9
AD
65
66static const char *amdgpu_asic_name[] = {
da69c161
KW
67 "TAHITI",
68 "PITCAIRN",
69 "VERDE",
70 "OLAND",
71 "HAINAN",
d38ceaf9
AD
72 "BONAIRE",
73 "KAVERI",
74 "KABINI",
75 "HAWAII",
76 "MULLINS",
77 "TOPAZ",
78 "TONGA",
48299f95 79 "FIJI",
d38ceaf9 80 "CARRIZO",
139f4917 81 "STONEY",
2cc0c0b5
FC
82 "POLARIS10",
83 "POLARIS11",
c4642a47 84 "POLARIS12",
d4196f01 85 "VEGA10",
2ca8a5d2 86 "RAVEN",
d38ceaf9
AD
87 "LAST",
88};
89
90bool amdgpu_device_is_px(struct drm_device *dev)
91{
92 struct amdgpu_device *adev = dev->dev_private;
93
2f7d10b3 94 if (adev->flags & AMD_IS_PX)
d38ceaf9
AD
95 return true;
96 return false;
97}
98
99/*
100 * MMIO register access helper functions.
101 */
102uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
15d72fd7 103 uint32_t acc_flags)
d38ceaf9 104{
f4b373f4
TSD
105 uint32_t ret;
106
15d72fd7 107 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
bc992ba5
XY
108 BUG_ON(in_interrupt());
109 return amdgpu_virt_kiq_rreg(adev, reg);
110 }
111
15d72fd7 112 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
f4b373f4 113 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
d38ceaf9
AD
114 else {
115 unsigned long flags;
d38ceaf9
AD
116
117 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
118 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
119 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
120 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
d38ceaf9 121 }
f4b373f4
TSD
122 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
123 return ret;
d38ceaf9
AD
124}
125
126void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
15d72fd7 127 uint32_t acc_flags)
d38ceaf9 128{
f4b373f4 129 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
4e99a44e 130
15d72fd7 131 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
bc992ba5
XY
132 BUG_ON(in_interrupt());
133 return amdgpu_virt_kiq_wreg(adev, reg, v);
134 }
135
15d72fd7 136 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
d38ceaf9
AD
137 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
138 else {
139 unsigned long flags;
140
141 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
142 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
143 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
144 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
145 }
146}
147
148u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
149{
150 if ((reg * 4) < adev->rio_mem_size)
151 return ioread32(adev->rio_mem + (reg * 4));
152 else {
153 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
154 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
155 }
156}
157
158void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
159{
160
161 if ((reg * 4) < adev->rio_mem_size)
162 iowrite32(v, adev->rio_mem + (reg * 4));
163 else {
164 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
165 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
166 }
167}
168
169/**
170 * amdgpu_mm_rdoorbell - read a doorbell dword
171 *
172 * @adev: amdgpu_device pointer
173 * @index: doorbell index
174 *
175 * Returns the value in the doorbell aperture at the
176 * requested doorbell index (CIK).
177 */
178u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
179{
180 if (index < adev->doorbell.num_doorbells) {
181 return readl(adev->doorbell.ptr + index);
182 } else {
183 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
184 return 0;
185 }
186}
187
188/**
189 * amdgpu_mm_wdoorbell - write a doorbell dword
190 *
191 * @adev: amdgpu_device pointer
192 * @index: doorbell index
193 * @v: value to write
194 *
195 * Writes @v to the doorbell aperture at the
196 * requested doorbell index (CIK).
197 */
198void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
199{
200 if (index < adev->doorbell.num_doorbells) {
201 writel(v, adev->doorbell.ptr + index);
202 } else {
203 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
204 }
205}
206
832be404
KW
207/**
208 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
209 *
210 * @adev: amdgpu_device pointer
211 * @index: doorbell index
212 *
213 * Returns the value in the doorbell aperture at the
214 * requested doorbell index (VEGA10+).
215 */
216u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
217{
218 if (index < adev->doorbell.num_doorbells) {
219 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
220 } else {
221 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
222 return 0;
223 }
224}
225
226/**
227 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
228 *
229 * @adev: amdgpu_device pointer
230 * @index: doorbell index
231 * @v: value to write
232 *
233 * Writes @v to the doorbell aperture at the
234 * requested doorbell index (VEGA10+).
235 */
236void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
237{
238 if (index < adev->doorbell.num_doorbells) {
239 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
240 } else {
241 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
242 }
243}
244
d38ceaf9
AD
245/**
246 * amdgpu_invalid_rreg - dummy reg read function
247 *
248 * @adev: amdgpu device pointer
249 * @reg: offset of register
250 *
251 * Dummy register read function. Used for register blocks
252 * that certain asics don't have (all asics).
253 * Returns the value in the register.
254 */
255static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
256{
257 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
258 BUG();
259 return 0;
260}
261
262/**
263 * amdgpu_invalid_wreg - dummy reg write function
264 *
265 * @adev: amdgpu device pointer
266 * @reg: offset of register
267 * @v: value to write to the register
268 *
269 * Dummy register read function. Used for register blocks
270 * that certain asics don't have (all asics).
271 */
272static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
273{
274 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
275 reg, v);
276 BUG();
277}
278
279/**
280 * amdgpu_block_invalid_rreg - dummy reg read function
281 *
282 * @adev: amdgpu device pointer
283 * @block: offset of instance
284 * @reg: offset of register
285 *
286 * Dummy register read function. Used for register blocks
287 * that certain asics don't have (all asics).
288 * Returns the value in the register.
289 */
290static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
291 uint32_t block, uint32_t reg)
292{
293 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
294 reg, block);
295 BUG();
296 return 0;
297}
298
299/**
300 * amdgpu_block_invalid_wreg - dummy reg write function
301 *
302 * @adev: amdgpu device pointer
303 * @block: offset of instance
304 * @reg: offset of register
305 * @v: value to write to the register
306 *
307 * Dummy register read function. Used for register blocks
308 * that certain asics don't have (all asics).
309 */
310static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
311 uint32_t block,
312 uint32_t reg, uint32_t v)
313{
314 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
315 reg, block, v);
316 BUG();
317}
318
319static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
320{
321 int r;
322
323 if (adev->vram_scratch.robj == NULL) {
324 r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
857d913d 325 PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
03f48dd5
CK
326 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
327 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
72d7668b 328 NULL, NULL, &adev->vram_scratch.robj);
d38ceaf9
AD
329 if (r) {
330 return r;
331 }
332 }
333
334 r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
335 if (unlikely(r != 0))
336 return r;
337 r = amdgpu_bo_pin(adev->vram_scratch.robj,
338 AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
339 if (r) {
340 amdgpu_bo_unreserve(adev->vram_scratch.robj);
341 return r;
342 }
343 r = amdgpu_bo_kmap(adev->vram_scratch.robj,
344 (void **)&adev->vram_scratch.ptr);
345 if (r)
346 amdgpu_bo_unpin(adev->vram_scratch.robj);
347 amdgpu_bo_unreserve(adev->vram_scratch.robj);
348
349 return r;
350}
351
352static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
353{
354 int r;
355
356 if (adev->vram_scratch.robj == NULL) {
357 return;
358 }
8ab25b4f 359 r = amdgpu_bo_reserve(adev->vram_scratch.robj, true);
d38ceaf9
AD
360 if (likely(r == 0)) {
361 amdgpu_bo_kunmap(adev->vram_scratch.robj);
362 amdgpu_bo_unpin(adev->vram_scratch.robj);
363 amdgpu_bo_unreserve(adev->vram_scratch.robj);
364 }
365 amdgpu_bo_unref(&adev->vram_scratch.robj);
366}
367
368/**
369 * amdgpu_program_register_sequence - program an array of registers.
370 *
371 * @adev: amdgpu_device pointer
372 * @registers: pointer to the register array
373 * @array_size: size of the register array
374 *
375 * Programs an array or registers with and and or masks.
376 * This is a helper for setting golden registers.
377 */
378void amdgpu_program_register_sequence(struct amdgpu_device *adev,
379 const u32 *registers,
380 const u32 array_size)
381{
382 u32 tmp, reg, and_mask, or_mask;
383 int i;
384
385 if (array_size % 3)
386 return;
387
388 for (i = 0; i < array_size; i +=3) {
389 reg = registers[i + 0];
390 and_mask = registers[i + 1];
391 or_mask = registers[i + 2];
392
393 if (and_mask == 0xffffffff) {
394 tmp = or_mask;
395 } else {
396 tmp = RREG32(reg);
397 tmp &= ~and_mask;
398 tmp |= or_mask;
399 }
400 WREG32(reg, tmp);
401 }
402}
403
404void amdgpu_pci_config_reset(struct amdgpu_device *adev)
405{
406 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
407}
408
409/*
410 * GPU doorbell aperture helpers function.
411 */
412/**
413 * amdgpu_doorbell_init - Init doorbell driver information.
414 *
415 * @adev: amdgpu_device pointer
416 *
417 * Init doorbell driver information (CIK)
418 * Returns 0 on success, error on failure.
419 */
420static int amdgpu_doorbell_init(struct amdgpu_device *adev)
421{
422 /* doorbell bar mapping */
423 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
424 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
425
edf600da 426 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
d38ceaf9
AD
427 AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
428 if (adev->doorbell.num_doorbells == 0)
429 return -EINVAL;
430
8972e5d2
CK
431 adev->doorbell.ptr = ioremap(adev->doorbell.base,
432 adev->doorbell.num_doorbells *
433 sizeof(u32));
434 if (adev->doorbell.ptr == NULL)
d38ceaf9 435 return -ENOMEM;
d38ceaf9
AD
436
437 return 0;
438}
439
440/**
441 * amdgpu_doorbell_fini - Tear down doorbell driver information.
442 *
443 * @adev: amdgpu_device pointer
444 *
445 * Tear down doorbell driver information (CIK)
446 */
447static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
448{
449 iounmap(adev->doorbell.ptr);
450 adev->doorbell.ptr = NULL;
451}
452
453/**
454 * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
455 * setup amdkfd
456 *
457 * @adev: amdgpu_device pointer
458 * @aperture_base: output returning doorbell aperture base physical address
459 * @aperture_size: output returning doorbell aperture size in bytes
460 * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
461 *
462 * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
463 * takes doorbells required for its own rings and reports the setup to amdkfd.
464 * amdgpu reserved doorbells are at the start of the doorbell aperture.
465 */
466void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
467 phys_addr_t *aperture_base,
468 size_t *aperture_size,
469 size_t *start_offset)
470{
471 /*
472 * The first num_doorbells are used by amdgpu.
473 * amdkfd takes whatever's left in the aperture.
474 */
475 if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
476 *aperture_base = adev->doorbell.base;
477 *aperture_size = adev->doorbell.size;
478 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
479 } else {
480 *aperture_base = 0;
481 *aperture_size = 0;
482 *start_offset = 0;
483 }
484}
485
486/*
487 * amdgpu_wb_*()
455a7bc2 488 * Writeback is the method by which the GPU updates special pages in memory
ea81a173 489 * with the status of certain GPU events (fences, ring pointers,etc.).
d38ceaf9
AD
490 */
491
492/**
493 * amdgpu_wb_fini - Disable Writeback and free memory
494 *
495 * @adev: amdgpu_device pointer
496 *
497 * Disables Writeback and frees the Writeback memory (all asics).
498 * Used at driver shutdown.
499 */
500static void amdgpu_wb_fini(struct amdgpu_device *adev)
501{
502 if (adev->wb.wb_obj) {
a76ed485
AD
503 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
504 &adev->wb.gpu_addr,
505 (void **)&adev->wb.wb);
d38ceaf9
AD
506 adev->wb.wb_obj = NULL;
507 }
508}
509
510/**
511 * amdgpu_wb_init- Init Writeback driver info and allocate memory
512 *
513 * @adev: amdgpu_device pointer
514 *
455a7bc2 515 * Initializes writeback and allocates writeback memory (all asics).
d38ceaf9
AD
516 * Used at driver startup.
517 * Returns 0 on success or an -error on failure.
518 */
519static int amdgpu_wb_init(struct amdgpu_device *adev)
520{
521 int r;
522
523 if (adev->wb.wb_obj == NULL) {
60a970a6 524 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t),
a76ed485
AD
525 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
526 &adev->wb.wb_obj, &adev->wb.gpu_addr,
527 (void **)&adev->wb.wb);
d38ceaf9
AD
528 if (r) {
529 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
530 return r;
531 }
d38ceaf9
AD
532
533 adev->wb.num_wb = AMDGPU_MAX_WB;
534 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
535
536 /* clear wb memory */
60a970a6 537 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
d38ceaf9
AD
538 }
539
540 return 0;
541}
542
543/**
544 * amdgpu_wb_get - Allocate a wb entry
545 *
546 * @adev: amdgpu_device pointer
547 * @wb: wb index
548 *
549 * Allocate a wb slot for use by the driver (all asics).
550 * Returns 0 on success or -EINVAL on failure.
551 */
552int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
553{
554 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
555 if (offset < adev->wb.num_wb) {
556 __set_bit(offset, adev->wb.used);
557 *wb = offset;
558 return 0;
559 } else {
560 return -EINVAL;
561 }
562}
563
7014285a
KW
564/**
565 * amdgpu_wb_get_64bit - Allocate a wb entry
566 *
567 * @adev: amdgpu_device pointer
568 * @wb: wb index
569 *
570 * Allocate a wb slot for use by the driver (all asics).
571 * Returns 0 on success or -EINVAL on failure.
572 */
573int amdgpu_wb_get_64bit(struct amdgpu_device *adev, u32 *wb)
574{
575 unsigned long offset = bitmap_find_next_zero_area_off(adev->wb.used,
576 adev->wb.num_wb, 0, 2, 7, 0);
577 if ((offset + 1) < adev->wb.num_wb) {
578 __set_bit(offset, adev->wb.used);
579 __set_bit(offset + 1, adev->wb.used);
580 *wb = offset;
581 return 0;
582 } else {
583 return -EINVAL;
584 }
585}
586
d38ceaf9
AD
587/**
588 * amdgpu_wb_free - Free a wb entry
589 *
590 * @adev: amdgpu_device pointer
591 * @wb: wb index
592 *
593 * Free a wb slot allocated for use by the driver (all asics)
594 */
595void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
596{
597 if (wb < adev->wb.num_wb)
598 __clear_bit(wb, adev->wb.used);
599}
600
7014285a
KW
601/**
602 * amdgpu_wb_free_64bit - Free a wb entry
603 *
604 * @adev: amdgpu_device pointer
605 * @wb: wb index
606 *
607 * Free a wb slot allocated for use by the driver (all asics)
608 */
609void amdgpu_wb_free_64bit(struct amdgpu_device *adev, u32 wb)
610{
611 if ((wb + 1) < adev->wb.num_wb) {
612 __clear_bit(wb, adev->wb.used);
613 __clear_bit(wb + 1, adev->wb.used);
614 }
615}
616
d38ceaf9
AD
617/**
618 * amdgpu_vram_location - try to find VRAM location
619 * @adev: amdgpu device structure holding all necessary informations
620 * @mc: memory controller structure holding memory informations
621 * @base: base address at which to put VRAM
622 *
455a7bc2 623 * Function will try to place VRAM at base address provided
d38ceaf9
AD
624 * as parameter (which is so far either PCI aperture address or
625 * for IGP TOM base address).
626 *
627 * If there is not enough space to fit the unvisible VRAM in the 32bits
628 * address space then we limit the VRAM size to the aperture.
629 *
630 * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
631 * this shouldn't be a problem as we are using the PCI aperture as a reference.
632 * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
633 * not IGP.
634 *
635 * Note: we use mc_vram_size as on some board we need to program the mc to
636 * cover the whole aperture even if VRAM size is inferior to aperture size
637 * Novell bug 204882 + along with lots of ubuntu ones
638 *
639 * Note: when limiting vram it's safe to overwritte real_vram_size because
640 * we are not in case where real_vram_size is inferior to mc_vram_size (ie
641 * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
642 * ones)
643 *
644 * Note: IGP TOM addr should be the same as the aperture addr, we don't
455a7bc2 645 * explicitly check for that though.
d38ceaf9
AD
646 *
647 * FIXME: when reducing VRAM size align new size on power of 2.
648 */
649void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
650{
651 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
652
653 mc->vram_start = base;
654 if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
655 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
656 mc->real_vram_size = mc->aper_size;
657 mc->mc_vram_size = mc->aper_size;
658 }
659 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
660 if (limit && limit < mc->real_vram_size)
661 mc->real_vram_size = limit;
662 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
663 mc->mc_vram_size >> 20, mc->vram_start,
664 mc->vram_end, mc->real_vram_size >> 20);
665}
666
667/**
668 * amdgpu_gtt_location - try to find GTT location
669 * @adev: amdgpu device structure holding all necessary informations
670 * @mc: memory controller structure holding memory informations
671 *
672 * Function will place try to place GTT before or after VRAM.
673 *
674 * If GTT size is bigger than space left then we ajust GTT size.
675 * Thus function will never fails.
676 *
677 * FIXME: when reducing GTT size align new size on power of 2.
678 */
679void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
680{
681 u64 size_af, size_bf;
682
683 size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
684 size_bf = mc->vram_start & ~mc->gtt_base_align;
685 if (size_bf > size_af) {
686 if (mc->gtt_size > size_bf) {
687 dev_warn(adev->dev, "limiting GTT\n");
688 mc->gtt_size = size_bf;
689 }
9dc5a91e 690 mc->gtt_start = 0;
d38ceaf9
AD
691 } else {
692 if (mc->gtt_size > size_af) {
693 dev_warn(adev->dev, "limiting GTT\n");
694 mc->gtt_size = size_af;
695 }
696 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
697 }
698 mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
699 dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
700 mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
701}
702
703/*
704 * GPU helpers function.
705 */
706/**
c836fec5 707 * amdgpu_need_post - check if the hw need post or not
d38ceaf9
AD
708 *
709 * @adev: amdgpu_device pointer
710 *
c836fec5
JQ
711 * Check if the asic has been initialized (all asics) at driver startup
712 * or post is needed if hw reset is performed.
713 * Returns true if need or false if not.
d38ceaf9 714 */
c836fec5 715bool amdgpu_need_post(struct amdgpu_device *adev)
d38ceaf9
AD
716{
717 uint32_t reg;
718
c836fec5
JQ
719 if (adev->has_hw_reset) {
720 adev->has_hw_reset = false;
721 return true;
722 }
d38ceaf9 723 /* then check MEM_SIZE, in case the crtcs are off */
bbf282d8 724 reg = amdgpu_asic_get_config_memsize(adev);
d38ceaf9 725
f2713e8c 726 if ((reg != 0) && (reg != 0xffffffff))
c836fec5 727 return false;
d38ceaf9 728
c836fec5 729 return true;
d38ceaf9
AD
730
731}
732
bec86378
ML
733static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
734{
735 if (amdgpu_sriov_vf(adev))
736 return false;
737
738 if (amdgpu_passthrough(adev)) {
1da2c326
ML
739 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
740 * some old smc fw still need driver do vPost otherwise gpu hang, while
741 * those smc fw version above 22.15 doesn't have this flaw, so we force
742 * vpost executed for smc version below 22.15
bec86378
ML
743 */
744 if (adev->asic_type == CHIP_FIJI) {
745 int err;
746 uint32_t fw_ver;
747 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
748 /* force vPost if error occured */
749 if (err)
750 return true;
751
752 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1da2c326
ML
753 if (fw_ver < 0x00160e00)
754 return true;
bec86378 755 }
bec86378 756 }
c836fec5 757 return amdgpu_need_post(adev);
bec86378
ML
758}
759
d38ceaf9
AD
760/**
761 * amdgpu_dummy_page_init - init dummy page used by the driver
762 *
763 * @adev: amdgpu_device pointer
764 *
765 * Allocate the dummy page used by the driver (all asics).
766 * This dummy page is used by the driver as a filler for gart entries
767 * when pages are taken out of the GART
768 * Returns 0 on sucess, -ENOMEM on failure.
769 */
770int amdgpu_dummy_page_init(struct amdgpu_device *adev)
771{
772 if (adev->dummy_page.page)
773 return 0;
774 adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
775 if (adev->dummy_page.page == NULL)
776 return -ENOMEM;
777 adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
778 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
779 if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
780 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
781 __free_page(adev->dummy_page.page);
782 adev->dummy_page.page = NULL;
783 return -ENOMEM;
784 }
785 return 0;
786}
787
788/**
789 * amdgpu_dummy_page_fini - free dummy page used by the driver
790 *
791 * @adev: amdgpu_device pointer
792 *
793 * Frees the dummy page used by the driver (all asics).
794 */
795void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
796{
797 if (adev->dummy_page.page == NULL)
798 return;
799 pci_unmap_page(adev->pdev, adev->dummy_page.addr,
800 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
801 __free_page(adev->dummy_page.page);
802 adev->dummy_page.page = NULL;
803}
804
805
806/* ATOM accessor methods */
807/*
808 * ATOM is an interpreted byte code stored in tables in the vbios. The
809 * driver registers callbacks to access registers and the interpreter
810 * in the driver parses the tables and executes then to program specific
811 * actions (set display modes, asic init, etc.). See amdgpu_atombios.c,
812 * atombios.h, and atom.c
813 */
814
815/**
816 * cail_pll_read - read PLL register
817 *
818 * @info: atom card_info pointer
819 * @reg: PLL register offset
820 *
821 * Provides a PLL register accessor for the atom interpreter (r4xx+).
822 * Returns the value of the PLL register.
823 */
824static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
825{
826 return 0;
827}
828
829/**
830 * cail_pll_write - write PLL register
831 *
832 * @info: atom card_info pointer
833 * @reg: PLL register offset
834 * @val: value to write to the pll register
835 *
836 * Provides a PLL register accessor for the atom interpreter (r4xx+).
837 */
838static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
839{
840
841}
842
843/**
844 * cail_mc_read - read MC (Memory Controller) register
845 *
846 * @info: atom card_info pointer
847 * @reg: MC register offset
848 *
849 * Provides an MC register accessor for the atom interpreter (r4xx+).
850 * Returns the value of the MC register.
851 */
852static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
853{
854 return 0;
855}
856
857/**
858 * cail_mc_write - write MC (Memory Controller) register
859 *
860 * @info: atom card_info pointer
861 * @reg: MC register offset
862 * @val: value to write to the pll register
863 *
864 * Provides a MC register accessor for the atom interpreter (r4xx+).
865 */
866static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
867{
868
869}
870
871/**
872 * cail_reg_write - write MMIO register
873 *
874 * @info: atom card_info pointer
875 * @reg: MMIO register offset
876 * @val: value to write to the pll register
877 *
878 * Provides a MMIO register accessor for the atom interpreter (r4xx+).
879 */
880static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
881{
882 struct amdgpu_device *adev = info->dev->dev_private;
883
884 WREG32(reg, val);
885}
886
887/**
888 * cail_reg_read - read MMIO register
889 *
890 * @info: atom card_info pointer
891 * @reg: MMIO register offset
892 *
893 * Provides an MMIO register accessor for the atom interpreter (r4xx+).
894 * Returns the value of the MMIO register.
895 */
896static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
897{
898 struct amdgpu_device *adev = info->dev->dev_private;
899 uint32_t r;
900
901 r = RREG32(reg);
902 return r;
903}
904
905/**
906 * cail_ioreg_write - write IO register
907 *
908 * @info: atom card_info pointer
909 * @reg: IO register offset
910 * @val: value to write to the pll register
911 *
912 * Provides a IO register accessor for the atom interpreter (r4xx+).
913 */
914static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
915{
916 struct amdgpu_device *adev = info->dev->dev_private;
917
918 WREG32_IO(reg, val);
919}
920
921/**
922 * cail_ioreg_read - read IO register
923 *
924 * @info: atom card_info pointer
925 * @reg: IO register offset
926 *
927 * Provides an IO register accessor for the atom interpreter (r4xx+).
928 * Returns the value of the IO register.
929 */
930static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
931{
932 struct amdgpu_device *adev = info->dev->dev_private;
933 uint32_t r;
934
935 r = RREG32_IO(reg);
936 return r;
937}
938
939/**
940 * amdgpu_atombios_fini - free the driver info and callbacks for atombios
941 *
942 * @adev: amdgpu_device pointer
943 *
944 * Frees the driver info and register access callbacks for the ATOM
945 * interpreter (r4xx+).
946 * Called at driver shutdown.
947 */
948static void amdgpu_atombios_fini(struct amdgpu_device *adev)
949{
89e0ec9f 950 if (adev->mode_info.atom_context) {
d38ceaf9 951 kfree(adev->mode_info.atom_context->scratch);
89e0ec9f
ML
952 kfree(adev->mode_info.atom_context->iio);
953 }
d38ceaf9
AD
954 kfree(adev->mode_info.atom_context);
955 adev->mode_info.atom_context = NULL;
956 kfree(adev->mode_info.atom_card_info);
957 adev->mode_info.atom_card_info = NULL;
958}
959
960/**
961 * amdgpu_atombios_init - init the driver info and callbacks for atombios
962 *
963 * @adev: amdgpu_device pointer
964 *
965 * Initializes the driver info and register access callbacks for the
966 * ATOM interpreter (r4xx+).
967 * Returns 0 on sucess, -ENOMEM on failure.
968 * Called at driver startup.
969 */
970static int amdgpu_atombios_init(struct amdgpu_device *adev)
971{
972 struct card_info *atom_card_info =
973 kzalloc(sizeof(struct card_info), GFP_KERNEL);
974
975 if (!atom_card_info)
976 return -ENOMEM;
977
978 adev->mode_info.atom_card_info = atom_card_info;
979 atom_card_info->dev = adev->ddev;
980 atom_card_info->reg_read = cail_reg_read;
981 atom_card_info->reg_write = cail_reg_write;
982 /* needed for iio ops */
983 if (adev->rio_mem) {
984 atom_card_info->ioreg_read = cail_ioreg_read;
985 atom_card_info->ioreg_write = cail_ioreg_write;
986 } else {
b64a18c5 987 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
d38ceaf9
AD
988 atom_card_info->ioreg_read = cail_reg_read;
989 atom_card_info->ioreg_write = cail_reg_write;
990 }
991 atom_card_info->mc_read = cail_mc_read;
992 atom_card_info->mc_write = cail_mc_write;
993 atom_card_info->pll_read = cail_pll_read;
994 atom_card_info->pll_write = cail_pll_write;
995
996 adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
997 if (!adev->mode_info.atom_context) {
998 amdgpu_atombios_fini(adev);
999 return -ENOMEM;
1000 }
1001
1002 mutex_init(&adev->mode_info.atom_context->mutex);
a5bde2f9
AD
1003 if (adev->is_atom_fw) {
1004 amdgpu_atomfirmware_scratch_regs_init(adev);
1005 amdgpu_atomfirmware_allocate_fb_scratch(adev);
1006 } else {
1007 amdgpu_atombios_scratch_regs_init(adev);
1008 amdgpu_atombios_allocate_fb_scratch(adev);
1009 }
d38ceaf9
AD
1010 return 0;
1011}
1012
1013/* if we get transitioned to only one device, take VGA back */
1014/**
1015 * amdgpu_vga_set_decode - enable/disable vga decode
1016 *
1017 * @cookie: amdgpu_device pointer
1018 * @state: enable/disable vga decode
1019 *
1020 * Enable/disable vga decode (all asics).
1021 * Returns VGA resource flags.
1022 */
1023static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
1024{
1025 struct amdgpu_device *adev = cookie;
1026 amdgpu_asic_set_vga_state(adev, state);
1027 if (state)
1028 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1029 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1030 else
1031 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1032}
1033
bab4fee7 1034static void amdgpu_check_block_size(struct amdgpu_device *adev)
a1adf8be
CZ
1035{
1036 /* defines number of bits in page table versus page directory,
1037 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1038 * page table and the remaining bits are in the page directory */
bab4fee7
JZ
1039 if (amdgpu_vm_block_size == -1)
1040 return;
a1adf8be 1041
bab4fee7 1042 if (amdgpu_vm_block_size < 9) {
a1adf8be
CZ
1043 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1044 amdgpu_vm_block_size);
bab4fee7 1045 goto def_value;
a1adf8be
CZ
1046 }
1047
1048 if (amdgpu_vm_block_size > 24 ||
1049 (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1050 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1051 amdgpu_vm_block_size);
bab4fee7 1052 goto def_value;
a1adf8be 1053 }
bab4fee7
JZ
1054
1055 return;
1056
1057def_value:
1058 amdgpu_vm_block_size = -1;
a1adf8be
CZ
1059}
1060
83ca145d
ZJ
1061static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1062{
64dab074
AD
1063 /* no need to check the default value */
1064 if (amdgpu_vm_size == -1)
1065 return;
1066
76117507 1067 if (!is_power_of_2(amdgpu_vm_size)) {
83ca145d
ZJ
1068 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1069 amdgpu_vm_size);
1070 goto def_value;
1071 }
1072
1073 if (amdgpu_vm_size < 1) {
1074 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1075 amdgpu_vm_size);
1076 goto def_value;
1077 }
1078
1079 /*
1080 * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
1081 */
1082 if (amdgpu_vm_size > 1024) {
1083 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1084 amdgpu_vm_size);
1085 goto def_value;
1086 }
1087
1088 return;
1089
1090def_value:
bab4fee7 1091 amdgpu_vm_size = -1;
83ca145d
ZJ
1092}
1093
d38ceaf9
AD
1094/**
1095 * amdgpu_check_arguments - validate module params
1096 *
1097 * @adev: amdgpu_device pointer
1098 *
1099 * Validates certain module parameters and updates
1100 * the associated values used by the driver (all asics).
1101 */
1102static void amdgpu_check_arguments(struct amdgpu_device *adev)
1103{
5b011235
CZ
1104 if (amdgpu_sched_jobs < 4) {
1105 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1106 amdgpu_sched_jobs);
1107 amdgpu_sched_jobs = 4;
76117507 1108 } else if (!is_power_of_2(amdgpu_sched_jobs)){
5b011235
CZ
1109 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1110 amdgpu_sched_jobs);
1111 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1112 }
d38ceaf9
AD
1113
1114 if (amdgpu_gart_size != -1) {
c4e1a13a 1115 /* gtt size must be greater or equal to 32M */
d38ceaf9
AD
1116 if (amdgpu_gart_size < 32) {
1117 dev_warn(adev->dev, "gart size (%d) too small\n",
1118 amdgpu_gart_size);
1119 amdgpu_gart_size = -1;
d38ceaf9
AD
1120 }
1121 }
1122
83ca145d 1123 amdgpu_check_vm_size(adev);
d38ceaf9 1124
bab4fee7 1125 amdgpu_check_block_size(adev);
6a7f76e7 1126
526bae37 1127 if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
76117507 1128 !is_power_of_2(amdgpu_vram_page_split))) {
6a7f76e7
CK
1129 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1130 amdgpu_vram_page_split);
1131 amdgpu_vram_page_split = 1024;
1132 }
d38ceaf9
AD
1133}
1134
1135/**
1136 * amdgpu_switcheroo_set_state - set switcheroo state
1137 *
1138 * @pdev: pci dev pointer
1694467b 1139 * @state: vga_switcheroo state
d38ceaf9
AD
1140 *
1141 * Callback for the switcheroo driver. Suspends or resumes the
1142 * the asics before or after it is powered up using ACPI methods.
1143 */
1144static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1145{
1146 struct drm_device *dev = pci_get_drvdata(pdev);
1147
1148 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1149 return;
1150
1151 if (state == VGA_SWITCHEROO_ON) {
1152 unsigned d3_delay = dev->pdev->d3_delay;
1153
7ca85295 1154 pr_info("amdgpu: switched on\n");
d38ceaf9
AD
1155 /* don't suspend or resume card normally */
1156 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1157
810ddc3a 1158 amdgpu_device_resume(dev, true, true);
d38ceaf9
AD
1159
1160 dev->pdev->d3_delay = d3_delay;
1161
1162 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1163 drm_kms_helper_poll_enable(dev);
1164 } else {
7ca85295 1165 pr_info("amdgpu: switched off\n");
d38ceaf9
AD
1166 drm_kms_helper_poll_disable(dev);
1167 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
810ddc3a 1168 amdgpu_device_suspend(dev, true, true);
d38ceaf9
AD
1169 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1170 }
1171}
1172
1173/**
1174 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1175 *
1176 * @pdev: pci dev pointer
1177 *
1178 * Callback for the switcheroo driver. Check of the switcheroo
1179 * state can be changed.
1180 * Returns true if the state can be changed, false if not.
1181 */
1182static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1183{
1184 struct drm_device *dev = pci_get_drvdata(pdev);
1185
1186 /*
1187 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1188 * locking inversion with the driver load path. And the access here is
1189 * completely racy anyway. So don't bother with locking for now.
1190 */
1191 return dev->open_count == 0;
1192}
1193
1194static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1195 .set_gpu_state = amdgpu_switcheroo_set_state,
1196 .reprobe = NULL,
1197 .can_switch = amdgpu_switcheroo_can_switch,
1198};
1199
1200int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
5fc3aeeb 1201 enum amd_ip_block_type block_type,
1202 enum amd_clockgating_state state)
d38ceaf9
AD
1203{
1204 int i, r = 0;
1205
1206 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1207 if (!adev->ip_blocks[i].status.valid)
9ecbe7f5 1208 continue;
c722865a
RZ
1209 if (adev->ip_blocks[i].version->type != block_type)
1210 continue;
1211 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1212 continue;
1213 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1214 (void *)adev, state);
1215 if (r)
1216 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1217 adev->ip_blocks[i].version->funcs->name, r);
d38ceaf9
AD
1218 }
1219 return r;
1220}
1221
1222int amdgpu_set_powergating_state(struct amdgpu_device *adev,
5fc3aeeb 1223 enum amd_ip_block_type block_type,
1224 enum amd_powergating_state state)
d38ceaf9
AD
1225{
1226 int i, r = 0;
1227
1228 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1229 if (!adev->ip_blocks[i].status.valid)
9ecbe7f5 1230 continue;
c722865a
RZ
1231 if (adev->ip_blocks[i].version->type != block_type)
1232 continue;
1233 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1234 continue;
1235 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1236 (void *)adev, state);
1237 if (r)
1238 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1239 adev->ip_blocks[i].version->funcs->name, r);
d38ceaf9
AD
1240 }
1241 return r;
1242}
1243
6cb2d4e4
HR
1244void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1245{
1246 int i;
1247
1248 for (i = 0; i < adev->num_ip_blocks; i++) {
1249 if (!adev->ip_blocks[i].status.valid)
1250 continue;
1251 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1252 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1253 }
1254}
1255
5dbbb60b
AD
1256int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1257 enum amd_ip_block_type block_type)
1258{
1259 int i, r;
1260
1261 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1262 if (!adev->ip_blocks[i].status.valid)
9ecbe7f5 1263 continue;
a1255107
AD
1264 if (adev->ip_blocks[i].version->type == block_type) {
1265 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
5dbbb60b
AD
1266 if (r)
1267 return r;
1268 break;
1269 }
1270 }
1271 return 0;
1272
1273}
1274
1275bool amdgpu_is_idle(struct amdgpu_device *adev,
1276 enum amd_ip_block_type block_type)
1277{
1278 int i;
1279
1280 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1281 if (!adev->ip_blocks[i].status.valid)
9ecbe7f5 1282 continue;
a1255107
AD
1283 if (adev->ip_blocks[i].version->type == block_type)
1284 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
5dbbb60b
AD
1285 }
1286 return true;
1287
1288}
1289
a1255107
AD
1290struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1291 enum amd_ip_block_type type)
d38ceaf9
AD
1292{
1293 int i;
1294
1295 for (i = 0; i < adev->num_ip_blocks; i++)
a1255107 1296 if (adev->ip_blocks[i].version->type == type)
d38ceaf9
AD
1297 return &adev->ip_blocks[i];
1298
1299 return NULL;
1300}
1301
1302/**
1303 * amdgpu_ip_block_version_cmp
1304 *
1305 * @adev: amdgpu_device pointer
5fc3aeeb 1306 * @type: enum amd_ip_block_type
d38ceaf9
AD
1307 * @major: major version
1308 * @minor: minor version
1309 *
1310 * return 0 if equal or greater
1311 * return 1 if smaller or the ip_block doesn't exist
1312 */
1313int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
5fc3aeeb 1314 enum amd_ip_block_type type,
d38ceaf9
AD
1315 u32 major, u32 minor)
1316{
a1255107 1317 struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
d38ceaf9 1318
a1255107
AD
1319 if (ip_block && ((ip_block->version->major > major) ||
1320 ((ip_block->version->major == major) &&
1321 (ip_block->version->minor >= minor))))
d38ceaf9
AD
1322 return 0;
1323
1324 return 1;
1325}
1326
a1255107
AD
1327/**
1328 * amdgpu_ip_block_add
1329 *
1330 * @adev: amdgpu_device pointer
1331 * @ip_block_version: pointer to the IP to add
1332 *
1333 * Adds the IP block driver information to the collection of IPs
1334 * on the asic.
1335 */
1336int amdgpu_ip_block_add(struct amdgpu_device *adev,
1337 const struct amdgpu_ip_block_version *ip_block_version)
1338{
1339 if (!ip_block_version)
1340 return -EINVAL;
1341
a0bae357
HR
1342 DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
1343 ip_block_version->funcs->name);
1344
a1255107
AD
1345 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1346
1347 return 0;
1348}
1349
483ef985 1350static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
9accf2fd
ED
1351{
1352 adev->enable_virtual_display = false;
1353
1354 if (amdgpu_virtual_display) {
1355 struct drm_device *ddev = adev->ddev;
1356 const char *pci_address_name = pci_name(ddev->pdev);
0f66356d 1357 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
9accf2fd
ED
1358
1359 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1360 pciaddstr_tmp = pciaddstr;
0f66356d
ED
1361 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1362 pciaddname = strsep(&pciaddname_tmp, ",");
967de2a9
YT
1363 if (!strcmp("all", pciaddname)
1364 || !strcmp(pci_address_name, pciaddname)) {
0f66356d
ED
1365 long num_crtc;
1366 int res = -1;
1367
9accf2fd 1368 adev->enable_virtual_display = true;
0f66356d
ED
1369
1370 if (pciaddname_tmp)
1371 res = kstrtol(pciaddname_tmp, 10,
1372 &num_crtc);
1373
1374 if (!res) {
1375 if (num_crtc < 1)
1376 num_crtc = 1;
1377 if (num_crtc > 6)
1378 num_crtc = 6;
1379 adev->mode_info.num_crtc = num_crtc;
1380 } else {
1381 adev->mode_info.num_crtc = 1;
1382 }
9accf2fd
ED
1383 break;
1384 }
1385 }
1386
0f66356d
ED
1387 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1388 amdgpu_virtual_display, pci_address_name,
1389 adev->enable_virtual_display, adev->mode_info.num_crtc);
9accf2fd
ED
1390
1391 kfree(pciaddstr);
1392 }
1393}
1394
e2a75f88
AD
1395static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1396{
e2a75f88
AD
1397 const char *chip_name;
1398 char fw_name[30];
1399 int err;
1400 const struct gpu_info_firmware_header_v1_0 *hdr;
1401
ab4fe3e1
HR
1402 adev->firmware.gpu_info_fw = NULL;
1403
e2a75f88
AD
1404 switch (adev->asic_type) {
1405 case CHIP_TOPAZ:
1406 case CHIP_TONGA:
1407 case CHIP_FIJI:
1408 case CHIP_POLARIS11:
1409 case CHIP_POLARIS10:
1410 case CHIP_POLARIS12:
1411 case CHIP_CARRIZO:
1412 case CHIP_STONEY:
1413#ifdef CONFIG_DRM_AMDGPU_SI
1414 case CHIP_VERDE:
1415 case CHIP_TAHITI:
1416 case CHIP_PITCAIRN:
1417 case CHIP_OLAND:
1418 case CHIP_HAINAN:
1419#endif
1420#ifdef CONFIG_DRM_AMDGPU_CIK
1421 case CHIP_BONAIRE:
1422 case CHIP_HAWAII:
1423 case CHIP_KAVERI:
1424 case CHIP_KABINI:
1425 case CHIP_MULLINS:
1426#endif
1427 default:
1428 return 0;
1429 case CHIP_VEGA10:
1430 chip_name = "vega10";
1431 break;
2d2e5e7e
AD
1432 case CHIP_RAVEN:
1433 chip_name = "raven";
1434 break;
e2a75f88
AD
1435 }
1436
1437 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
ab4fe3e1 1438 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
e2a75f88
AD
1439 if (err) {
1440 dev_err(adev->dev,
1441 "Failed to load gpu_info firmware \"%s\"\n",
1442 fw_name);
1443 goto out;
1444 }
ab4fe3e1 1445 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
e2a75f88
AD
1446 if (err) {
1447 dev_err(adev->dev,
1448 "Failed to validate gpu_info firmware \"%s\"\n",
1449 fw_name);
1450 goto out;
1451 }
1452
ab4fe3e1 1453 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
e2a75f88
AD
1454 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1455
1456 switch (hdr->version_major) {
1457 case 1:
1458 {
1459 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
ab4fe3e1 1460 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
e2a75f88
AD
1461 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1462
b5ab16bf
AD
1463 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1464 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1465 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1466 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
e2a75f88 1467 adev->gfx.config.max_texture_channel_caches =
b5ab16bf
AD
1468 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1469 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1470 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1471 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1472 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
e2a75f88 1473 adev->gfx.config.double_offchip_lds_buf =
b5ab16bf
AD
1474 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1475 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
51fd0370
HZ
1476 adev->gfx.cu_info.max_waves_per_simd =
1477 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1478 adev->gfx.cu_info.max_scratch_slots_per_cu =
1479 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1480 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
e2a75f88
AD
1481 break;
1482 }
1483 default:
1484 dev_err(adev->dev,
1485 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1486 err = -EINVAL;
1487 goto out;
1488 }
1489out:
e2a75f88
AD
1490 return err;
1491}
1492
d38ceaf9
AD
1493static int amdgpu_early_init(struct amdgpu_device *adev)
1494{
aaa36a97 1495 int i, r;
d38ceaf9 1496
483ef985 1497 amdgpu_device_enable_virtual_display(adev);
a6be7570 1498
d38ceaf9 1499 switch (adev->asic_type) {
aaa36a97
AD
1500 case CHIP_TOPAZ:
1501 case CHIP_TONGA:
48299f95 1502 case CHIP_FIJI:
2cc0c0b5
FC
1503 case CHIP_POLARIS11:
1504 case CHIP_POLARIS10:
c4642a47 1505 case CHIP_POLARIS12:
aaa36a97 1506 case CHIP_CARRIZO:
39bb0c92
SL
1507 case CHIP_STONEY:
1508 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
aaa36a97
AD
1509 adev->family = AMDGPU_FAMILY_CZ;
1510 else
1511 adev->family = AMDGPU_FAMILY_VI;
1512
1513 r = vi_set_ip_blocks(adev);
1514 if (r)
1515 return r;
1516 break;
33f34802
KW
1517#ifdef CONFIG_DRM_AMDGPU_SI
1518 case CHIP_VERDE:
1519 case CHIP_TAHITI:
1520 case CHIP_PITCAIRN:
1521 case CHIP_OLAND:
1522 case CHIP_HAINAN:
295d0daf 1523 adev->family = AMDGPU_FAMILY_SI;
33f34802
KW
1524 r = si_set_ip_blocks(adev);
1525 if (r)
1526 return r;
1527 break;
1528#endif
a2e73f56
AD
1529#ifdef CONFIG_DRM_AMDGPU_CIK
1530 case CHIP_BONAIRE:
1531 case CHIP_HAWAII:
1532 case CHIP_KAVERI:
1533 case CHIP_KABINI:
1534 case CHIP_MULLINS:
1535 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1536 adev->family = AMDGPU_FAMILY_CI;
1537 else
1538 adev->family = AMDGPU_FAMILY_KV;
1539
1540 r = cik_set_ip_blocks(adev);
1541 if (r)
1542 return r;
1543 break;
1544#endif
2ca8a5d2
CZ
1545 case CHIP_VEGA10:
1546 case CHIP_RAVEN:
1547 if (adev->asic_type == CHIP_RAVEN)
1548 adev->family = AMDGPU_FAMILY_RV;
1549 else
1550 adev->family = AMDGPU_FAMILY_AI;
460826e6
KW
1551
1552 r = soc15_set_ip_blocks(adev);
1553 if (r)
1554 return r;
1555 break;
d38ceaf9
AD
1556 default:
1557 /* FIXME: not supported yet */
1558 return -EINVAL;
1559 }
1560
e2a75f88
AD
1561 r = amdgpu_device_parse_gpu_info_fw(adev);
1562 if (r)
1563 return r;
1564
3149d9da
XY
1565 if (amdgpu_sriov_vf(adev)) {
1566 r = amdgpu_virt_request_full_gpu(adev, true);
1567 if (r)
1568 return r;
1569 }
1570
d38ceaf9
AD
1571 for (i = 0; i < adev->num_ip_blocks; i++) {
1572 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
ed8cf00c
HR
1573 DRM_ERROR("disabled ip block: %d <%s>\n",
1574 i, adev->ip_blocks[i].version->funcs->name);
a1255107 1575 adev->ip_blocks[i].status.valid = false;
d38ceaf9 1576 } else {
a1255107
AD
1577 if (adev->ip_blocks[i].version->funcs->early_init) {
1578 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2c1a2784 1579 if (r == -ENOENT) {
a1255107 1580 adev->ip_blocks[i].status.valid = false;
2c1a2784 1581 } else if (r) {
a1255107
AD
1582 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1583 adev->ip_blocks[i].version->funcs->name, r);
d38ceaf9 1584 return r;
2c1a2784 1585 } else {
a1255107 1586 adev->ip_blocks[i].status.valid = true;
2c1a2784 1587 }
974e6b64 1588 } else {
a1255107 1589 adev->ip_blocks[i].status.valid = true;
d38ceaf9 1590 }
d38ceaf9
AD
1591 }
1592 }
1593
395d1fb9
NH
1594 adev->cg_flags &= amdgpu_cg_mask;
1595 adev->pg_flags &= amdgpu_pg_mask;
1596
d38ceaf9
AD
1597 return 0;
1598}
1599
1600static int amdgpu_init(struct amdgpu_device *adev)
1601{
1602 int i, r;
1603
1604 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1605 if (!adev->ip_blocks[i].status.valid)
d38ceaf9 1606 continue;
a1255107 1607 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2c1a2784 1608 if (r) {
a1255107
AD
1609 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1610 adev->ip_blocks[i].version->funcs->name, r);
d38ceaf9 1611 return r;
2c1a2784 1612 }
a1255107 1613 adev->ip_blocks[i].status.sw = true;
d38ceaf9 1614 /* need to do gmc hw init early so we can allocate gpu mem */
a1255107 1615 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
d38ceaf9 1616 r = amdgpu_vram_scratch_init(adev);
2c1a2784
AD
1617 if (r) {
1618 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
d38ceaf9 1619 return r;
2c1a2784 1620 }
a1255107 1621 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2c1a2784
AD
1622 if (r) {
1623 DRM_ERROR("hw_init %d failed %d\n", i, r);
d38ceaf9 1624 return r;
2c1a2784 1625 }
d38ceaf9 1626 r = amdgpu_wb_init(adev);
2c1a2784
AD
1627 if (r) {
1628 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
d38ceaf9 1629 return r;
2c1a2784 1630 }
a1255107 1631 adev->ip_blocks[i].status.hw = true;
2493664f
ML
1632
1633 /* right after GMC hw init, we create CSA */
1634 if (amdgpu_sriov_vf(adev)) {
1635 r = amdgpu_allocate_static_csa(adev);
1636 if (r) {
1637 DRM_ERROR("allocate CSA failed %d\n", r);
1638 return r;
1639 }
1640 }
d38ceaf9
AD
1641 }
1642 }
1643
1644 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1645 if (!adev->ip_blocks[i].status.sw)
d38ceaf9
AD
1646 continue;
1647 /* gmc hw init is done early */
a1255107 1648 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
d38ceaf9 1649 continue;
a1255107 1650 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2c1a2784 1651 if (r) {
a1255107
AD
1652 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1653 adev->ip_blocks[i].version->funcs->name, r);
d38ceaf9 1654 return r;
2c1a2784 1655 }
a1255107 1656 adev->ip_blocks[i].status.hw = true;
d38ceaf9
AD
1657 }
1658
1659 return 0;
1660}
1661
0c49e0b8
CZ
1662static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
1663{
1664 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1665}
1666
1667static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
1668{
1669 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1670 AMDGPU_RESET_MAGIC_NUM);
1671}
1672
2dc80b00 1673static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
d38ceaf9
AD
1674{
1675 int i = 0, r;
1676
1677 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1678 if (!adev->ip_blocks[i].status.valid)
d38ceaf9 1679 continue;
4a446d55 1680 /* skip CG for VCE/UVD, it's handled specially */
a1255107
AD
1681 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1682 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
4a446d55 1683 /* enable clockgating to save power */
a1255107
AD
1684 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1685 AMD_CG_STATE_GATE);
4a446d55
AD
1686 if (r) {
1687 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
a1255107 1688 adev->ip_blocks[i].version->funcs->name, r);
4a446d55
AD
1689 return r;
1690 }
b0b00ff1 1691 }
d38ceaf9 1692 }
2dc80b00
S
1693 return 0;
1694}
1695
1696static int amdgpu_late_init(struct amdgpu_device *adev)
1697{
1698 int i = 0, r;
1699
1700 for (i = 0; i < adev->num_ip_blocks; i++) {
1701 if (!adev->ip_blocks[i].status.valid)
1702 continue;
1703 if (adev->ip_blocks[i].version->funcs->late_init) {
1704 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1705 if (r) {
1706 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1707 adev->ip_blocks[i].version->funcs->name, r);
1708 return r;
1709 }
1710 adev->ip_blocks[i].status.late_initialized = true;
1711 }
1712 }
1713
1714 mod_delayed_work(system_wq, &adev->late_init_work,
1715 msecs_to_jiffies(AMDGPU_RESUME_MS));
d38ceaf9 1716
0c49e0b8
CZ
1717 amdgpu_fill_reset_magic(adev);
1718
d38ceaf9
AD
1719 return 0;
1720}
1721
1722static int amdgpu_fini(struct amdgpu_device *adev)
1723{
1724 int i, r;
1725
3e96dbfd
AD
1726 /* need to disable SMC first */
1727 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1728 if (!adev->ip_blocks[i].status.hw)
3e96dbfd 1729 continue;
a1255107 1730 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3e96dbfd 1731 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
a1255107
AD
1732 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1733 AMD_CG_STATE_UNGATE);
3e96dbfd
AD
1734 if (r) {
1735 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
a1255107 1736 adev->ip_blocks[i].version->funcs->name, r);
3e96dbfd
AD
1737 return r;
1738 }
a1255107 1739 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3e96dbfd
AD
1740 /* XXX handle errors */
1741 if (r) {
1742 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
a1255107 1743 adev->ip_blocks[i].version->funcs->name, r);
3e96dbfd 1744 }
a1255107 1745 adev->ip_blocks[i].status.hw = false;
3e96dbfd
AD
1746 break;
1747 }
1748 }
1749
d38ceaf9 1750 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
a1255107 1751 if (!adev->ip_blocks[i].status.hw)
d38ceaf9 1752 continue;
a1255107 1753 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
d38ceaf9
AD
1754 amdgpu_wb_fini(adev);
1755 amdgpu_vram_scratch_fini(adev);
1756 }
8201a67a
RZ
1757
1758 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1759 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1760 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1761 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1762 AMD_CG_STATE_UNGATE);
1763 if (r) {
1764 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1765 adev->ip_blocks[i].version->funcs->name, r);
1766 return r;
1767 }
2c1a2784 1768 }
8201a67a 1769
a1255107 1770 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
d38ceaf9 1771 /* XXX handle errors */
2c1a2784 1772 if (r) {
a1255107
AD
1773 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1774 adev->ip_blocks[i].version->funcs->name, r);
2c1a2784 1775 }
8201a67a 1776
a1255107 1777 adev->ip_blocks[i].status.hw = false;
d38ceaf9
AD
1778 }
1779
1780 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
a1255107 1781 if (!adev->ip_blocks[i].status.sw)
d38ceaf9 1782 continue;
a1255107 1783 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
d38ceaf9 1784 /* XXX handle errors */
2c1a2784 1785 if (r) {
a1255107
AD
1786 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1787 adev->ip_blocks[i].version->funcs->name, r);
2c1a2784 1788 }
a1255107
AD
1789 adev->ip_blocks[i].status.sw = false;
1790 adev->ip_blocks[i].status.valid = false;
d38ceaf9
AD
1791 }
1792
a6dcfd9c 1793 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
a1255107 1794 if (!adev->ip_blocks[i].status.late_initialized)
8a2eef1d 1795 continue;
a1255107
AD
1796 if (adev->ip_blocks[i].version->funcs->late_fini)
1797 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1798 adev->ip_blocks[i].status.late_initialized = false;
a6dcfd9c
ML
1799 }
1800
3149d9da 1801 if (amdgpu_sriov_vf(adev)) {
2493664f 1802 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
3149d9da
XY
1803 amdgpu_virt_release_full_gpu(adev, false);
1804 }
2493664f 1805
d38ceaf9
AD
1806 return 0;
1807}
1808
2dc80b00
S
1809static void amdgpu_late_init_func_handler(struct work_struct *work)
1810{
1811 struct amdgpu_device *adev =
1812 container_of(work, struct amdgpu_device, late_init_work.work);
1813 amdgpu_late_set_cg_state(adev);
1814}
1815
faefba95 1816int amdgpu_suspend(struct amdgpu_device *adev)
d38ceaf9
AD
1817{
1818 int i, r;
1819
e941ea99
XY
1820 if (amdgpu_sriov_vf(adev))
1821 amdgpu_virt_request_full_gpu(adev, false);
1822
c5a93a28
FC
1823 /* ungate SMC block first */
1824 r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1825 AMD_CG_STATE_UNGATE);
1826 if (r) {
1827 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1828 }
1829
d38ceaf9 1830 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
a1255107 1831 if (!adev->ip_blocks[i].status.valid)
d38ceaf9
AD
1832 continue;
1833 /* ungate blocks so that suspend can properly shut them down */
c5a93a28 1834 if (i != AMD_IP_BLOCK_TYPE_SMC) {
a1255107
AD
1835 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1836 AMD_CG_STATE_UNGATE);
c5a93a28 1837 if (r) {
a1255107
AD
1838 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1839 adev->ip_blocks[i].version->funcs->name, r);
c5a93a28 1840 }
2c1a2784 1841 }
d38ceaf9 1842 /* XXX handle errors */
a1255107 1843 r = adev->ip_blocks[i].version->funcs->suspend(adev);
d38ceaf9 1844 /* XXX handle errors */
2c1a2784 1845 if (r) {
a1255107
AD
1846 DRM_ERROR("suspend of IP block <%s> failed %d\n",
1847 adev->ip_blocks[i].version->funcs->name, r);
2c1a2784 1848 }
d38ceaf9
AD
1849 }
1850
e941ea99
XY
1851 if (amdgpu_sriov_vf(adev))
1852 amdgpu_virt_release_full_gpu(adev, false);
1853
d38ceaf9
AD
1854 return 0;
1855}
1856
e4f0fdcc 1857static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
a90ad3c2
ML
1858{
1859 int i, r;
1860
2cb681b6
ML
1861 static enum amd_ip_block_type ip_order[] = {
1862 AMD_IP_BLOCK_TYPE_GMC,
1863 AMD_IP_BLOCK_TYPE_COMMON,
2cb681b6
ML
1864 AMD_IP_BLOCK_TYPE_IH,
1865 };
1866
1867 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1868 int j;
1869 struct amdgpu_ip_block *block;
1870
1871 for (j = 0; j < adev->num_ip_blocks; j++) {
1872 block = &adev->ip_blocks[j];
1873
1874 if (block->version->type != ip_order[i] ||
1875 !block->status.valid)
1876 continue;
1877
1878 r = block->version->funcs->hw_init(adev);
1879 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
a90ad3c2
ML
1880 }
1881 }
1882
1883 return 0;
1884}
1885
e4f0fdcc 1886static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
a90ad3c2
ML
1887{
1888 int i, r;
1889
2cb681b6
ML
1890 static enum amd_ip_block_type ip_order[] = {
1891 AMD_IP_BLOCK_TYPE_SMC,
1892 AMD_IP_BLOCK_TYPE_DCE,
1893 AMD_IP_BLOCK_TYPE_GFX,
1894 AMD_IP_BLOCK_TYPE_SDMA,
1895 AMD_IP_BLOCK_TYPE_VCE,
1896 };
a90ad3c2 1897
2cb681b6
ML
1898 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1899 int j;
1900 struct amdgpu_ip_block *block;
a90ad3c2 1901
2cb681b6
ML
1902 for (j = 0; j < adev->num_ip_blocks; j++) {
1903 block = &adev->ip_blocks[j];
1904
1905 if (block->version->type != ip_order[i] ||
1906 !block->status.valid)
1907 continue;
1908
1909 r = block->version->funcs->hw_init(adev);
1910 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
a90ad3c2
ML
1911 }
1912 }
1913
1914 return 0;
1915}
1916
fcf0649f 1917static int amdgpu_resume_phase1(struct amdgpu_device *adev)
d38ceaf9
AD
1918{
1919 int i, r;
1920
1921 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 1922 if (!adev->ip_blocks[i].status.valid)
d38ceaf9 1923 continue;
fcf0649f
CZ
1924 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1925 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1926 adev->ip_blocks[i].version->type ==
1927 AMD_IP_BLOCK_TYPE_IH) {
1928 r = adev->ip_blocks[i].version->funcs->resume(adev);
1929 if (r) {
1930 DRM_ERROR("resume of IP block <%s> failed %d\n",
1931 adev->ip_blocks[i].version->funcs->name, r);
1932 return r;
1933 }
1934 }
1935 }
1936
1937 return 0;
1938}
1939
1940static int amdgpu_resume_phase2(struct amdgpu_device *adev)
1941{
1942 int i, r;
1943
1944 for (i = 0; i < adev->num_ip_blocks; i++) {
1945 if (!adev->ip_blocks[i].status.valid)
1946 continue;
1947 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1948 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1949 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
1950 continue;
a1255107 1951 r = adev->ip_blocks[i].version->funcs->resume(adev);
2c1a2784 1952 if (r) {
a1255107
AD
1953 DRM_ERROR("resume of IP block <%s> failed %d\n",
1954 adev->ip_blocks[i].version->funcs->name, r);
d38ceaf9 1955 return r;
2c1a2784 1956 }
d38ceaf9
AD
1957 }
1958
1959 return 0;
1960}
1961
fcf0649f
CZ
1962static int amdgpu_resume(struct amdgpu_device *adev)
1963{
1964 int r;
1965
1966 r = amdgpu_resume_phase1(adev);
1967 if (r)
1968 return r;
1969 r = amdgpu_resume_phase2(adev);
1970
1971 return r;
1972}
1973
4e99a44e 1974static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
048765ad 1975{
a5bde2f9
AD
1976 if (adev->is_atom_fw) {
1977 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
1978 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1979 } else {
1980 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1981 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1982 }
048765ad
AR
1983}
1984
d38ceaf9
AD
1985/**
1986 * amdgpu_device_init - initialize the driver
1987 *
1988 * @adev: amdgpu_device pointer
1989 * @pdev: drm dev pointer
1990 * @pdev: pci dev pointer
1991 * @flags: driver flags
1992 *
1993 * Initializes the driver info and hw (all asics).
1994 * Returns 0 for success or an error on failure.
1995 * Called at driver startup.
1996 */
1997int amdgpu_device_init(struct amdgpu_device *adev,
1998 struct drm_device *ddev,
1999 struct pci_dev *pdev,
2000 uint32_t flags)
2001{
2002 int r, i;
2003 bool runtime = false;
95844d20 2004 u32 max_MBps;
d38ceaf9
AD
2005
2006 adev->shutdown = false;
2007 adev->dev = &pdev->dev;
2008 adev->ddev = ddev;
2009 adev->pdev = pdev;
2010 adev->flags = flags;
2f7d10b3 2011 adev->asic_type = flags & AMD_ASIC_MASK;
d38ceaf9
AD
2012 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2013 adev->mc.gtt_size = 512 * 1024 * 1024;
2014 adev->accel_working = false;
2015 adev->num_rings = 0;
2016 adev->mman.buffer_funcs = NULL;
2017 adev->mman.buffer_funcs_ring = NULL;
2018 adev->vm_manager.vm_pte_funcs = NULL;
2d55e45a 2019 adev->vm_manager.vm_pte_num_rings = 0;
d38ceaf9 2020 adev->gart.gart_funcs = NULL;
f54d1867 2021 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
d38ceaf9
AD
2022
2023 adev->smc_rreg = &amdgpu_invalid_rreg;
2024 adev->smc_wreg = &amdgpu_invalid_wreg;
2025 adev->pcie_rreg = &amdgpu_invalid_rreg;
2026 adev->pcie_wreg = &amdgpu_invalid_wreg;
36b9a952
HR
2027 adev->pciep_rreg = &amdgpu_invalid_rreg;
2028 adev->pciep_wreg = &amdgpu_invalid_wreg;
d38ceaf9
AD
2029 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2030 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2031 adev->didt_rreg = &amdgpu_invalid_rreg;
2032 adev->didt_wreg = &amdgpu_invalid_wreg;
ccdbb20a
RZ
2033 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2034 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
d38ceaf9
AD
2035 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2036 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2037
ccdbb20a 2038
3e39ab90
AD
2039 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2040 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2041 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
d38ceaf9
AD
2042
2043 /* mutex initialization are all done here so we
2044 * can recall function without having locking issues */
d38ceaf9 2045 atomic_set(&adev->irq.ih.lock, 0);
0e5ca0d1 2046 mutex_init(&adev->firmware.mutex);
d38ceaf9
AD
2047 mutex_init(&adev->pm.mutex);
2048 mutex_init(&adev->gfx.gpu_clock_mutex);
2049 mutex_init(&adev->srbm_mutex);
2050 mutex_init(&adev->grbm_idx_mutex);
d38ceaf9
AD
2051 mutex_init(&adev->mn_lock);
2052 hash_init(adev->mn_hash);
2053
2054 amdgpu_check_arguments(adev);
2055
d38ceaf9
AD
2056 spin_lock_init(&adev->mmio_idx_lock);
2057 spin_lock_init(&adev->smc_idx_lock);
2058 spin_lock_init(&adev->pcie_idx_lock);
2059 spin_lock_init(&adev->uvd_ctx_idx_lock);
2060 spin_lock_init(&adev->didt_idx_lock);
ccdbb20a 2061 spin_lock_init(&adev->gc_cac_idx_lock);
d38ceaf9 2062 spin_lock_init(&adev->audio_endpt_idx_lock);
95844d20 2063 spin_lock_init(&adev->mm_stats.lock);
d38ceaf9 2064
0c4e7fa5
CZ
2065 INIT_LIST_HEAD(&adev->shadow_list);
2066 mutex_init(&adev->shadow_list_lock);
2067
5c1354bd
CZ
2068 INIT_LIST_HEAD(&adev->gtt_list);
2069 spin_lock_init(&adev->gtt_list_lock);
2070
795f2813
AR
2071 INIT_LIST_HEAD(&adev->ring_lru_list);
2072 spin_lock_init(&adev->ring_lru_list_lock);
2073
2dc80b00
S
2074 INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
2075
0fa49558
AX
2076 /* Registers mapping */
2077 /* TODO: block userspace mapping of io register */
da69c161
KW
2078 if (adev->asic_type >= CHIP_BONAIRE) {
2079 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2080 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2081 } else {
2082 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2083 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2084 }
d38ceaf9 2085
d38ceaf9
AD
2086 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2087 if (adev->rmmio == NULL) {
2088 return -ENOMEM;
2089 }
2090 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2091 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2092
da69c161
KW
2093 if (adev->asic_type >= CHIP_BONAIRE)
2094 /* doorbell bar mapping */
2095 amdgpu_doorbell_init(adev);
d38ceaf9
AD
2096
2097 /* io port mapping */
2098 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2099 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2100 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2101 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2102 break;
2103 }
2104 }
2105 if (adev->rio_mem == NULL)
b64a18c5 2106 DRM_INFO("PCI I/O BAR is not found.\n");
d38ceaf9
AD
2107
2108 /* early init functions */
2109 r = amdgpu_early_init(adev);
2110 if (r)
2111 return r;
2112
2113 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2114 /* this will fail for cards that aren't VGA class devices, just
2115 * ignore it */
2116 vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
2117
2118 if (amdgpu_runtime_pm == 1)
2119 runtime = true;
e9bef455 2120 if (amdgpu_device_is_px(ddev))
d38ceaf9 2121 runtime = true;
84c8b22e
LW
2122 if (!pci_is_thunderbolt_attached(adev->pdev))
2123 vga_switcheroo_register_client(adev->pdev,
2124 &amdgpu_switcheroo_ops, runtime);
d38ceaf9
AD
2125 if (runtime)
2126 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2127
2128 /* Read BIOS */
83ba126a
AD
2129 if (!amdgpu_get_bios(adev)) {
2130 r = -EINVAL;
2131 goto failed;
2132 }
f7e9e9fe 2133
d38ceaf9 2134 r = amdgpu_atombios_init(adev);
2c1a2784
AD
2135 if (r) {
2136 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
83ba126a 2137 goto failed;
2c1a2784 2138 }
d38ceaf9 2139
4e99a44e
ML
2140 /* detect if we are with an SRIOV vbios */
2141 amdgpu_device_detect_sriov_bios(adev);
048765ad 2142
d38ceaf9 2143 /* Post card if necessary */
bec86378 2144 if (amdgpu_vpost_needed(adev)) {
d38ceaf9 2145 if (!adev->bios) {
bec86378 2146 dev_err(adev->dev, "no vBIOS found\n");
83ba126a
AD
2147 r = -EINVAL;
2148 goto failed;
d38ceaf9 2149 }
bec86378 2150 DRM_INFO("GPU posting now...\n");
4e99a44e
ML
2151 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2152 if (r) {
2153 dev_err(adev->dev, "gpu post error!\n");
2154 goto failed;
2155 }
2156 } else {
2157 DRM_INFO("GPU post is not needed\n");
d38ceaf9
AD
2158 }
2159
a5bde2f9
AD
2160 if (!adev->is_atom_fw) {
2161 /* Initialize clocks */
2162 r = amdgpu_atombios_get_clock_info(adev);
2163 if (r) {
2164 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2165 return r;
2166 }
2167 /* init i2c buses */
2168 amdgpu_atombios_i2c_init(adev);
2c1a2784 2169 }
d38ceaf9
AD
2170
2171 /* Fence driver */
2172 r = amdgpu_fence_driver_init(adev);
2c1a2784
AD
2173 if (r) {
2174 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
83ba126a 2175 goto failed;
2c1a2784 2176 }
d38ceaf9
AD
2177
2178 /* init the mode config */
2179 drm_mode_config_init(adev->ddev);
2180
2181 r = amdgpu_init(adev);
2182 if (r) {
2c1a2784 2183 dev_err(adev->dev, "amdgpu_init failed\n");
d38ceaf9 2184 amdgpu_fini(adev);
83ba126a 2185 goto failed;
d38ceaf9
AD
2186 }
2187
2188 adev->accel_working = true;
2189
e59c0205
AX
2190 amdgpu_vm_check_compute_bug(adev);
2191
95844d20
MO
2192 /* Initialize the buffer migration limit. */
2193 if (amdgpu_moverate >= 0)
2194 max_MBps = amdgpu_moverate;
2195 else
2196 max_MBps = 8; /* Allow 8 MB/s. */
2197 /* Get a log2 for easy divisions. */
2198 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2199
d38ceaf9
AD
2200 r = amdgpu_ib_pool_init(adev);
2201 if (r) {
2202 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
83ba126a 2203 goto failed;
d38ceaf9
AD
2204 }
2205
2206 r = amdgpu_ib_ring_tests(adev);
2207 if (r)
2208 DRM_ERROR("ib ring test failed (%d).\n", r);
2209
9bc92b9c
ML
2210 amdgpu_fbdev_init(adev);
2211
d38ceaf9 2212 r = amdgpu_gem_debugfs_init(adev);
3f14e623 2213 if (r)
d38ceaf9 2214 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
d38ceaf9
AD
2215
2216 r = amdgpu_debugfs_regs_init(adev);
3f14e623 2217 if (r)
d38ceaf9 2218 DRM_ERROR("registering register debugfs failed (%d).\n", r);
d38ceaf9 2219
4f0955fc
HR
2220 r = amdgpu_debugfs_test_ib_ring_init(adev);
2221 if (r)
2222 DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
2223
50ab2533 2224 r = amdgpu_debugfs_firmware_init(adev);
3f14e623 2225 if (r)
50ab2533 2226 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
50ab2533 2227
d38ceaf9
AD
2228 if ((amdgpu_testing & 1)) {
2229 if (adev->accel_working)
2230 amdgpu_test_moves(adev);
2231 else
2232 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2233 }
d38ceaf9
AD
2234 if (amdgpu_benchmarking) {
2235 if (adev->accel_working)
2236 amdgpu_benchmark(adev, amdgpu_benchmarking);
2237 else
2238 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2239 }
2240
2241 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2242 * explicit gating rather than handling it automatically.
2243 */
2244 r = amdgpu_late_init(adev);
2c1a2784
AD
2245 if (r) {
2246 dev_err(adev->dev, "amdgpu_late_init failed\n");
83ba126a 2247 goto failed;
2c1a2784 2248 }
d38ceaf9
AD
2249
2250 return 0;
83ba126a
AD
2251
2252failed:
2253 if (runtime)
2254 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2255 return r;
d38ceaf9
AD
2256}
2257
d38ceaf9
AD
2258/**
2259 * amdgpu_device_fini - tear down the driver
2260 *
2261 * @adev: amdgpu_device pointer
2262 *
2263 * Tear down the driver info (all asics).
2264 * Called at driver shutdown.
2265 */
2266void amdgpu_device_fini(struct amdgpu_device *adev)
2267{
2268 int r;
2269
2270 DRM_INFO("amdgpu: finishing device.\n");
2271 adev->shutdown = true;
db2c2a97
PD
2272 if (adev->mode_info.mode_config_initialized)
2273 drm_crtc_force_disable_all(adev->ddev);
d38ceaf9
AD
2274 /* evict vram memory */
2275 amdgpu_bo_evict_vram(adev);
2276 amdgpu_ib_pool_fini(adev);
2277 amdgpu_fence_driver_fini(adev);
2278 amdgpu_fbdev_fini(adev);
2279 r = amdgpu_fini(adev);
ab4fe3e1
HR
2280 if (adev->firmware.gpu_info_fw) {
2281 release_firmware(adev->firmware.gpu_info_fw);
2282 adev->firmware.gpu_info_fw = NULL;
2283 }
d38ceaf9 2284 adev->accel_working = false;
2dc80b00 2285 cancel_delayed_work_sync(&adev->late_init_work);
d38ceaf9
AD
2286 /* free i2c buses */
2287 amdgpu_i2c_fini(adev);
2288 amdgpu_atombios_fini(adev);
2289 kfree(adev->bios);
2290 adev->bios = NULL;
84c8b22e
LW
2291 if (!pci_is_thunderbolt_attached(adev->pdev))
2292 vga_switcheroo_unregister_client(adev->pdev);
83ba126a
AD
2293 if (adev->flags & AMD_IS_PX)
2294 vga_switcheroo_fini_domain_pm_ops(adev->dev);
d38ceaf9
AD
2295 vga_client_register(adev->pdev, NULL, NULL, NULL);
2296 if (adev->rio_mem)
2297 pci_iounmap(adev->pdev, adev->rio_mem);
2298 adev->rio_mem = NULL;
2299 iounmap(adev->rmmio);
2300 adev->rmmio = NULL;
da69c161
KW
2301 if (adev->asic_type >= CHIP_BONAIRE)
2302 amdgpu_doorbell_fini(adev);
d38ceaf9 2303 amdgpu_debugfs_regs_cleanup(adev);
d38ceaf9
AD
2304}
2305
2306
2307/*
2308 * Suspend & resume.
2309 */
2310/**
810ddc3a 2311 * amdgpu_device_suspend - initiate device suspend
d38ceaf9
AD
2312 *
2313 * @pdev: drm dev pointer
2314 * @state: suspend state
2315 *
2316 * Puts the hw in the suspend state (all asics).
2317 * Returns 0 for success or an error on failure.
2318 * Called at driver suspend.
2319 */
810ddc3a 2320int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
d38ceaf9
AD
2321{
2322 struct amdgpu_device *adev;
2323 struct drm_crtc *crtc;
2324 struct drm_connector *connector;
5ceb54c6 2325 int r;
d38ceaf9
AD
2326
2327 if (dev == NULL || dev->dev_private == NULL) {
2328 return -ENODEV;
2329 }
2330
2331 adev = dev->dev_private;
2332
2333 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2334 return 0;
2335
2336 drm_kms_helper_poll_disable(dev);
2337
2338 /* turn off display hw */
4c7fbc39 2339 drm_modeset_lock_all(dev);
d38ceaf9
AD
2340 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2341 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2342 }
4c7fbc39 2343 drm_modeset_unlock_all(dev);
d38ceaf9 2344
756e6880 2345 /* unpin the front buffers and cursors */
d38ceaf9 2346 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
756e6880 2347 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
d38ceaf9
AD
2348 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2349 struct amdgpu_bo *robj;
2350
756e6880
AD
2351 if (amdgpu_crtc->cursor_bo) {
2352 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
7a6901d7 2353 r = amdgpu_bo_reserve(aobj, true);
756e6880
AD
2354 if (r == 0) {
2355 amdgpu_bo_unpin(aobj);
2356 amdgpu_bo_unreserve(aobj);
2357 }
2358 }
2359
d38ceaf9
AD
2360 if (rfb == NULL || rfb->obj == NULL) {
2361 continue;
2362 }
2363 robj = gem_to_amdgpu_bo(rfb->obj);
2364 /* don't unpin kernel fb objects */
2365 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
7a6901d7 2366 r = amdgpu_bo_reserve(robj, true);
d38ceaf9
AD
2367 if (r == 0) {
2368 amdgpu_bo_unpin(robj);
2369 amdgpu_bo_unreserve(robj);
2370 }
2371 }
2372 }
2373 /* evict vram memory */
2374 amdgpu_bo_evict_vram(adev);
2375
5ceb54c6 2376 amdgpu_fence_driver_suspend(adev);
d38ceaf9
AD
2377
2378 r = amdgpu_suspend(adev);
2379
a0a71e49
AD
2380 /* evict remaining vram memory
2381 * This second call to evict vram is to evict the gart page table
2382 * using the CPU.
2383 */
d38ceaf9
AD
2384 amdgpu_bo_evict_vram(adev);
2385
be34d3bf
AD
2386 if (adev->is_atom_fw)
2387 amdgpu_atomfirmware_scratch_regs_save(adev);
2388 else
2389 amdgpu_atombios_scratch_regs_save(adev);
d38ceaf9
AD
2390 pci_save_state(dev->pdev);
2391 if (suspend) {
2392 /* Shut down the device */
2393 pci_disable_device(dev->pdev);
2394 pci_set_power_state(dev->pdev, PCI_D3hot);
74b0b157 2395 } else {
2396 r = amdgpu_asic_reset(adev);
2397 if (r)
2398 DRM_ERROR("amdgpu asic reset failed\n");
d38ceaf9
AD
2399 }
2400
2401 if (fbcon) {
2402 console_lock();
2403 amdgpu_fbdev_set_suspend(adev, 1);
2404 console_unlock();
2405 }
2406 return 0;
2407}
2408
2409/**
810ddc3a 2410 * amdgpu_device_resume - initiate device resume
d38ceaf9
AD
2411 *
2412 * @pdev: drm dev pointer
2413 *
2414 * Bring the hw back to operating state (all asics).
2415 * Returns 0 for success or an error on failure.
2416 * Called at driver resume.
2417 */
810ddc3a 2418int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
d38ceaf9
AD
2419{
2420 struct drm_connector *connector;
2421 struct amdgpu_device *adev = dev->dev_private;
756e6880 2422 struct drm_crtc *crtc;
03161a6e 2423 int r = 0;
d38ceaf9
AD
2424
2425 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2426 return 0;
2427
74b0b157 2428 if (fbcon)
d38ceaf9 2429 console_lock();
74b0b157 2430
d38ceaf9
AD
2431 if (resume) {
2432 pci_set_power_state(dev->pdev, PCI_D0);
2433 pci_restore_state(dev->pdev);
74b0b157 2434 r = pci_enable_device(dev->pdev);
03161a6e
HR
2435 if (r)
2436 goto unlock;
d38ceaf9 2437 }
be34d3bf
AD
2438 if (adev->is_atom_fw)
2439 amdgpu_atomfirmware_scratch_regs_restore(adev);
2440 else
2441 amdgpu_atombios_scratch_regs_restore(adev);
d38ceaf9
AD
2442
2443 /* post card */
c836fec5 2444 if (amdgpu_need_post(adev)) {
74b0b157 2445 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2446 if (r)
2447 DRM_ERROR("amdgpu asic init failed\n");
2448 }
d38ceaf9
AD
2449
2450 r = amdgpu_resume(adev);
e6707218 2451 if (r) {
ca198528 2452 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
03161a6e 2453 goto unlock;
e6707218 2454 }
5ceb54c6
AD
2455 amdgpu_fence_driver_resume(adev);
2456
ca198528
FC
2457 if (resume) {
2458 r = amdgpu_ib_ring_tests(adev);
2459 if (r)
2460 DRM_ERROR("ib ring test failed (%d).\n", r);
2461 }
d38ceaf9
AD
2462
2463 r = amdgpu_late_init(adev);
03161a6e
HR
2464 if (r)
2465 goto unlock;
d38ceaf9 2466
756e6880
AD
2467 /* pin cursors */
2468 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2469 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2470
2471 if (amdgpu_crtc->cursor_bo) {
2472 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
7a6901d7 2473 r = amdgpu_bo_reserve(aobj, true);
756e6880
AD
2474 if (r == 0) {
2475 r = amdgpu_bo_pin(aobj,
2476 AMDGPU_GEM_DOMAIN_VRAM,
2477 &amdgpu_crtc->cursor_addr);
2478 if (r != 0)
2479 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2480 amdgpu_bo_unreserve(aobj);
2481 }
2482 }
2483 }
2484
d38ceaf9
AD
2485 /* blat the mode back in */
2486 if (fbcon) {
2487 drm_helper_resume_force_mode(dev);
2488 /* turn on display hw */
4c7fbc39 2489 drm_modeset_lock_all(dev);
d38ceaf9
AD
2490 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2491 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2492 }
4c7fbc39 2493 drm_modeset_unlock_all(dev);
d38ceaf9
AD
2494 }
2495
2496 drm_kms_helper_poll_enable(dev);
23a1a9e5
L
2497
2498 /*
2499 * Most of the connector probing functions try to acquire runtime pm
2500 * refs to ensure that the GPU is powered on when connector polling is
2501 * performed. Since we're calling this from a runtime PM callback,
2502 * trying to acquire rpm refs will cause us to deadlock.
2503 *
2504 * Since we're guaranteed to be holding the rpm lock, it's safe to
2505 * temporarily disable the rpm helpers so this doesn't deadlock us.
2506 */
2507#ifdef CONFIG_PM
2508 dev->dev->power.disable_depth++;
2509#endif
54fb2a5c 2510 drm_helper_hpd_irq_event(dev);
23a1a9e5
L
2511#ifdef CONFIG_PM
2512 dev->dev->power.disable_depth--;
2513#endif
d38ceaf9 2514
03161a6e 2515 if (fbcon)
d38ceaf9 2516 amdgpu_fbdev_set_suspend(adev, 0);
03161a6e
HR
2517
2518unlock:
2519 if (fbcon)
d38ceaf9 2520 console_unlock();
d38ceaf9 2521
03161a6e 2522 return r;
d38ceaf9
AD
2523}
2524
63fbf42f
CZ
2525static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2526{
2527 int i;
2528 bool asic_hang = false;
2529
2530 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 2531 if (!adev->ip_blocks[i].status.valid)
63fbf42f 2532 continue;
a1255107
AD
2533 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2534 adev->ip_blocks[i].status.hang =
2535 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2536 if (adev->ip_blocks[i].status.hang) {
2537 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
63fbf42f
CZ
2538 asic_hang = true;
2539 }
2540 }
2541 return asic_hang;
2542}
2543
4d446656 2544static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
d31a501e
CZ
2545{
2546 int i, r = 0;
2547
2548 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 2549 if (!adev->ip_blocks[i].status.valid)
d31a501e 2550 continue;
a1255107
AD
2551 if (adev->ip_blocks[i].status.hang &&
2552 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2553 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
d31a501e
CZ
2554 if (r)
2555 return r;
2556 }
2557 }
2558
2559 return 0;
2560}
2561
35d782fe
CZ
2562static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2563{
da146d3b
AD
2564 int i;
2565
2566 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 2567 if (!adev->ip_blocks[i].status.valid)
da146d3b 2568 continue;
a1255107
AD
2569 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2570 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2571 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2572 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
2573 if (adev->ip_blocks[i].status.hang) {
da146d3b
AD
2574 DRM_INFO("Some block need full reset!\n");
2575 return true;
2576 }
2577 }
35d782fe
CZ
2578 }
2579 return false;
2580}
2581
2582static int amdgpu_soft_reset(struct amdgpu_device *adev)
2583{
2584 int i, r = 0;
2585
2586 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 2587 if (!adev->ip_blocks[i].status.valid)
35d782fe 2588 continue;
a1255107
AD
2589 if (adev->ip_blocks[i].status.hang &&
2590 adev->ip_blocks[i].version->funcs->soft_reset) {
2591 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
35d782fe
CZ
2592 if (r)
2593 return r;
2594 }
2595 }
2596
2597 return 0;
2598}
2599
2600static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2601{
2602 int i, r = 0;
2603
2604 for (i = 0; i < adev->num_ip_blocks; i++) {
a1255107 2605 if (!adev->ip_blocks[i].status.valid)
35d782fe 2606 continue;
a1255107
AD
2607 if (adev->ip_blocks[i].status.hang &&
2608 adev->ip_blocks[i].version->funcs->post_soft_reset)
2609 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
35d782fe
CZ
2610 if (r)
2611 return r;
2612 }
2613
2614 return 0;
2615}
2616
3ad81f16
CZ
2617bool amdgpu_need_backup(struct amdgpu_device *adev)
2618{
2619 if (adev->flags & AMD_IS_APU)
2620 return false;
2621
2622 return amdgpu_lockup_timeout > 0 ? true : false;
2623}
2624
53cdccd5
CZ
2625static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2626 struct amdgpu_ring *ring,
2627 struct amdgpu_bo *bo,
f54d1867 2628 struct dma_fence **fence)
53cdccd5
CZ
2629{
2630 uint32_t domain;
2631 int r;
2632
23d2e504
RH
2633 if (!bo->shadow)
2634 return 0;
2635
1d284797 2636 r = amdgpu_bo_reserve(bo, true);
23d2e504
RH
2637 if (r)
2638 return r;
2639 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2640 /* if bo has been evicted, then no need to recover */
2641 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
82521316
RH
2642 r = amdgpu_bo_validate(bo->shadow);
2643 if (r) {
2644 DRM_ERROR("bo validate failed!\n");
2645 goto err;
2646 }
2647
2648 r = amdgpu_ttm_bind(&bo->shadow->tbo, &bo->shadow->tbo.mem);
2649 if (r) {
2650 DRM_ERROR("%p bind failed\n", bo->shadow);
2651 goto err;
2652 }
2653
23d2e504 2654 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
53cdccd5 2655 NULL, fence, true);
23d2e504
RH
2656 if (r) {
2657 DRM_ERROR("recover page table failed!\n");
2658 goto err;
2659 }
2660 }
53cdccd5 2661err:
23d2e504
RH
2662 amdgpu_bo_unreserve(bo);
2663 return r;
53cdccd5
CZ
2664}
2665
a90ad3c2
ML
2666/**
2667 * amdgpu_sriov_gpu_reset - reset the asic
2668 *
2669 * @adev: amdgpu device pointer
7225f873 2670 * @job: which job trigger hang
a90ad3c2
ML
2671 *
2672 * Attempt the reset the GPU if it has hung (all asics).
2673 * for SRIOV case.
2674 * Returns 0 for success or an error on failure.
2675 */
7225f873 2676int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
a90ad3c2 2677{
65781c78 2678 int i, j, r = 0;
a90ad3c2
ML
2679 int resched;
2680 struct amdgpu_bo *bo, *tmp;
2681 struct amdgpu_ring *ring;
2682 struct dma_fence *fence = NULL, *next = NULL;
2683
147b5983 2684 mutex_lock(&adev->virt.lock_reset);
a90ad3c2 2685 atomic_inc(&adev->gpu_reset_counter);
1fb37a3d 2686 adev->gfx.in_reset = true;
a90ad3c2
ML
2687
2688 /* block TTM */
2689 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2690
65781c78
ML
2691 /* we start from the ring trigger GPU hang */
2692 j = job ? job->ring->idx : 0;
a90ad3c2 2693
65781c78
ML
2694 /* block scheduler */
2695 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2696 ring = adev->rings[i % AMDGPU_MAX_RINGS];
a90ad3c2
ML
2697 if (!ring || !ring->sched.thread)
2698 continue;
2699
2700 kthread_park(ring->sched.thread);
65781c78
ML
2701
2702 if (job && j != i)
2703 continue;
2704
4f059ecd 2705 /* here give the last chance to check if job removed from mirror-list
65781c78 2706 * since we already pay some time on kthread_park */
4f059ecd 2707 if (job && list_empty(&job->base.node)) {
65781c78
ML
2708 kthread_unpark(ring->sched.thread);
2709 goto give_up_reset;
2710 }
2711
2712 if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
2713 amd_sched_job_kickout(&job->base);
2714
2715 /* only do job_reset on the hang ring if @job not NULL */
a90ad3c2 2716 amd_sched_hw_job_reset(&ring->sched);
a90ad3c2 2717
65781c78
ML
2718 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2719 amdgpu_fence_driver_force_completion_ring(ring);
2720 }
a90ad3c2
ML
2721
2722 /* request to take full control of GPU before re-initialization */
7225f873 2723 if (job)
a90ad3c2
ML
2724 amdgpu_virt_reset_gpu(adev);
2725 else
2726 amdgpu_virt_request_full_gpu(adev, true);
2727
2728
2729 /* Resume IP prior to SMC */
e4f0fdcc 2730 amdgpu_sriov_reinit_early(adev);
a90ad3c2
ML
2731
2732 /* we need recover gart prior to run SMC/CP/SDMA resume */
2733 amdgpu_ttm_recover_gart(adev);
2734
2735 /* now we are okay to resume SMC/CP/SDMA */
e4f0fdcc 2736 amdgpu_sriov_reinit_late(adev);
a90ad3c2
ML
2737
2738 amdgpu_irq_gpu_reset_resume_helper(adev);
2739
2740 if (amdgpu_ib_ring_tests(adev))
2741 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2742
2743 /* release full control of GPU after ib test */
2744 amdgpu_virt_release_full_gpu(adev, true);
2745
2746 DRM_INFO("recover vram bo from shadow\n");
2747
2748 ring = adev->mman.buffer_funcs_ring;
2749 mutex_lock(&adev->shadow_list_lock);
2750 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
236763d3 2751 next = NULL;
a90ad3c2
ML
2752 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2753 if (fence) {
2754 r = dma_fence_wait(fence, false);
2755 if (r) {
2756 WARN(r, "recovery from shadow isn't completed\n");
2757 break;
2758 }
2759 }
2760
2761 dma_fence_put(fence);
2762 fence = next;
2763 }
2764 mutex_unlock(&adev->shadow_list_lock);
2765
2766 if (fence) {
2767 r = dma_fence_wait(fence, false);
2768 if (r)
2769 WARN(r, "recovery from shadow isn't completed\n");
2770 }
2771 dma_fence_put(fence);
2772
65781c78
ML
2773 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2774 ring = adev->rings[i % AMDGPU_MAX_RINGS];
a90ad3c2
ML
2775 if (!ring || !ring->sched.thread)
2776 continue;
2777
65781c78
ML
2778 if (job && j != i) {
2779 kthread_unpark(ring->sched.thread);
2780 continue;
2781 }
2782
a90ad3c2
ML
2783 amd_sched_job_recovery(&ring->sched);
2784 kthread_unpark(ring->sched.thread);
2785 }
2786
2787 drm_helper_resume_force_mode(adev->ddev);
65781c78 2788give_up_reset:
a90ad3c2
ML
2789 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2790 if (r) {
2791 /* bad news, how to tell it to userspace ? */
2792 dev_info(adev->dev, "GPU reset failed\n");
65781c78
ML
2793 } else {
2794 dev_info(adev->dev, "GPU reset successed!\n");
a90ad3c2
ML
2795 }
2796
1fb37a3d 2797 adev->gfx.in_reset = false;
147b5983 2798 mutex_unlock(&adev->virt.lock_reset);
a90ad3c2
ML
2799 return r;
2800}
2801
d38ceaf9
AD
2802/**
2803 * amdgpu_gpu_reset - reset the asic
2804 *
2805 * @adev: amdgpu device pointer
2806 *
2807 * Attempt the reset the GPU if it has hung (all asics).
2808 * Returns 0 for success or an error on failure.
2809 */
2810int amdgpu_gpu_reset(struct amdgpu_device *adev)
2811{
d38ceaf9
AD
2812 int i, r;
2813 int resched;
0c49e0b8 2814 bool need_full_reset, vram_lost = false;
d38ceaf9 2815
63fbf42f
CZ
2816 if (!amdgpu_check_soft_reset(adev)) {
2817 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2818 return 0;
2819 }
d38ceaf9 2820
d94aed5a 2821 atomic_inc(&adev->gpu_reset_counter);
d38ceaf9 2822
a3c47d6b
CZ
2823 /* block TTM */
2824 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2825
0875dc9e
CZ
2826 /* block scheduler */
2827 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2828 struct amdgpu_ring *ring = adev->rings[i];
2829
51687759 2830 if (!ring || !ring->sched.thread)
0875dc9e
CZ
2831 continue;
2832 kthread_park(ring->sched.thread);
aa1c8900 2833 amd_sched_hw_job_reset(&ring->sched);
0875dc9e 2834 }
2200edac
CZ
2835 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2836 amdgpu_fence_driver_force_completion(adev);
d38ceaf9 2837
35d782fe 2838 need_full_reset = amdgpu_need_full_reset(adev);
d38ceaf9 2839
35d782fe
CZ
2840 if (!need_full_reset) {
2841 amdgpu_pre_soft_reset(adev);
2842 r = amdgpu_soft_reset(adev);
2843 amdgpu_post_soft_reset(adev);
2844 if (r || amdgpu_check_soft_reset(adev)) {
2845 DRM_INFO("soft reset failed, will fallback to full reset!\n");
2846 need_full_reset = true;
2847 }
f1aa7e08
CZ
2848 }
2849
35d782fe 2850 if (need_full_reset) {
35d782fe 2851 r = amdgpu_suspend(adev);
bfa99269 2852
35d782fe 2853retry:
be34d3bf
AD
2854 if (adev->is_atom_fw)
2855 amdgpu_atomfirmware_scratch_regs_save(adev);
2856 else
2857 amdgpu_atombios_scratch_regs_save(adev);
35d782fe 2858 r = amdgpu_asic_reset(adev);
be34d3bf
AD
2859 if (adev->is_atom_fw)
2860 amdgpu_atomfirmware_scratch_regs_restore(adev);
2861 else
2862 amdgpu_atombios_scratch_regs_restore(adev);
35d782fe
CZ
2863 /* post card */
2864 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2865
2866 if (!r) {
2867 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
fcf0649f
CZ
2868 r = amdgpu_resume_phase1(adev);
2869 if (r)
2870 goto out;
0c49e0b8 2871 vram_lost = amdgpu_check_vram_lost(adev);
f1892138 2872 if (vram_lost) {
0c49e0b8 2873 DRM_ERROR("VRAM is lost!\n");
f1892138
CZ
2874 atomic_inc(&adev->vram_lost_counter);
2875 }
fcf0649f
CZ
2876 r = amdgpu_ttm_recover_gart(adev);
2877 if (r)
2878 goto out;
2879 r = amdgpu_resume_phase2(adev);
2880 if (r)
2881 goto out;
0c49e0b8
CZ
2882 if (vram_lost)
2883 amdgpu_fill_reset_magic(adev);
35d782fe 2884 }
d38ceaf9 2885 }
fcf0649f 2886out:
d38ceaf9 2887 if (!r) {
e72cfd58 2888 amdgpu_irq_gpu_reset_resume_helper(adev);
1f465087
CZ
2889 r = amdgpu_ib_ring_tests(adev);
2890 if (r) {
2891 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
40019dc4 2892 r = amdgpu_suspend(adev);
53cdccd5 2893 need_full_reset = true;
40019dc4 2894 goto retry;
1f465087 2895 }
53cdccd5
CZ
2896 /**
2897 * recovery vm page tables, since we cannot depend on VRAM is
2898 * consistent after gpu full reset.
2899 */
2900 if (need_full_reset && amdgpu_need_backup(adev)) {
2901 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2902 struct amdgpu_bo *bo, *tmp;
f54d1867 2903 struct dma_fence *fence = NULL, *next = NULL;
53cdccd5
CZ
2904
2905 DRM_INFO("recover vram bo from shadow\n");
2906 mutex_lock(&adev->shadow_list_lock);
2907 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
236763d3 2908 next = NULL;
53cdccd5
CZ
2909 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2910 if (fence) {
f54d1867 2911 r = dma_fence_wait(fence, false);
53cdccd5 2912 if (r) {
1d7b17b0 2913 WARN(r, "recovery from shadow isn't completed\n");
53cdccd5
CZ
2914 break;
2915 }
2916 }
1f465087 2917
f54d1867 2918 dma_fence_put(fence);
53cdccd5
CZ
2919 fence = next;
2920 }
2921 mutex_unlock(&adev->shadow_list_lock);
2922 if (fence) {
f54d1867 2923 r = dma_fence_wait(fence, false);
53cdccd5 2924 if (r)
1d7b17b0 2925 WARN(r, "recovery from shadow isn't completed\n");
53cdccd5 2926 }
f54d1867 2927 dma_fence_put(fence);
53cdccd5 2928 }
d38ceaf9
AD
2929 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2930 struct amdgpu_ring *ring = adev->rings[i];
51687759
CZ
2931
2932 if (!ring || !ring->sched.thread)
d38ceaf9 2933 continue;
53cdccd5 2934
aa1c8900 2935 amd_sched_job_recovery(&ring->sched);
0875dc9e 2936 kthread_unpark(ring->sched.thread);
d38ceaf9 2937 }
d38ceaf9 2938 } else {
2200edac 2939 dev_err(adev->dev, "asic resume failed (%d).\n", r);
d38ceaf9 2940 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
51687759 2941 if (adev->rings[i] && adev->rings[i]->sched.thread) {
0875dc9e 2942 kthread_unpark(adev->rings[i]->sched.thread);
0875dc9e 2943 }
d38ceaf9
AD
2944 }
2945 }
2946
2947 drm_helper_resume_force_mode(adev->ddev);
2948
2949 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
6643be65 2950 if (r)
d38ceaf9
AD
2951 /* bad news, how to tell it to userspace ? */
2952 dev_info(adev->dev, "GPU reset failed\n");
6643be65
CZ
2953 else
2954 dev_info(adev->dev, "GPU reset successed!\n");
d38ceaf9 2955
d38ceaf9
AD
2956 return r;
2957}
2958
d0dd7f0c
AD
2959void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2960{
2961 u32 mask;
2962 int ret;
2963
cd474ba0
AD
2964 if (amdgpu_pcie_gen_cap)
2965 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
d0dd7f0c 2966
cd474ba0
AD
2967 if (amdgpu_pcie_lane_cap)
2968 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
d0dd7f0c 2969
cd474ba0
AD
2970 /* covers APUs as well */
2971 if (pci_is_root_bus(adev->pdev->bus)) {
2972 if (adev->pm.pcie_gen_mask == 0)
2973 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2974 if (adev->pm.pcie_mlw_mask == 0)
2975 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
d0dd7f0c 2976 return;
cd474ba0 2977 }
d0dd7f0c 2978
cd474ba0
AD
2979 if (adev->pm.pcie_gen_mask == 0) {
2980 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2981 if (!ret) {
2982 adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2983 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2984 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2985
2986 if (mask & DRM_PCIE_SPEED_25)
2987 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2988 if (mask & DRM_PCIE_SPEED_50)
2989 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2990 if (mask & DRM_PCIE_SPEED_80)
2991 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2992 } else {
2993 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2994 }
2995 }
2996 if (adev->pm.pcie_mlw_mask == 0) {
2997 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
2998 if (!ret) {
2999 switch (mask) {
3000 case 32:
3001 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3002 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3003 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3004 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3005 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3006 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3007 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3008 break;
3009 case 16:
3010 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3011 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3012 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3013 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3014 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3015 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3016 break;
3017 case 12:
3018 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3019 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3020 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3021 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3022 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3023 break;
3024 case 8:
3025 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3026 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3027 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3028 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3029 break;
3030 case 4:
3031 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3032 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3033 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3034 break;
3035 case 2:
3036 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3037 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3038 break;
3039 case 1:
3040 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3041 break;
3042 default:
3043 break;
3044 }
3045 } else {
3046 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
d0dd7f0c
AD
3047 }
3048 }
3049}
d38ceaf9
AD
3050
3051/*
3052 * Debugfs
3053 */
3054int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
06ab6832 3055 const struct drm_info_list *files,
d38ceaf9
AD
3056 unsigned nfiles)
3057{
3058 unsigned i;
3059
3060 for (i = 0; i < adev->debugfs_count; i++) {
3061 if (adev->debugfs[i].files == files) {
3062 /* Already registered */
3063 return 0;
3064 }
3065 }
3066
3067 i = adev->debugfs_count + 1;
3068 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
3069 DRM_ERROR("Reached maximum number of debugfs components.\n");
3070 DRM_ERROR("Report so we increase "
3071 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
3072 return -EINVAL;
3073 }
3074 adev->debugfs[adev->debugfs_count].files = files;
3075 adev->debugfs[adev->debugfs_count].num_files = nfiles;
3076 adev->debugfs_count = i;
3077#if defined(CONFIG_DEBUG_FS)
d38ceaf9
AD
3078 drm_debugfs_create_files(files, nfiles,
3079 adev->ddev->primary->debugfs_root,
3080 adev->ddev->primary);
3081#endif
3082 return 0;
3083}
3084
d38ceaf9
AD
3085#if defined(CONFIG_DEBUG_FS)
3086
3087static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
3088 size_t size, loff_t *pos)
3089{
45063097 3090 struct amdgpu_device *adev = file_inode(f)->i_private;
d38ceaf9
AD
3091 ssize_t result = 0;
3092 int r;
bd12267d 3093 bool pm_pg_lock, use_bank;
56628159 3094 unsigned instance_bank, sh_bank, se_bank;
d38ceaf9
AD
3095
3096 if (size & 0x3 || *pos & 0x3)
3097 return -EINVAL;
3098
bd12267d
TSD
3099 /* are we reading registers for which a PG lock is necessary? */
3100 pm_pg_lock = (*pos >> 23) & 1;
3101
56628159
TSD
3102 if (*pos & (1ULL << 62)) {
3103 se_bank = (*pos >> 24) & 0x3FF;
3104 sh_bank = (*pos >> 34) & 0x3FF;
3105 instance_bank = (*pos >> 44) & 0x3FF;
32977f93
TSD
3106
3107 if (se_bank == 0x3FF)
3108 se_bank = 0xFFFFFFFF;
3109 if (sh_bank == 0x3FF)
3110 sh_bank = 0xFFFFFFFF;
3111 if (instance_bank == 0x3FF)
3112 instance_bank = 0xFFFFFFFF;
56628159 3113 use_bank = 1;
56628159
TSD
3114 } else {
3115 use_bank = 0;
3116 }
3117
801a6aa9 3118 *pos &= (1UL << 22) - 1;
bd12267d 3119
56628159 3120 if (use_bank) {
32977f93
TSD
3121 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3122 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
56628159
TSD
3123 return -EINVAL;
3124 mutex_lock(&adev->grbm_idx_mutex);
3125 amdgpu_gfx_select_se_sh(adev, se_bank,
3126 sh_bank, instance_bank);
3127 }
3128
bd12267d
TSD
3129 if (pm_pg_lock)
3130 mutex_lock(&adev->pm.mutex);
3131
d38ceaf9
AD
3132 while (size) {
3133 uint32_t value;
3134
3135 if (*pos > adev->rmmio_size)
56628159 3136 goto end;
d38ceaf9
AD
3137
3138 value = RREG32(*pos >> 2);
3139 r = put_user(value, (uint32_t *)buf);
56628159
TSD
3140 if (r) {
3141 result = r;
3142 goto end;
3143 }
d38ceaf9
AD
3144
3145 result += 4;
3146 buf += 4;
3147 *pos += 4;
3148 size -= 4;
3149 }
3150
56628159
TSD
3151end:
3152 if (use_bank) {
3153 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3154 mutex_unlock(&adev->grbm_idx_mutex);
3155 }
3156
bd12267d
TSD
3157 if (pm_pg_lock)
3158 mutex_unlock(&adev->pm.mutex);
3159
d38ceaf9
AD
3160 return result;
3161}
3162
3163static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
3164 size_t size, loff_t *pos)
3165{
45063097 3166 struct amdgpu_device *adev = file_inode(f)->i_private;
d38ceaf9
AD
3167 ssize_t result = 0;
3168 int r;
394fdde2
TSD
3169 bool pm_pg_lock, use_bank;
3170 unsigned instance_bank, sh_bank, se_bank;
d38ceaf9
AD
3171
3172 if (size & 0x3 || *pos & 0x3)
3173 return -EINVAL;
3174
394fdde2
TSD
3175 /* are we reading registers for which a PG lock is necessary? */
3176 pm_pg_lock = (*pos >> 23) & 1;
3177
3178 if (*pos & (1ULL << 62)) {
3179 se_bank = (*pos >> 24) & 0x3FF;
3180 sh_bank = (*pos >> 34) & 0x3FF;
3181 instance_bank = (*pos >> 44) & 0x3FF;
3182
3183 if (se_bank == 0x3FF)
3184 se_bank = 0xFFFFFFFF;
3185 if (sh_bank == 0x3FF)
3186 sh_bank = 0xFFFFFFFF;
3187 if (instance_bank == 0x3FF)
3188 instance_bank = 0xFFFFFFFF;
3189 use_bank = 1;
3190 } else {
3191 use_bank = 0;
3192 }
3193
801a6aa9 3194 *pos &= (1UL << 22) - 1;
394fdde2
TSD
3195
3196 if (use_bank) {
3197 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3198 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3199 return -EINVAL;
3200 mutex_lock(&adev->grbm_idx_mutex);
3201 amdgpu_gfx_select_se_sh(adev, se_bank,
3202 sh_bank, instance_bank);
3203 }
3204
3205 if (pm_pg_lock)
3206 mutex_lock(&adev->pm.mutex);
3207
d38ceaf9
AD
3208 while (size) {
3209 uint32_t value;
3210
3211 if (*pos > adev->rmmio_size)
3212 return result;
3213
3214 r = get_user(value, (uint32_t *)buf);
3215 if (r)
3216 return r;
3217
3218 WREG32(*pos >> 2, value);
3219
3220 result += 4;
3221 buf += 4;
3222 *pos += 4;
3223 size -= 4;
3224 }
3225
394fdde2
TSD
3226 if (use_bank) {
3227 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3228 mutex_unlock(&adev->grbm_idx_mutex);
3229 }
3230
3231 if (pm_pg_lock)
3232 mutex_unlock(&adev->pm.mutex);
3233
d38ceaf9
AD
3234 return result;
3235}
3236
adcec288
TSD
3237static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
3238 size_t size, loff_t *pos)
3239{
45063097 3240 struct amdgpu_device *adev = file_inode(f)->i_private;
adcec288
TSD
3241 ssize_t result = 0;
3242 int r;
3243
3244 if (size & 0x3 || *pos & 0x3)
3245 return -EINVAL;
3246
3247 while (size) {
3248 uint32_t value;
3249
3250 value = RREG32_PCIE(*pos >> 2);
3251 r = put_user(value, (uint32_t *)buf);
3252 if (r)
3253 return r;
3254
3255 result += 4;
3256 buf += 4;
3257 *pos += 4;
3258 size -= 4;
3259 }
3260
3261 return result;
3262}
3263
3264static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
3265 size_t size, loff_t *pos)
3266{
45063097 3267 struct amdgpu_device *adev = file_inode(f)->i_private;
adcec288
TSD
3268 ssize_t result = 0;
3269 int r;
3270
3271 if (size & 0x3 || *pos & 0x3)
3272 return -EINVAL;
3273
3274 while (size) {
3275 uint32_t value;
3276
3277 r = get_user(value, (uint32_t *)buf);
3278 if (r)
3279 return r;
3280
3281 WREG32_PCIE(*pos >> 2, value);
3282
3283 result += 4;
3284 buf += 4;
3285 *pos += 4;
3286 size -= 4;
3287 }
3288
3289 return result;
3290}
3291
3292static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
3293 size_t size, loff_t *pos)
3294{
45063097 3295 struct amdgpu_device *adev = file_inode(f)->i_private;
adcec288
TSD
3296 ssize_t result = 0;
3297 int r;
3298
3299 if (size & 0x3 || *pos & 0x3)
3300 return -EINVAL;
3301
3302 while (size) {
3303 uint32_t value;
3304
3305 value = RREG32_DIDT(*pos >> 2);
3306 r = put_user(value, (uint32_t *)buf);
3307 if (r)
3308 return r;
3309
3310 result += 4;
3311 buf += 4;
3312 *pos += 4;
3313 size -= 4;
3314 }
3315
3316 return result;
3317}
3318
3319static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
3320 size_t size, loff_t *pos)
3321{
45063097 3322 struct amdgpu_device *adev = file_inode(f)->i_private;
adcec288
TSD
3323 ssize_t result = 0;
3324 int r;
3325
3326 if (size & 0x3 || *pos & 0x3)
3327 return -EINVAL;
3328
3329 while (size) {
3330 uint32_t value;
3331
3332 r = get_user(value, (uint32_t *)buf);
3333 if (r)
3334 return r;
3335
3336 WREG32_DIDT(*pos >> 2, value);
3337
3338 result += 4;
3339 buf += 4;
3340 *pos += 4;
3341 size -= 4;
3342 }
3343
3344 return result;
3345}
3346
3347static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
3348 size_t size, loff_t *pos)
3349{
45063097 3350 struct amdgpu_device *adev = file_inode(f)->i_private;
adcec288
TSD
3351 ssize_t result = 0;
3352 int r;
3353
3354 if (size & 0x3 || *pos & 0x3)
3355 return -EINVAL;
3356
3357 while (size) {
3358 uint32_t value;
3359
6fc0deaf 3360 value = RREG32_SMC(*pos);
adcec288
TSD
3361 r = put_user(value, (uint32_t *)buf);
3362 if (r)
3363 return r;
3364
3365 result += 4;
3366 buf += 4;
3367 *pos += 4;
3368 size -= 4;
3369 }
3370
3371 return result;
3372}
3373
3374static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
3375 size_t size, loff_t *pos)
3376{
45063097 3377 struct amdgpu_device *adev = file_inode(f)->i_private;
adcec288
TSD
3378 ssize_t result = 0;
3379 int r;
3380
3381 if (size & 0x3 || *pos & 0x3)
3382 return -EINVAL;
3383
3384 while (size) {
3385 uint32_t value;
3386
3387 r = get_user(value, (uint32_t *)buf);
3388 if (r)
3389 return r;
3390
6fc0deaf 3391 WREG32_SMC(*pos, value);
adcec288
TSD
3392
3393 result += 4;
3394 buf += 4;
3395 *pos += 4;
3396 size -= 4;
3397 }
3398
3399 return result;
3400}
3401
1e051413
TSD
3402static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
3403 size_t size, loff_t *pos)
3404{
45063097 3405 struct amdgpu_device *adev = file_inode(f)->i_private;
1e051413
TSD
3406 ssize_t result = 0;
3407 int r;
3408 uint32_t *config, no_regs = 0;
3409
3410 if (size & 0x3 || *pos & 0x3)
3411 return -EINVAL;
3412
ecab7668 3413 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
1e051413
TSD
3414 if (!config)
3415 return -ENOMEM;
3416
3417 /* version, increment each time something is added */
9a999359 3418 config[no_regs++] = 3;
1e051413
TSD
3419 config[no_regs++] = adev->gfx.config.max_shader_engines;
3420 config[no_regs++] = adev->gfx.config.max_tile_pipes;
3421 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
3422 config[no_regs++] = adev->gfx.config.max_sh_per_se;
3423 config[no_regs++] = adev->gfx.config.max_backends_per_se;
3424 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
3425 config[no_regs++] = adev->gfx.config.max_gprs;
3426 config[no_regs++] = adev->gfx.config.max_gs_threads;
3427 config[no_regs++] = adev->gfx.config.max_hw_contexts;
3428 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
3429 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
3430 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
3431 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
3432 config[no_regs++] = adev->gfx.config.num_tile_pipes;
3433 config[no_regs++] = adev->gfx.config.backend_enable_mask;
3434 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
3435 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
3436 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
3437 config[no_regs++] = adev->gfx.config.num_gpus;
3438 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
3439 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
3440 config[no_regs++] = adev->gfx.config.gb_addr_config;
3441 config[no_regs++] = adev->gfx.config.num_rbs;
3442
89a8f309
TSD
3443 /* rev==1 */
3444 config[no_regs++] = adev->rev_id;
3445 config[no_regs++] = adev->pg_flags;
3446 config[no_regs++] = adev->cg_flags;
3447
e9f11dc8
TSD
3448 /* rev==2 */
3449 config[no_regs++] = adev->family;
3450 config[no_regs++] = adev->external_rev_id;
3451
9a999359
TSD
3452 /* rev==3 */
3453 config[no_regs++] = adev->pdev->device;
3454 config[no_regs++] = adev->pdev->revision;
3455 config[no_regs++] = adev->pdev->subsystem_device;
3456 config[no_regs++] = adev->pdev->subsystem_vendor;
3457
1e051413
TSD
3458 while (size && (*pos < no_regs * 4)) {
3459 uint32_t value;
3460
3461 value = config[*pos >> 2];
3462 r = put_user(value, (uint32_t *)buf);
3463 if (r) {
3464 kfree(config);
3465 return r;
3466 }
3467
3468 result += 4;
3469 buf += 4;
3470 *pos += 4;
3471 size -= 4;
3472 }
3473
3474 kfree(config);
3475 return result;
3476}
3477
f2cdaf20
TSD
3478static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
3479 size_t size, loff_t *pos)
3480{
45063097 3481 struct amdgpu_device *adev = file_inode(f)->i_private;
9f8df7d7
TSD
3482 int idx, x, outsize, r, valuesize;
3483 uint32_t values[16];
f2cdaf20 3484
9f8df7d7 3485 if (size & 3 || *pos & 0x3)
f2cdaf20
TSD
3486 return -EINVAL;
3487
3cbc614f
SP
3488 if (amdgpu_dpm == 0)
3489 return -EINVAL;
3490
f2cdaf20
TSD
3491 /* convert offset to sensor number */
3492 idx = *pos >> 2;
3493
9f8df7d7 3494 valuesize = sizeof(values);
f2cdaf20 3495 if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
9f8df7d7 3496 r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
3cbc614f
SP
3497 else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
3498 r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
3499 &valuesize);
f2cdaf20
TSD
3500 else
3501 return -EINVAL;
3502
9f8df7d7
TSD
3503 if (size > valuesize)
3504 return -EINVAL;
3505
3506 outsize = 0;
3507 x = 0;
3508 if (!r) {
3509 while (size) {
3510 r = put_user(values[x++], (int32_t *)buf);
3511 buf += 4;
3512 size -= 4;
3513 outsize += 4;
3514 }
3515 }
f2cdaf20 3516
9f8df7d7 3517 return !r ? outsize : r;
f2cdaf20 3518}
1e051413 3519
273d7aa1
TSD
3520static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
3521 size_t size, loff_t *pos)
3522{
3523 struct amdgpu_device *adev = f->f_inode->i_private;
3524 int r, x;
3525 ssize_t result=0;
472259f0 3526 uint32_t offset, se, sh, cu, wave, simd, data[32];
273d7aa1
TSD
3527
3528 if (size & 3 || *pos & 3)
3529 return -EINVAL;
3530
3531 /* decode offset */
3532 offset = (*pos & 0x7F);
3533 se = ((*pos >> 7) & 0xFF);
3534 sh = ((*pos >> 15) & 0xFF);
3535 cu = ((*pos >> 23) & 0xFF);
3536 wave = ((*pos >> 31) & 0xFF);
3537 simd = ((*pos >> 37) & 0xFF);
273d7aa1
TSD
3538
3539 /* switch to the specific se/sh/cu */
3540 mutex_lock(&adev->grbm_idx_mutex);
3541 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3542
3543 x = 0;
472259f0
TSD
3544 if (adev->gfx.funcs->read_wave_data)
3545 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
273d7aa1
TSD
3546
3547 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3548 mutex_unlock(&adev->grbm_idx_mutex);
3549
5ecfb3b8
TSD
3550 if (!x)
3551 return -EINVAL;
3552
472259f0 3553 while (size && (offset < x * 4)) {
273d7aa1
TSD
3554 uint32_t value;
3555
472259f0 3556 value = data[offset >> 2];
273d7aa1
TSD
3557 r = put_user(value, (uint32_t *)buf);
3558 if (r)
3559 return r;
3560
3561 result += 4;
3562 buf += 4;
472259f0 3563 offset += 4;
273d7aa1
TSD
3564 size -= 4;
3565 }
3566
3567 return result;
3568}
3569
c5a60ce8
TSD
3570static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3571 size_t size, loff_t *pos)
3572{
3573 struct amdgpu_device *adev = f->f_inode->i_private;
3574 int r;
3575 ssize_t result = 0;
3576 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3577
3578 if (size & 3 || *pos & 3)
3579 return -EINVAL;
3580
3581 /* decode offset */
3582 offset = (*pos & 0xFFF); /* in dwords */
3583 se = ((*pos >> 12) & 0xFF);
3584 sh = ((*pos >> 20) & 0xFF);
3585 cu = ((*pos >> 28) & 0xFF);
3586 wave = ((*pos >> 36) & 0xFF);
3587 simd = ((*pos >> 44) & 0xFF);
3588 thread = ((*pos >> 52) & 0xFF);
3589 bank = ((*pos >> 60) & 1);
3590
3591 data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3592 if (!data)
3593 return -ENOMEM;
3594
3595 /* switch to the specific se/sh/cu */
3596 mutex_lock(&adev->grbm_idx_mutex);
3597 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3598
3599 if (bank == 0) {
3600 if (adev->gfx.funcs->read_wave_vgprs)
3601 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3602 } else {
3603 if (adev->gfx.funcs->read_wave_sgprs)
3604 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3605 }
3606
3607 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3608 mutex_unlock(&adev->grbm_idx_mutex);
3609
3610 while (size) {
3611 uint32_t value;
3612
3613 value = data[offset++];
3614 r = put_user(value, (uint32_t *)buf);
3615 if (r) {
3616 result = r;
3617 goto err;
3618 }
3619
3620 result += 4;
3621 buf += 4;
3622 size -= 4;
3623 }
3624
3625err:
3626 kfree(data);
3627 return result;
3628}
3629
d38ceaf9
AD
3630static const struct file_operations amdgpu_debugfs_regs_fops = {
3631 .owner = THIS_MODULE,
3632 .read = amdgpu_debugfs_regs_read,
3633 .write = amdgpu_debugfs_regs_write,
3634 .llseek = default_llseek
3635};
adcec288
TSD
3636static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3637 .owner = THIS_MODULE,
3638 .read = amdgpu_debugfs_regs_didt_read,
3639 .write = amdgpu_debugfs_regs_didt_write,
3640 .llseek = default_llseek
3641};
3642static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3643 .owner = THIS_MODULE,
3644 .read = amdgpu_debugfs_regs_pcie_read,
3645 .write = amdgpu_debugfs_regs_pcie_write,
3646 .llseek = default_llseek
3647};
3648static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3649 .owner = THIS_MODULE,
3650 .read = amdgpu_debugfs_regs_smc_read,
3651 .write = amdgpu_debugfs_regs_smc_write,
3652 .llseek = default_llseek
3653};
3654
1e051413
TSD
3655static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3656 .owner = THIS_MODULE,
3657 .read = amdgpu_debugfs_gca_config_read,
3658 .llseek = default_llseek
3659};
3660
f2cdaf20
TSD
3661static const struct file_operations amdgpu_debugfs_sensors_fops = {
3662 .owner = THIS_MODULE,
3663 .read = amdgpu_debugfs_sensor_read,
3664 .llseek = default_llseek
3665};
3666
273d7aa1
TSD
3667static const struct file_operations amdgpu_debugfs_wave_fops = {
3668 .owner = THIS_MODULE,
3669 .read = amdgpu_debugfs_wave_read,
3670 .llseek = default_llseek
3671};
c5a60ce8
TSD
3672static const struct file_operations amdgpu_debugfs_gpr_fops = {
3673 .owner = THIS_MODULE,
3674 .read = amdgpu_debugfs_gpr_read,
3675 .llseek = default_llseek
3676};
273d7aa1 3677
adcec288
TSD
3678static const struct file_operations *debugfs_regs[] = {
3679 &amdgpu_debugfs_regs_fops,
3680 &amdgpu_debugfs_regs_didt_fops,
3681 &amdgpu_debugfs_regs_pcie_fops,
3682 &amdgpu_debugfs_regs_smc_fops,
1e051413 3683 &amdgpu_debugfs_gca_config_fops,
f2cdaf20 3684 &amdgpu_debugfs_sensors_fops,
273d7aa1 3685 &amdgpu_debugfs_wave_fops,
c5a60ce8 3686 &amdgpu_debugfs_gpr_fops,
adcec288
TSD
3687};
3688
3689static const char *debugfs_regs_names[] = {
3690 "amdgpu_regs",
3691 "amdgpu_regs_didt",
3692 "amdgpu_regs_pcie",
3693 "amdgpu_regs_smc",
1e051413 3694 "amdgpu_gca_config",
f2cdaf20 3695 "amdgpu_sensors",
273d7aa1 3696 "amdgpu_wave",
c5a60ce8 3697 "amdgpu_gpr",
adcec288 3698};
d38ceaf9
AD
3699
3700static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3701{
3702 struct drm_minor *minor = adev->ddev->primary;
3703 struct dentry *ent, *root = minor->debugfs_root;
adcec288
TSD
3704 unsigned i, j;
3705
3706 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3707 ent = debugfs_create_file(debugfs_regs_names[i],
3708 S_IFREG | S_IRUGO, root,
3709 adev, debugfs_regs[i]);
3710 if (IS_ERR(ent)) {
3711 for (j = 0; j < i; j++) {
3712 debugfs_remove(adev->debugfs_regs[i]);
3713 adev->debugfs_regs[i] = NULL;
3714 }
3715 return PTR_ERR(ent);
3716 }
d38ceaf9 3717
adcec288
TSD
3718 if (!i)
3719 i_size_write(ent->d_inode, adev->rmmio_size);
3720 adev->debugfs_regs[i] = ent;
3721 }
d38ceaf9
AD
3722
3723 return 0;
3724}
3725
3726static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3727{
adcec288
TSD
3728 unsigned i;
3729
3730 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3731 if (adev->debugfs_regs[i]) {
3732 debugfs_remove(adev->debugfs_regs[i]);
3733 adev->debugfs_regs[i] = NULL;
3734 }
3735 }
d38ceaf9
AD
3736}
3737
4f0955fc
HR
3738static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
3739{
3740 struct drm_info_node *node = (struct drm_info_node *) m->private;
3741 struct drm_device *dev = node->minor->dev;
3742 struct amdgpu_device *adev = dev->dev_private;
3743 int r = 0, i;
3744
3745 /* hold on the scheduler */
3746 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3747 struct amdgpu_ring *ring = adev->rings[i];
3748
3749 if (!ring || !ring->sched.thread)
3750 continue;
3751 kthread_park(ring->sched.thread);
3752 }
3753
3754 seq_printf(m, "run ib test:\n");
3755 r = amdgpu_ib_ring_tests(adev);
3756 if (r)
3757 seq_printf(m, "ib ring tests failed (%d).\n", r);
3758 else
3759 seq_printf(m, "ib ring tests passed.\n");
3760
3761 /* go on the scheduler */
3762 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3763 struct amdgpu_ring *ring = adev->rings[i];
3764
3765 if (!ring || !ring->sched.thread)
3766 continue;
3767 kthread_unpark(ring->sched.thread);
3768 }
3769
3770 return 0;
3771}
3772
3773static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
3774 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
3775};
3776
3777static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3778{
3779 return amdgpu_debugfs_add_files(adev,
3780 amdgpu_debugfs_test_ib_ring_list, 1);
3781}
3782
d38ceaf9
AD
3783int amdgpu_debugfs_init(struct drm_minor *minor)
3784{
3785 return 0;
3786}
7cebc728 3787#else
27bad5b9 3788static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
4f0955fc
HR
3789{
3790 return 0;
3791}
7cebc728
AK
3792static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3793{
3794 return 0;
3795}
3796static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
d38ceaf9 3797#endif