drm/amd/display: VGA black screen from s3 when attached to hook
[linux-block.git] / drivers / gpu / drm / radeon / radeon_device.c
CommitLineData
771fe6b9
JG
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/console.h>
5a0e3ad6 29#include <linux/slab.h>
771fe6b9
JG
30#include <drm/drmP.h>
31#include <drm/drm_crtc_helper.h>
32#include <drm/radeon_drm.h>
b8751946 33#include <linux/pm_runtime.h>
28d52043 34#include <linux/vgaarb.h>
6a9ee8af 35#include <linux/vga_switcheroo.h>
bcc65fd8 36#include <linux/efi.h>
771fe6b9
JG
37#include "radeon_reg.h"
38#include "radeon.h"
771fe6b9
JG
39#include "atom.h"
40
1b5331d9
JG
41static const char radeon_family_name[][16] = {
42 "R100",
43 "RV100",
44 "RS100",
45 "RV200",
46 "RS200",
47 "R200",
48 "RV250",
49 "RS300",
50 "RV280",
51 "R300",
52 "R350",
53 "RV350",
54 "RV380",
55 "R420",
56 "R423",
57 "RV410",
58 "RS400",
59 "RS480",
60 "RS600",
61 "RS690",
62 "RS740",
63 "RV515",
64 "R520",
65 "RV530",
66 "RV560",
67 "RV570",
68 "R580",
69 "R600",
70 "RV610",
71 "RV630",
72 "RV670",
73 "RV620",
74 "RV635",
75 "RS780",
76 "RS880",
77 "RV770",
78 "RV730",
79 "RV710",
80 "RV740",
81 "CEDAR",
82 "REDWOOD",
83 "JUNIPER",
84 "CYPRESS",
85 "HEMLOCK",
b08ebe7e 86 "PALM",
4df64e65
AD
87 "SUMO",
88 "SUMO2",
1fe18305
AD
89 "BARTS",
90 "TURKS",
91 "CAICOS",
b7cfc9fe 92 "CAYMAN",
8848f759 93 "ARUBA",
cb28bb34
AD
94 "TAHITI",
95 "PITCAIRN",
96 "VERDE",
624d3524 97 "OLAND",
b5d9d726 98 "HAINAN",
6eac752e
AD
99 "BONAIRE",
100 "KAVERI",
101 "KABINI",
3bf599e8 102 "HAWAII",
b0a9f22a 103 "MULLINS",
1b5331d9
JG
104 "LAST",
105};
106
066f1f0b
AD
107#if defined(CONFIG_VGA_SWITCHEROO)
108bool radeon_has_atpx_dgpu_power_cntl(void);
109bool radeon_is_atpx_hybrid(void);
110#else
111static inline bool radeon_has_atpx_dgpu_power_cntl(void) { return false; }
112static inline bool radeon_is_atpx_hybrid(void) { return false; }
113#endif
114
4807c5a8 115#define RADEON_PX_QUIRK_DISABLE_PX (1 << 0)
4807c5a8
AD
116
117struct radeon_px_quirk {
118 u32 chip_vendor;
119 u32 chip_device;
120 u32 subsys_vendor;
121 u32 subsys_device;
122 u32 px_quirk_flags;
123};
124
125static struct radeon_px_quirk radeon_px_quirk_list[] = {
126 /* Acer aspire 5560g (CPU: AMD A4-3305M; GPU: AMD Radeon HD 6480g + 7470m)
127 * https://bugzilla.kernel.org/show_bug.cgi?id=74551
128 */
129 { PCI_VENDOR_ID_ATI, 0x6760, 0x1025, 0x0672, RADEON_PX_QUIRK_DISABLE_PX },
130 /* Asus K73TA laptop with AMD A6-3400M APU and Radeon 6550 GPU
131 * https://bugzilla.kernel.org/show_bug.cgi?id=51381
132 */
133 { PCI_VENDOR_ID_ATI, 0x6741, 0x1043, 0x108c, RADEON_PX_QUIRK_DISABLE_PX },
ff1b1294
AD
134 /* Asus K53TK laptop with AMD A6-3420M APU and Radeon 7670m GPU
135 * https://bugzilla.kernel.org/show_bug.cgi?id=51381
136 */
137 { PCI_VENDOR_ID_ATI, 0x6840, 0x1043, 0x2122, RADEON_PX_QUIRK_DISABLE_PX },
4eb59793
AD
138 /* Asus K53TK laptop with AMD A6-3420M APU and Radeon 7670m GPU
139 * https://bugs.freedesktop.org/show_bug.cgi?id=101491
140 */
141 { PCI_VENDOR_ID_ATI, 0x6741, 0x1043, 0x2122, RADEON_PX_QUIRK_DISABLE_PX },
4807c5a8
AD
142 { 0, 0, 0, 0, 0 },
143};
144
90c4cde9
AD
145bool radeon_is_px(struct drm_device *dev)
146{
147 struct radeon_device *rdev = dev->dev_private;
148
149 if (rdev->flags & RADEON_IS_PX)
150 return true;
151 return false;
152}
10ebc0bc 153
4807c5a8
AD
154static void radeon_device_handle_px_quirks(struct radeon_device *rdev)
155{
156 struct radeon_px_quirk *p = radeon_px_quirk_list;
157
158 /* Apply PX quirks */
159 while (p && p->chip_device != 0) {
160 if (rdev->pdev->vendor == p->chip_vendor &&
161 rdev->pdev->device == p->chip_device &&
162 rdev->pdev->subsystem_vendor == p->subsys_vendor &&
163 rdev->pdev->subsystem_device == p->subsys_device) {
164 rdev->px_quirk_flags = p->px_quirk_flags;
165 break;
166 }
167 ++p;
168 }
169
170 if (rdev->px_quirk_flags & RADEON_PX_QUIRK_DISABLE_PX)
171 rdev->flags &= ~RADEON_IS_PX;
066f1f0b
AD
172
173 /* disable PX is the system doesn't support dGPU power control or hybrid gfx */
174 if (!radeon_is_atpx_hybrid() &&
175 !radeon_has_atpx_dgpu_power_cntl())
176 rdev->flags &= ~RADEON_IS_PX;
4807c5a8
AD
177}
178
2e1b65f9
AD
179/**
180 * radeon_program_register_sequence - program an array of registers.
181 *
182 * @rdev: radeon_device pointer
183 * @registers: pointer to the register array
184 * @array_size: size of the register array
185 *
186 * Programs an array or registers with and and or masks.
187 * This is a helper for setting golden registers.
188 */
189void radeon_program_register_sequence(struct radeon_device *rdev,
190 const u32 *registers,
191 const u32 array_size)
192{
193 u32 tmp, reg, and_mask, or_mask;
194 int i;
195
196 if (array_size % 3)
197 return;
198
199 for (i = 0; i < array_size; i +=3) {
200 reg = registers[i + 0];
201 and_mask = registers[i + 1];
202 or_mask = registers[i + 2];
203
204 if (and_mask == 0xffffffff) {
205 tmp = or_mask;
206 } else {
207 tmp = RREG32(reg);
208 tmp &= ~and_mask;
209 tmp |= or_mask;
210 }
211 WREG32(reg, tmp);
212 }
213}
214
1a0041b8
AD
215void radeon_pci_config_reset(struct radeon_device *rdev)
216{
217 pci_write_config_dword(rdev->pdev, 0x7c, RADEON_ASIC_RESET_DATA);
218}
219
0c195119
AD
220/**
221 * radeon_surface_init - Clear GPU surface registers.
222 *
223 * @rdev: radeon_device pointer
224 *
225 * Clear GPU surface registers (r1xx-r5xx).
b1e3a6d1 226 */
3ce0a23d 227void radeon_surface_init(struct radeon_device *rdev)
b1e3a6d1
MD
228{
229 /* FIXME: check this out */
230 if (rdev->family < CHIP_R600) {
231 int i;
232
550e2d92
DA
233 for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
234 if (rdev->surface_regs[i].bo)
235 radeon_bo_get_surface_reg(rdev->surface_regs[i].bo);
236 else
237 radeon_clear_surface_reg(rdev, i);
b1e3a6d1 238 }
e024e110
DA
239 /* enable surfaces */
240 WREG32(RADEON_SURFACE_CNTL, 0);
b1e3a6d1
MD
241 }
242}
243
771fe6b9
JG
244/*
245 * GPU scratch registers helpers function.
246 */
0c195119
AD
247/**
248 * radeon_scratch_init - Init scratch register driver information.
249 *
250 * @rdev: radeon_device pointer
251 *
252 * Init CP scratch register driver information (r1xx-r5xx)
253 */
3ce0a23d 254void radeon_scratch_init(struct radeon_device *rdev)
771fe6b9
JG
255{
256 int i;
257
258 /* FIXME: check this out */
259 if (rdev->family < CHIP_R300) {
260 rdev->scratch.num_reg = 5;
261 } else {
262 rdev->scratch.num_reg = 7;
263 }
724c80e1 264 rdev->scratch.reg_base = RADEON_SCRATCH_REG0;
771fe6b9
JG
265 for (i = 0; i < rdev->scratch.num_reg; i++) {
266 rdev->scratch.free[i] = true;
724c80e1 267 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
771fe6b9
JG
268 }
269}
270
0c195119
AD
271/**
272 * radeon_scratch_get - Allocate a scratch register
273 *
274 * @rdev: radeon_device pointer
275 * @reg: scratch register mmio offset
276 *
277 * Allocate a CP scratch register for use by the driver (all asics).
278 * Returns 0 on success or -EINVAL on failure.
279 */
771fe6b9
JG
280int radeon_scratch_get(struct radeon_device *rdev, uint32_t *reg)
281{
282 int i;
283
284 for (i = 0; i < rdev->scratch.num_reg; i++) {
285 if (rdev->scratch.free[i]) {
286 rdev->scratch.free[i] = false;
287 *reg = rdev->scratch.reg[i];
288 return 0;
289 }
290 }
291 return -EINVAL;
292}
293
0c195119
AD
294/**
295 * radeon_scratch_free - Free a scratch register
296 *
297 * @rdev: radeon_device pointer
298 * @reg: scratch register mmio offset
299 *
300 * Free a CP scratch register allocated for use by the driver (all asics)
301 */
771fe6b9
JG
302void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg)
303{
304 int i;
305
306 for (i = 0; i < rdev->scratch.num_reg; i++) {
307 if (rdev->scratch.reg[i] == reg) {
308 rdev->scratch.free[i] = true;
309 return;
310 }
311 }
312}
313
75efdee1
AD
314/*
315 * GPU doorbell aperture helpers function.
316 */
317/**
318 * radeon_doorbell_init - Init doorbell driver information.
319 *
320 * @rdev: radeon_device pointer
321 *
322 * Init doorbell driver information (CIK)
323 * Returns 0 on success, error on failure.
324 */
28f5a6cd 325static int radeon_doorbell_init(struct radeon_device *rdev)
75efdee1 326{
75efdee1
AD
327 /* doorbell bar mapping */
328 rdev->doorbell.base = pci_resource_start(rdev->pdev, 2);
329 rdev->doorbell.size = pci_resource_len(rdev->pdev, 2);
330
d5754ab8
AL
331 rdev->doorbell.num_doorbells = min_t(u32, rdev->doorbell.size / sizeof(u32), RADEON_MAX_DOORBELLS);
332 if (rdev->doorbell.num_doorbells == 0)
333 return -EINVAL;
75efdee1 334
d5754ab8 335 rdev->doorbell.ptr = ioremap(rdev->doorbell.base, rdev->doorbell.num_doorbells * sizeof(u32));
75efdee1
AD
336 if (rdev->doorbell.ptr == NULL) {
337 return -ENOMEM;
338 }
339 DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)rdev->doorbell.base);
340 DRM_INFO("doorbell mmio size: %u\n", (unsigned)rdev->doorbell.size);
341
d5754ab8 342 memset(&rdev->doorbell.used, 0, sizeof(rdev->doorbell.used));
75efdee1 343
75efdee1
AD
344 return 0;
345}
346
347/**
348 * radeon_doorbell_fini - Tear down doorbell driver information.
349 *
350 * @rdev: radeon_device pointer
351 *
352 * Tear down doorbell driver information (CIK)
353 */
28f5a6cd 354static void radeon_doorbell_fini(struct radeon_device *rdev)
75efdee1
AD
355{
356 iounmap(rdev->doorbell.ptr);
357 rdev->doorbell.ptr = NULL;
358}
359
360/**
d5754ab8 361 * radeon_doorbell_get - Allocate a doorbell entry
75efdee1
AD
362 *
363 * @rdev: radeon_device pointer
d5754ab8 364 * @doorbell: doorbell index
75efdee1 365 *
d5754ab8 366 * Allocate a doorbell for use by the driver (all asics).
75efdee1
AD
367 * Returns 0 on success or -EINVAL on failure.
368 */
369int radeon_doorbell_get(struct radeon_device *rdev, u32 *doorbell)
370{
d5754ab8
AL
371 unsigned long offset = find_first_zero_bit(rdev->doorbell.used, rdev->doorbell.num_doorbells);
372 if (offset < rdev->doorbell.num_doorbells) {
373 __set_bit(offset, rdev->doorbell.used);
374 *doorbell = offset;
375 return 0;
376 } else {
377 return -EINVAL;
75efdee1 378 }
75efdee1
AD
379}
380
381/**
d5754ab8 382 * radeon_doorbell_free - Free a doorbell entry
75efdee1
AD
383 *
384 * @rdev: radeon_device pointer
d5754ab8 385 * @doorbell: doorbell index
75efdee1 386 *
d5754ab8 387 * Free a doorbell allocated for use by the driver (all asics)
75efdee1
AD
388 */
389void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell)
390{
d5754ab8
AL
391 if (doorbell < rdev->doorbell.num_doorbells)
392 __clear_bit(doorbell, rdev->doorbell.used);
75efdee1
AD
393}
394
0c195119
AD
395/*
396 * radeon_wb_*()
397 * Writeback is the the method by which the the GPU updates special pages
398 * in memory with the status of certain GPU events (fences, ring pointers,
399 * etc.).
400 */
401
402/**
403 * radeon_wb_disable - Disable Writeback
404 *
405 * @rdev: radeon_device pointer
406 *
407 * Disables Writeback (all asics). Used for suspend.
408 */
724c80e1
AD
409void radeon_wb_disable(struct radeon_device *rdev)
410{
724c80e1
AD
411 rdev->wb.enabled = false;
412}
413
0c195119
AD
414/**
415 * radeon_wb_fini - Disable Writeback and free memory
416 *
417 * @rdev: radeon_device pointer
418 *
419 * Disables Writeback and frees the Writeback memory (all asics).
420 * Used at driver shutdown.
421 */
724c80e1
AD
422void radeon_wb_fini(struct radeon_device *rdev)
423{
424 radeon_wb_disable(rdev);
425 if (rdev->wb.wb_obj) {
089920f2
JG
426 if (!radeon_bo_reserve(rdev->wb.wb_obj, false)) {
427 radeon_bo_kunmap(rdev->wb.wb_obj);
428 radeon_bo_unpin(rdev->wb.wb_obj);
429 radeon_bo_unreserve(rdev->wb.wb_obj);
430 }
724c80e1
AD
431 radeon_bo_unref(&rdev->wb.wb_obj);
432 rdev->wb.wb = NULL;
433 rdev->wb.wb_obj = NULL;
434 }
435}
436
0c195119
AD
437/**
438 * radeon_wb_init- Init Writeback driver info and allocate memory
439 *
440 * @rdev: radeon_device pointer
441 *
442 * Disables Writeback and frees the Writeback memory (all asics).
443 * Used at driver startup.
444 * Returns 0 on success or an -error on failure.
445 */
724c80e1
AD
446int radeon_wb_init(struct radeon_device *rdev)
447{
448 int r;
449
450 if (rdev->wb.wb_obj == NULL) {
441921d5 451 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
831b6966 452 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
02376d82 453 &rdev->wb.wb_obj);
724c80e1
AD
454 if (r) {
455 dev_warn(rdev->dev, "(%d) create WB bo failed\n", r);
456 return r;
457 }
089920f2
JG
458 r = radeon_bo_reserve(rdev->wb.wb_obj, false);
459 if (unlikely(r != 0)) {
460 radeon_wb_fini(rdev);
461 return r;
462 }
463 r = radeon_bo_pin(rdev->wb.wb_obj, RADEON_GEM_DOMAIN_GTT,
464 &rdev->wb.gpu_addr);
465 if (r) {
466 radeon_bo_unreserve(rdev->wb.wb_obj);
467 dev_warn(rdev->dev, "(%d) pin WB bo failed\n", r);
468 radeon_wb_fini(rdev);
469 return r;
470 }
471 r = radeon_bo_kmap(rdev->wb.wb_obj, (void **)&rdev->wb.wb);
724c80e1 472 radeon_bo_unreserve(rdev->wb.wb_obj);
089920f2
JG
473 if (r) {
474 dev_warn(rdev->dev, "(%d) map WB bo failed\n", r);
475 radeon_wb_fini(rdev);
476 return r;
477 }
724c80e1
AD
478 }
479
e6ba7599
AD
480 /* clear wb memory */
481 memset((char *)rdev->wb.wb, 0, RADEON_GPU_PAGE_SIZE);
d0f8a854
AD
482 /* disable event_write fences */
483 rdev->wb.use_event = false;
724c80e1 484 /* disabled via module param */
3b7a2b24 485 if (radeon_no_wb == 1) {
724c80e1 486 rdev->wb.enabled = false;
3b7a2b24 487 } else {
724c80e1 488 if (rdev->flags & RADEON_IS_AGP) {
28eebb70
AD
489 /* often unreliable on AGP */
490 rdev->wb.enabled = false;
491 } else if (rdev->family < CHIP_R300) {
492 /* often unreliable on pre-r300 */
724c80e1 493 rdev->wb.enabled = false;
d0f8a854 494 } else {
724c80e1 495 rdev->wb.enabled = true;
d0f8a854 496 /* event_write fences are only available on r600+ */
3b7a2b24 497 if (rdev->family >= CHIP_R600) {
d0f8a854 498 rdev->wb.use_event = true;
3b7a2b24 499 }
d0f8a854 500 }
724c80e1 501 }
c994ead6
AD
502 /* always use writeback/events on NI, APUs */
503 if (rdev->family >= CHIP_PALM) {
7d52785d
AD
504 rdev->wb.enabled = true;
505 rdev->wb.use_event = true;
506 }
724c80e1
AD
507
508 dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
509
510 return 0;
511}
512
d594e46a
JG
513/**
514 * radeon_vram_location - try to find VRAM location
515 * @rdev: radeon device structure holding all necessary informations
516 * @mc: memory controller structure holding memory informations
517 * @base: base address at which to put VRAM
518 *
519 * Function will place try to place VRAM at base address provided
520 * as parameter (which is so far either PCI aperture address or
521 * for IGP TOM base address).
522 *
523 * If there is not enough space to fit the unvisible VRAM in the 32bits
524 * address space then we limit the VRAM size to the aperture.
525 *
526 * If we are using AGP and if the AGP aperture doesn't allow us to have
527 * room for all the VRAM than we restrict the VRAM to the PCI aperture
528 * size and print a warning.
529 *
530 * This function will never fails, worst case are limiting VRAM.
531 *
532 * Note: GTT start, end, size should be initialized before calling this
533 * function on AGP platform.
534 *
25985edc 535 * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
d594e46a
JG
536 * this shouldn't be a problem as we are using the PCI aperture as a reference.
537 * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
538 * not IGP.
539 *
540 * Note: we use mc_vram_size as on some board we need to program the mc to
541 * cover the whole aperture even if VRAM size is inferior to aperture size
542 * Novell bug 204882 + along with lots of ubuntu ones
543 *
544 * Note: when limiting vram it's safe to overwritte real_vram_size because
545 * we are not in case where real_vram_size is inferior to mc_vram_size (ie
546 * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
547 * ones)
548 *
549 * Note: IGP TOM addr should be the same as the aperture addr, we don't
550 * explicitly check for that thought.
551 *
552 * FIXME: when reducing VRAM size align new size on power of 2.
771fe6b9 553 */
d594e46a 554void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base)
771fe6b9 555{
1bcb04f7
CK
556 uint64_t limit = (uint64_t)radeon_vram_limit << 20;
557
d594e46a 558 mc->vram_start = base;
9ed8b1f9 559 if (mc->mc_vram_size > (rdev->mc.mc_mask - base + 1)) {
d594e46a
JG
560 dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
561 mc->real_vram_size = mc->aper_size;
562 mc->mc_vram_size = mc->aper_size;
563 }
564 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
2cbeb4ef 565 if (rdev->flags & RADEON_IS_AGP && mc->vram_end > mc->gtt_start && mc->vram_start <= mc->gtt_end) {
d594e46a
JG
566 dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n");
567 mc->real_vram_size = mc->aper_size;
568 mc->mc_vram_size = mc->aper_size;
569 }
570 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1bcb04f7
CK
571 if (limit && limit < mc->real_vram_size)
572 mc->real_vram_size = limit;
dd7cc55a 573 dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
d594e46a
JG
574 mc->mc_vram_size >> 20, mc->vram_start,
575 mc->vram_end, mc->real_vram_size >> 20);
576}
771fe6b9 577
d594e46a
JG
578/**
579 * radeon_gtt_location - try to find GTT location
580 * @rdev: radeon device structure holding all necessary informations
581 * @mc: memory controller structure holding memory informations
582 *
583 * Function will place try to place GTT before or after VRAM.
584 *
585 * If GTT size is bigger than space left then we ajust GTT size.
586 * Thus function will never fails.
587 *
588 * FIXME: when reducing GTT size align new size on power of 2.
589 */
590void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
591{
592 u64 size_af, size_bf;
593
9ed8b1f9 594 size_af = ((rdev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
8d369bb1 595 size_bf = mc->vram_start & ~mc->gtt_base_align;
d594e46a
JG
596 if (size_bf > size_af) {
597 if (mc->gtt_size > size_bf) {
598 dev_warn(rdev->dev, "limiting GTT\n");
599 mc->gtt_size = size_bf;
771fe6b9 600 }
8d369bb1 601 mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
771fe6b9 602 } else {
d594e46a
JG
603 if (mc->gtt_size > size_af) {
604 dev_warn(rdev->dev, "limiting GTT\n");
605 mc->gtt_size = size_af;
606 }
8d369bb1 607 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
771fe6b9 608 }
d594e46a 609 mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
dd7cc55a 610 dev_info(rdev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
d594e46a 611 mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
771fe6b9
JG
612}
613
771fe6b9
JG
614/*
615 * GPU helpers function.
616 */
05082b8b
AD
617
618/**
619 * radeon_device_is_virtual - check if we are running is a virtual environment
620 *
621 * Check if the asic has been passed through to a VM (all asics).
622 * Used at driver startup.
623 * Returns true if virtual or false if not.
624 */
a801abe4 625bool radeon_device_is_virtual(void)
05082b8b
AD
626{
627#ifdef CONFIG_X86
628 return boot_cpu_has(X86_FEATURE_HYPERVISOR);
629#else
630 return false;
631#endif
632}
633
0c195119
AD
634/**
635 * radeon_card_posted - check if the hw has already been initialized
636 *
637 * @rdev: radeon_device pointer
638 *
639 * Check if the asic has been initialized (all asics).
640 * Used at driver startup.
641 * Returns true if initialized or false if not.
642 */
9f022ddf 643bool radeon_card_posted(struct radeon_device *rdev)
771fe6b9
JG
644{
645 uint32_t reg;
646
884031f0
AD
647 /* for pass through, always force asic_init for CI */
648 if (rdev->family >= CHIP_BONAIRE &&
649 radeon_device_is_virtual())
05082b8b
AD
650 return false;
651
50a583f6 652 /* required for EFI mode on macbook2,1 which uses an r5xx asic */
83e68189 653 if (efi_enabled(EFI_BOOT) &&
50a583f6
AD
654 (rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) &&
655 (rdev->family < CHIP_R600))
bcc65fd8
MG
656 return false;
657
2cf3a4fc
AD
658 if (ASIC_IS_NODCE(rdev))
659 goto check_memsize;
660
771fe6b9 661 /* first check CRTCs */
09fb8bd1 662 if (ASIC_IS_DCE4(rdev)) {
18007401
AD
663 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
664 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
09fb8bd1
AD
665 if (rdev->num_crtc >= 4) {
666 reg |= RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
667 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
668 }
669 if (rdev->num_crtc >= 6) {
670 reg |= RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) |
671 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
672 }
bcc1c2a1
AD
673 if (reg & EVERGREEN_CRTC_MASTER_EN)
674 return true;
675 } else if (ASIC_IS_AVIVO(rdev)) {
771fe6b9
JG
676 reg = RREG32(AVIVO_D1CRTC_CONTROL) |
677 RREG32(AVIVO_D2CRTC_CONTROL);
678 if (reg & AVIVO_CRTC_EN) {
679 return true;
680 }
681 } else {
682 reg = RREG32(RADEON_CRTC_GEN_CNTL) |
683 RREG32(RADEON_CRTC2_GEN_CNTL);
684 if (reg & RADEON_CRTC_EN) {
685 return true;
686 }
687 }
688
2cf3a4fc 689check_memsize:
771fe6b9
JG
690 /* then check MEM_SIZE, in case the crtcs are off */
691 if (rdev->family >= CHIP_R600)
692 reg = RREG32(R600_CONFIG_MEMSIZE);
693 else
694 reg = RREG32(RADEON_CONFIG_MEMSIZE);
695
696 if (reg)
697 return true;
698
699 return false;
700
701}
702
0c195119
AD
703/**
704 * radeon_update_bandwidth_info - update display bandwidth params
705 *
706 * @rdev: radeon_device pointer
707 *
708 * Used when sclk/mclk are switched or display modes are set.
709 * params are used to calculate display watermarks (all asics)
710 */
f47299c5
AD
711void radeon_update_bandwidth_info(struct radeon_device *rdev)
712{
713 fixed20_12 a;
8807286e
AD
714 u32 sclk = rdev->pm.current_sclk;
715 u32 mclk = rdev->pm.current_mclk;
f47299c5 716
8807286e
AD
717 /* sclk/mclk in Mhz */
718 a.full = dfixed_const(100);
719 rdev->pm.sclk.full = dfixed_const(sclk);
720 rdev->pm.sclk.full = dfixed_div(rdev->pm.sclk, a);
721 rdev->pm.mclk.full = dfixed_const(mclk);
722 rdev->pm.mclk.full = dfixed_div(rdev->pm.mclk, a);
f47299c5 723
8807286e 724 if (rdev->flags & RADEON_IS_IGP) {
68adac5e 725 a.full = dfixed_const(16);
f47299c5 726 /* core_bandwidth = sclk(Mhz) * 16 */
68adac5e 727 rdev->pm.core_bandwidth.full = dfixed_div(rdev->pm.sclk, a);
f47299c5
AD
728 }
729}
730
0c195119
AD
731/**
732 * radeon_boot_test_post_card - check and possibly initialize the hw
733 *
734 * @rdev: radeon_device pointer
735 *
736 * Check if the asic is initialized and if not, attempt to initialize
737 * it (all asics).
738 * Returns true if initialized or false if not.
739 */
72542d77
DA
740bool radeon_boot_test_post_card(struct radeon_device *rdev)
741{
742 if (radeon_card_posted(rdev))
743 return true;
744
745 if (rdev->bios) {
746 DRM_INFO("GPU not posted. posting now...\n");
747 if (rdev->is_atom_bios)
748 atom_asic_init(rdev->mode_info.atom_context);
749 else
750 radeon_combios_asic_init(rdev->ddev);
751 return true;
752 } else {
753 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
754 return false;
755 }
756}
757
0c195119
AD
758/**
759 * radeon_dummy_page_init - init dummy page used by the driver
760 *
761 * @rdev: radeon_device pointer
762 *
763 * Allocate the dummy page used by the driver (all asics).
764 * This dummy page is used by the driver as a filler for gart entries
765 * when pages are taken out of the GART
766 * Returns 0 on sucess, -ENOMEM on failure.
767 */
3ce0a23d
JG
768int radeon_dummy_page_init(struct radeon_device *rdev)
769{
82568565
DA
770 if (rdev->dummy_page.page)
771 return 0;
3ce0a23d
JG
772 rdev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
773 if (rdev->dummy_page.page == NULL)
774 return -ENOMEM;
775 rdev->dummy_page.addr = pci_map_page(rdev->pdev, rdev->dummy_page.page,
776 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
a30f6fb7
BH
777 if (pci_dma_mapping_error(rdev->pdev, rdev->dummy_page.addr)) {
778 dev_err(&rdev->pdev->dev, "Failed to DMA MAP the dummy page\n");
3ce0a23d
JG
779 __free_page(rdev->dummy_page.page);
780 rdev->dummy_page.page = NULL;
781 return -ENOMEM;
782 }
cb658906
MD
783 rdev->dummy_page.entry = radeon_gart_get_page_entry(rdev->dummy_page.addr,
784 RADEON_GART_PAGE_DUMMY);
3ce0a23d
JG
785 return 0;
786}
787
0c195119
AD
788/**
789 * radeon_dummy_page_fini - free dummy page used by the driver
790 *
791 * @rdev: radeon_device pointer
792 *
793 * Frees the dummy page used by the driver (all asics).
794 */
3ce0a23d
JG
795void radeon_dummy_page_fini(struct radeon_device *rdev)
796{
797 if (rdev->dummy_page.page == NULL)
798 return;
799 pci_unmap_page(rdev->pdev, rdev->dummy_page.addr,
800 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
801 __free_page(rdev->dummy_page.page);
802 rdev->dummy_page.page = NULL;
803}
804
771fe6b9 805
771fe6b9 806/* ATOM accessor methods */
0c195119
AD
807/*
808 * ATOM is an interpreted byte code stored in tables in the vbios. The
809 * driver registers callbacks to access registers and the interpreter
810 * in the driver parses the tables and executes then to program specific
811 * actions (set display modes, asic init, etc.). See radeon_atombios.c,
812 * atombios.h, and atom.c
813 */
814
815/**
816 * cail_pll_read - read PLL register
817 *
818 * @info: atom card_info pointer
819 * @reg: PLL register offset
820 *
821 * Provides a PLL register accessor for the atom interpreter (r4xx+).
822 * Returns the value of the PLL register.
823 */
771fe6b9
JG
824static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
825{
826 struct radeon_device *rdev = info->dev->dev_private;
827 uint32_t r;
828
829 r = rdev->pll_rreg(rdev, reg);
830 return r;
831}
832
0c195119
AD
833/**
834 * cail_pll_write - write PLL register
835 *
836 * @info: atom card_info pointer
837 * @reg: PLL register offset
838 * @val: value to write to the pll register
839 *
840 * Provides a PLL register accessor for the atom interpreter (r4xx+).
841 */
771fe6b9
JG
842static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
843{
844 struct radeon_device *rdev = info->dev->dev_private;
845
846 rdev->pll_wreg(rdev, reg, val);
847}
848
0c195119
AD
849/**
850 * cail_mc_read - read MC (Memory Controller) register
851 *
852 * @info: atom card_info pointer
853 * @reg: MC register offset
854 *
855 * Provides an MC register accessor for the atom interpreter (r4xx+).
856 * Returns the value of the MC register.
857 */
771fe6b9
JG
858static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
859{
860 struct radeon_device *rdev = info->dev->dev_private;
861 uint32_t r;
862
863 r = rdev->mc_rreg(rdev, reg);
864 return r;
865}
866
0c195119
AD
867/**
868 * cail_mc_write - write MC (Memory Controller) register
869 *
870 * @info: atom card_info pointer
871 * @reg: MC register offset
872 * @val: value to write to the pll register
873 *
874 * Provides a MC register accessor for the atom interpreter (r4xx+).
875 */
771fe6b9
JG
876static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
877{
878 struct radeon_device *rdev = info->dev->dev_private;
879
880 rdev->mc_wreg(rdev, reg, val);
881}
882
0c195119
AD
883/**
884 * cail_reg_write - write MMIO register
885 *
886 * @info: atom card_info pointer
887 * @reg: MMIO register offset
888 * @val: value to write to the pll register
889 *
890 * Provides a MMIO register accessor for the atom interpreter (r4xx+).
891 */
771fe6b9
JG
892static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
893{
894 struct radeon_device *rdev = info->dev->dev_private;
895
896 WREG32(reg*4, val);
897}
898
0c195119
AD
899/**
900 * cail_reg_read - read MMIO register
901 *
902 * @info: atom card_info pointer
903 * @reg: MMIO register offset
904 *
905 * Provides an MMIO register accessor for the atom interpreter (r4xx+).
906 * Returns the value of the MMIO register.
907 */
771fe6b9
JG
908static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
909{
910 struct radeon_device *rdev = info->dev->dev_private;
911 uint32_t r;
912
913 r = RREG32(reg*4);
914 return r;
915}
916
0c195119
AD
917/**
918 * cail_ioreg_write - write IO register
919 *
920 * @info: atom card_info pointer
921 * @reg: IO register offset
922 * @val: value to write to the pll register
923 *
924 * Provides a IO register accessor for the atom interpreter (r4xx+).
925 */
351a52a2
AD
926static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
927{
928 struct radeon_device *rdev = info->dev->dev_private;
929
930 WREG32_IO(reg*4, val);
931}
932
0c195119
AD
933/**
934 * cail_ioreg_read - read IO register
935 *
936 * @info: atom card_info pointer
937 * @reg: IO register offset
938 *
939 * Provides an IO register accessor for the atom interpreter (r4xx+).
940 * Returns the value of the IO register.
941 */
351a52a2
AD
942static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
943{
944 struct radeon_device *rdev = info->dev->dev_private;
945 uint32_t r;
946
947 r = RREG32_IO(reg*4);
948 return r;
949}
950
0c195119
AD
951/**
952 * radeon_atombios_init - init the driver info and callbacks for atombios
953 *
954 * @rdev: radeon_device pointer
955 *
956 * Initializes the driver info and register access callbacks for the
957 * ATOM interpreter (r4xx+).
958 * Returns 0 on sucess, -ENOMEM on failure.
959 * Called at driver startup.
960 */
771fe6b9
JG
961int radeon_atombios_init(struct radeon_device *rdev)
962{
61c4b24b
MF
963 struct card_info *atom_card_info =
964 kzalloc(sizeof(struct card_info), GFP_KERNEL);
965
966 if (!atom_card_info)
967 return -ENOMEM;
968
969 rdev->mode_info.atom_card_info = atom_card_info;
970 atom_card_info->dev = rdev->ddev;
971 atom_card_info->reg_read = cail_reg_read;
972 atom_card_info->reg_write = cail_reg_write;
351a52a2
AD
973 /* needed for iio ops */
974 if (rdev->rio_mem) {
975 atom_card_info->ioreg_read = cail_ioreg_read;
976 atom_card_info->ioreg_write = cail_ioreg_write;
977 } else {
978 DRM_ERROR("Unable to find PCI I/O BAR; using MMIO for ATOM IIO\n");
979 atom_card_info->ioreg_read = cail_reg_read;
980 atom_card_info->ioreg_write = cail_reg_write;
981 }
61c4b24b
MF
982 atom_card_info->mc_read = cail_mc_read;
983 atom_card_info->mc_write = cail_mc_write;
984 atom_card_info->pll_read = cail_pll_read;
985 atom_card_info->pll_write = cail_pll_write;
986
987 rdev->mode_info.atom_context = atom_parse(atom_card_info, rdev->bios);
0e34d094
TG
988 if (!rdev->mode_info.atom_context) {
989 radeon_atombios_fini(rdev);
990 return -ENOMEM;
991 }
992
c31ad97f 993 mutex_init(&rdev->mode_info.atom_context->mutex);
1c949842 994 mutex_init(&rdev->mode_info.atom_context->scratch_mutex);
771fe6b9 995 radeon_atom_initialize_bios_scratch_regs(rdev->ddev);
d904ef9b 996 atom_allocate_fb_scratch(rdev->mode_info.atom_context);
771fe6b9
JG
997 return 0;
998}
999
0c195119
AD
1000/**
1001 * radeon_atombios_fini - free the driver info and callbacks for atombios
1002 *
1003 * @rdev: radeon_device pointer
1004 *
1005 * Frees the driver info and register access callbacks for the ATOM
1006 * interpreter (r4xx+).
1007 * Called at driver shutdown.
1008 */
771fe6b9
JG
1009void radeon_atombios_fini(struct radeon_device *rdev)
1010{
4a04a844
JG
1011 if (rdev->mode_info.atom_context) {
1012 kfree(rdev->mode_info.atom_context->scratch);
4a04a844 1013 }
0e34d094
TG
1014 kfree(rdev->mode_info.atom_context);
1015 rdev->mode_info.atom_context = NULL;
61c4b24b 1016 kfree(rdev->mode_info.atom_card_info);
0e34d094 1017 rdev->mode_info.atom_card_info = NULL;
771fe6b9
JG
1018}
1019
0c195119
AD
1020/* COMBIOS */
1021/*
1022 * COMBIOS is the bios format prior to ATOM. It provides
1023 * command tables similar to ATOM, but doesn't have a unified
1024 * parser. See radeon_combios.c
1025 */
1026
1027/**
1028 * radeon_combios_init - init the driver info for combios
1029 *
1030 * @rdev: radeon_device pointer
1031 *
1032 * Initializes the driver info for combios (r1xx-r3xx).
1033 * Returns 0 on sucess.
1034 * Called at driver startup.
1035 */
771fe6b9
JG
1036int radeon_combios_init(struct radeon_device *rdev)
1037{
1038 radeon_combios_initialize_bios_scratch_regs(rdev->ddev);
1039 return 0;
1040}
1041
0c195119
AD
1042/**
1043 * radeon_combios_fini - free the driver info for combios
1044 *
1045 * @rdev: radeon_device pointer
1046 *
1047 * Frees the driver info for combios (r1xx-r3xx).
1048 * Called at driver shutdown.
1049 */
771fe6b9
JG
1050void radeon_combios_fini(struct radeon_device *rdev)
1051{
1052}
1053
0c195119
AD
1054/* if we get transitioned to only one device, take VGA back */
1055/**
1056 * radeon_vga_set_decode - enable/disable vga decode
1057 *
1058 * @cookie: radeon_device pointer
1059 * @state: enable/disable vga decode
1060 *
1061 * Enable/disable vga decode (all asics).
1062 * Returns VGA resource flags.
1063 */
28d52043
DA
1064static unsigned int radeon_vga_set_decode(void *cookie, bool state)
1065{
1066 struct radeon_device *rdev = cookie;
28d52043
DA
1067 radeon_vga_set_state(rdev, state);
1068 if (state)
1069 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1070 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1071 else
1072 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1073}
c1176d6f 1074
1bcb04f7
CK
1075/**
1076 * radeon_check_pot_argument - check that argument is a power of two
1077 *
1078 * @arg: value to check
1079 *
1080 * Validates that a certain argument is a power of two (all asics).
1081 * Returns true if argument is valid.
1082 */
1083static bool radeon_check_pot_argument(int arg)
1084{
1085 return (arg & (arg - 1)) == 0;
1086}
1087
5e3c4f90
GG
1088/**
1089 * Determine a sensible default GART size according to ASIC family.
1090 *
1091 * @family ASIC family name
1092 */
1093static int radeon_gart_size_auto(enum radeon_family family)
1094{
1095 /* default to a larger gart size on newer asics */
1096 if (family >= CHIP_TAHITI)
1097 return 2048;
1098 else if (family >= CHIP_RV770)
1099 return 1024;
1100 else
1101 return 512;
1102}
1103
0c195119
AD
1104/**
1105 * radeon_check_arguments - validate module params
1106 *
1107 * @rdev: radeon_device pointer
1108 *
1109 * Validates certain module parameters and updates
1110 * the associated values used by the driver (all asics).
1111 */
1109ca09 1112static void radeon_check_arguments(struct radeon_device *rdev)
36421338
JG
1113{
1114 /* vramlimit must be a power of two */
1bcb04f7 1115 if (!radeon_check_pot_argument(radeon_vram_limit)) {
36421338
JG
1116 dev_warn(rdev->dev, "vram limit (%d) must be a power of 2\n",
1117 radeon_vram_limit);
1118 radeon_vram_limit = 0;
36421338 1119 }
1bcb04f7 1120
edcd26e8 1121 if (radeon_gart_size == -1) {
5e3c4f90 1122 radeon_gart_size = radeon_gart_size_auto(rdev->family);
edcd26e8 1123 }
36421338 1124 /* gtt size must be power of two and greater or equal to 32M */
1bcb04f7 1125 if (radeon_gart_size < 32) {
edcd26e8 1126 dev_warn(rdev->dev, "gart size (%d) too small\n",
36421338 1127 radeon_gart_size);
5e3c4f90 1128 radeon_gart_size = radeon_gart_size_auto(rdev->family);
1bcb04f7 1129 } else if (!radeon_check_pot_argument(radeon_gart_size)) {
36421338
JG
1130 dev_warn(rdev->dev, "gart size (%d) must be a power of 2\n",
1131 radeon_gart_size);
5e3c4f90 1132 radeon_gart_size = radeon_gart_size_auto(rdev->family);
36421338 1133 }
1bcb04f7
CK
1134 rdev->mc.gtt_size = (uint64_t)radeon_gart_size << 20;
1135
36421338
JG
1136 /* AGP mode can only be -1, 1, 2, 4, 8 */
1137 switch (radeon_agpmode) {
1138 case -1:
1139 case 0:
1140 case 1:
1141 case 2:
1142 case 4:
1143 case 8:
1144 break;
1145 default:
1146 dev_warn(rdev->dev, "invalid AGP mode %d (valid mode: "
1147 "-1, 0, 1, 2, 4, 8)\n", radeon_agpmode);
1148 radeon_agpmode = 0;
1149 break;
1150 }
c1c44132
CK
1151
1152 if (!radeon_check_pot_argument(radeon_vm_size)) {
1153 dev_warn(rdev->dev, "VM size (%d) must be a power of 2\n",
1154 radeon_vm_size);
20b2656d 1155 radeon_vm_size = 4;
c1c44132
CK
1156 }
1157
20b2656d 1158 if (radeon_vm_size < 1) {
13c240ef 1159 dev_warn(rdev->dev, "VM size (%d) too small, min is 1GB\n",
c1c44132 1160 radeon_vm_size);
20b2656d 1161 radeon_vm_size = 4;
c1c44132
CK
1162 }
1163
3cf8bb1a
JG
1164 /*
1165 * Max GPUVM size for Cayman, SI and CI are 40 bits.
1166 */
20b2656d
CK
1167 if (radeon_vm_size > 1024) {
1168 dev_warn(rdev->dev, "VM size (%d) too large, max is 1TB\n",
c1c44132 1169 radeon_vm_size);
20b2656d 1170 radeon_vm_size = 4;
c1c44132 1171 }
4510fb98
CK
1172
1173 /* defines number of bits in page table versus page directory,
1174 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1175 * page table and the remaining bits are in the page directory */
dfc230f9
CK
1176 if (radeon_vm_block_size == -1) {
1177
1178 /* Total bits covered by PD + PTs */
8e66e134 1179 unsigned bits = ilog2(radeon_vm_size) + 18;
dfc230f9
CK
1180
1181 /* Make sure the PD is 4K in size up to 8GB address space.
1182 Above that split equal between PD and PTs */
1183 if (radeon_vm_size <= 8)
1184 radeon_vm_block_size = bits - 9;
1185 else
1186 radeon_vm_block_size = (bits + 3) / 2;
1187
1188 } else if (radeon_vm_block_size < 9) {
20b2656d 1189 dev_warn(rdev->dev, "VM page table size (%d) too small\n",
4510fb98
CK
1190 radeon_vm_block_size);
1191 radeon_vm_block_size = 9;
1192 }
1193
1194 if (radeon_vm_block_size > 24 ||
20b2656d
CK
1195 (radeon_vm_size * 1024) < (1ull << radeon_vm_block_size)) {
1196 dev_warn(rdev->dev, "VM page table size (%d) too large\n",
4510fb98
CK
1197 radeon_vm_block_size);
1198 radeon_vm_block_size = 9;
1199 }
36421338
JG
1200}
1201
0c195119
AD
1202/**
1203 * radeon_switcheroo_set_state - set switcheroo state
1204 *
1205 * @pdev: pci dev pointer
8e5de1d8 1206 * @state: vga_switcheroo state
0c195119
AD
1207 *
1208 * Callback for the switcheroo driver. Suspends or resumes the
1209 * the asics before or after it is powered up using ACPI methods.
1210 */
6a9ee8af
DA
1211static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1212{
1213 struct drm_device *dev = pci_get_drvdata(pdev);
10ebc0bc 1214
90c4cde9 1215 if (radeon_is_px(dev) && state == VGA_SWITCHEROO_OFF)
10ebc0bc
DA
1216 return;
1217
6a9ee8af 1218 if (state == VGA_SWITCHEROO_ON) {
7ca85295 1219 pr_info("radeon: switched on\n");
6a9ee8af 1220 /* don't suspend or resume card normally */
5bcf719b 1221 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
d1f9809e 1222
10ebc0bc 1223 radeon_resume_kms(dev, true, true);
d1f9809e 1224
5bcf719b 1225 dev->switch_power_state = DRM_SWITCH_POWER_ON;
fbf81762 1226 drm_kms_helper_poll_enable(dev);
6a9ee8af 1227 } else {
7ca85295 1228 pr_info("radeon: switched off\n");
fbf81762 1229 drm_kms_helper_poll_disable(dev);
5bcf719b 1230 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
274ad65c 1231 radeon_suspend_kms(dev, true, true, false);
5bcf719b 1232 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
6a9ee8af
DA
1233 }
1234}
1235
0c195119
AD
1236/**
1237 * radeon_switcheroo_can_switch - see if switcheroo state can change
1238 *
1239 * @pdev: pci dev pointer
1240 *
1241 * Callback for the switcheroo driver. Check of the switcheroo
1242 * state can be changed.
1243 * Returns true if the state can be changed, false if not.
1244 */
6a9ee8af
DA
1245static bool radeon_switcheroo_can_switch(struct pci_dev *pdev)
1246{
1247 struct drm_device *dev = pci_get_drvdata(pdev);
6a9ee8af 1248
fc8fd40e
DV
1249 /*
1250 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1251 * locking inversion with the driver load path. And the access here is
1252 * completely racy anyway. So don't bother with locking for now.
1253 */
1254 return dev->open_count == 0;
6a9ee8af
DA
1255}
1256
26ec685f
TI
1257static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = {
1258 .set_gpu_state = radeon_switcheroo_set_state,
1259 .reprobe = NULL,
1260 .can_switch = radeon_switcheroo_can_switch,
1261};
6a9ee8af 1262
0c195119
AD
1263/**
1264 * radeon_device_init - initialize the driver
1265 *
1266 * @rdev: radeon_device pointer
1267 * @pdev: drm dev pointer
1268 * @pdev: pci dev pointer
1269 * @flags: driver flags
1270 *
1271 * Initializes the driver info and hw (all asics).
1272 * Returns 0 for success or an error on failure.
1273 * Called at driver startup.
1274 */
771fe6b9
JG
1275int radeon_device_init(struct radeon_device *rdev,
1276 struct drm_device *ddev,
1277 struct pci_dev *pdev,
1278 uint32_t flags)
1279{
351a52a2 1280 int r, i;
ad49f501 1281 int dma_bits;
10ebc0bc 1282 bool runtime = false;
771fe6b9 1283
771fe6b9 1284 rdev->shutdown = false;
9f022ddf 1285 rdev->dev = &pdev->dev;
771fe6b9
JG
1286 rdev->ddev = ddev;
1287 rdev->pdev = pdev;
1288 rdev->flags = flags;
1289 rdev->family = flags & RADEON_FAMILY_MASK;
1290 rdev->is_atom_bios = false;
1291 rdev->usec_timeout = RADEON_MAX_USEC_TIMEOUT;
edcd26e8 1292 rdev->mc.gtt_size = 512 * 1024 * 1024;
733289c2 1293 rdev->accel_working = false;
8b25ed34
AD
1294 /* set up ring ids */
1295 for (i = 0; i < RADEON_NUM_RINGS; i++) {
1296 rdev->ring[i].idx = i;
1297 }
f54d1867 1298 rdev->fence_context = dma_fence_context_alloc(RADEON_NUM_RINGS);
1b5331d9 1299
fe0d36e0
AD
1300 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1301 radeon_family_name[rdev->family], pdev->vendor, pdev->device,
1302 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1b5331d9 1303
771fe6b9
JG
1304 /* mutex initialization are all done here so we
1305 * can recall function without having locking issues */
d6999bc7 1306 mutex_init(&rdev->ring_lock);
40bacf16 1307 mutex_init(&rdev->dc_hw_i2c_mutex);
c20dc369 1308 atomic_set(&rdev->ih.lock, 0);
4c788679 1309 mutex_init(&rdev->gem.mutex);
c913e23a 1310 mutex_init(&rdev->pm.mutex);
6759a0a7 1311 mutex_init(&rdev->gpu_clock_mutex);
f61d5b46 1312 mutex_init(&rdev->srbm_mutex);
db7fce39 1313 init_rwsem(&rdev->pm.mclk_lock);
dee53e7f 1314 init_rwsem(&rdev->exclusive_lock);
73a6d3fc 1315 init_waitqueue_head(&rdev->irq.vblank_queue);
341cb9e4
CK
1316 mutex_init(&rdev->mn_lock);
1317 hash_init(rdev->mn_hash);
1b9c3dd0
AD
1318 r = radeon_gem_init(rdev);
1319 if (r)
1320 return r;
529364e0 1321
c1c44132 1322 radeon_check_arguments(rdev);
23d4f1f2 1323 /* Adjust VM size here.
c1c44132 1324 * Max GPUVM size for cayman+ is 40 bits.
23d4f1f2 1325 */
20b2656d 1326 rdev->vm_manager.max_pfn = radeon_vm_size << 18;
771fe6b9 1327
4aac0473
JG
1328 /* Set asic functions */
1329 r = radeon_asic_init(rdev);
36421338 1330 if (r)
4aac0473 1331 return r;
4aac0473 1332
f95df9ca
AD
1333 /* all of the newer IGP chips have an internal gart
1334 * However some rs4xx report as AGP, so remove that here.
1335 */
1336 if ((rdev->family >= CHIP_RS400) &&
1337 (rdev->flags & RADEON_IS_IGP)) {
1338 rdev->flags &= ~RADEON_IS_AGP;
1339 }
1340
30256a3f 1341 if (rdev->flags & RADEON_IS_AGP && radeon_agpmode == -1) {
b574f251 1342 radeon_agp_disable(rdev);
771fe6b9
JG
1343 }
1344
9ed8b1f9
AD
1345 /* Set the internal MC address mask
1346 * This is the max address of the GPU's
1347 * internal address space.
1348 */
1349 if (rdev->family >= CHIP_CAYMAN)
1350 rdev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
1351 else if (rdev->family >= CHIP_CEDAR)
1352 rdev->mc.mc_mask = 0xfffffffffULL; /* 36 bit MC */
1353 else
1354 rdev->mc.mc_mask = 0xffffffffULL; /* 32 bit MC */
1355
ad49f501
DA
1356 /* set DMA mask + need_dma32 flags.
1357 * PCIE - can handle 40-bits.
005a83f1 1358 * IGP - can handle 40-bits
ad49f501 1359 * AGP - generally dma32 is safest
005a83f1 1360 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
ad49f501
DA
1361 */
1362 rdev->need_dma32 = false;
1363 if (rdev->flags & RADEON_IS_AGP)
1364 rdev->need_dma32 = true;
005a83f1 1365 if ((rdev->flags & RADEON_IS_PCI) &&
4a2b6662 1366 (rdev->family <= CHIP_RS740))
ad49f501
DA
1367 rdev->need_dma32 = true;
1368
1369 dma_bits = rdev->need_dma32 ? 32 : 40;
1370 r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
771fe6b9 1371 if (r) {
62fff811 1372 rdev->need_dma32 = true;
c52494f6 1373 dma_bits = 32;
7ca85295 1374 pr_warn("radeon: No suitable DMA available\n");
771fe6b9 1375 }
c52494f6
KRW
1376 r = pci_set_consistent_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
1377 if (r) {
1378 pci_set_consistent_dma_mask(rdev->pdev, DMA_BIT_MASK(32));
7ca85295 1379 pr_warn("radeon: No coherent DMA available\n");
c52494f6 1380 }
771fe6b9
JG
1381
1382 /* Registers mapping */
1383 /* TODO: block userspace mapping of io register */
2c385151 1384 spin_lock_init(&rdev->mmio_idx_lock);
fe78118c 1385 spin_lock_init(&rdev->smc_idx_lock);
0a5b7b0b
AD
1386 spin_lock_init(&rdev->pll_idx_lock);
1387 spin_lock_init(&rdev->mc_idx_lock);
1388 spin_lock_init(&rdev->pcie_idx_lock);
1389 spin_lock_init(&rdev->pciep_idx_lock);
1390 spin_lock_init(&rdev->pif_idx_lock);
1391 spin_lock_init(&rdev->cg_idx_lock);
1392 spin_lock_init(&rdev->uvd_idx_lock);
1393 spin_lock_init(&rdev->rcu_idx_lock);
1394 spin_lock_init(&rdev->didt_idx_lock);
1395 spin_lock_init(&rdev->end_idx_lock);
efad86db
AD
1396 if (rdev->family >= CHIP_BONAIRE) {
1397 rdev->rmmio_base = pci_resource_start(rdev->pdev, 5);
1398 rdev->rmmio_size = pci_resource_len(rdev->pdev, 5);
1399 } else {
1400 rdev->rmmio_base = pci_resource_start(rdev->pdev, 2);
1401 rdev->rmmio_size = pci_resource_len(rdev->pdev, 2);
1402 }
771fe6b9 1403 rdev->rmmio = ioremap(rdev->rmmio_base, rdev->rmmio_size);
a33c1a82 1404 if (rdev->rmmio == NULL)
771fe6b9 1405 return -ENOMEM;
771fe6b9 1406
75efdee1
AD
1407 /* doorbell bar mapping */
1408 if (rdev->family >= CHIP_BONAIRE)
1409 radeon_doorbell_init(rdev);
1410
351a52a2
AD
1411 /* io port mapping */
1412 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1413 if (pci_resource_flags(rdev->pdev, i) & IORESOURCE_IO) {
1414 rdev->rio_mem_size = pci_resource_len(rdev->pdev, i);
1415 rdev->rio_mem = pci_iomap(rdev->pdev, i, rdev->rio_mem_size);
1416 break;
1417 }
1418 }
1419 if (rdev->rio_mem == NULL)
1420 DRM_ERROR("Unable to find PCI I/O BAR\n");
1421
4807c5a8
AD
1422 if (rdev->flags & RADEON_IS_PX)
1423 radeon_device_handle_px_quirks(rdev);
1424
28d52043 1425 /* if we have > 1 VGA cards, then disable the radeon VGA resources */
93239ea1
DA
1426 /* this will fail for cards that aren't VGA class devices, just
1427 * ignore it */
1428 vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
10ebc0bc 1429
bfaddd9f 1430 if (rdev->flags & RADEON_IS_PX)
10ebc0bc 1431 runtime = true;
7ffb0ce3
LW
1432 if (!pci_is_thunderbolt_attached(rdev->pdev))
1433 vga_switcheroo_register_client(rdev->pdev,
1434 &radeon_switcheroo_ops, runtime);
10ebc0bc
DA
1435 if (runtime)
1436 vga_switcheroo_init_domain_pm_ops(rdev->dev, &rdev->vga_pm_domain);
28d52043 1437
3ce0a23d 1438 r = radeon_init(rdev);
b574f251 1439 if (r)
2e97140d 1440 goto failed;
3ce0a23d 1441
409851f4
JG
1442 r = radeon_gem_debugfs_init(rdev);
1443 if (r) {
1444 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
9843ead0
DA
1445 }
1446
1447 r = radeon_mst_debugfs_init(rdev);
1448 if (r) {
1449 DRM_ERROR("registering mst debugfs failed (%d).\n", r);
409851f4
JG
1450 }
1451
b574f251
JG
1452 if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) {
1453 /* Acceleration not working on AGP card try again
1454 * with fallback to PCI or PCIE GART
1455 */
a2d07b74 1456 radeon_asic_reset(rdev);
b574f251
JG
1457 radeon_fini(rdev);
1458 radeon_agp_disable(rdev);
1459 r = radeon_init(rdev);
4aac0473 1460 if (r)
2e97140d 1461 goto failed;
771fe6b9 1462 }
6c7bccea 1463
13a7d299
CK
1464 r = radeon_ib_ring_tests(rdev);
1465 if (r)
1466 DRM_ERROR("ib ring test failed (%d).\n", r);
1467
6dfd1972
JG
1468 /*
1469 * Turks/Thames GPU will freeze whole laptop if DPM is not restarted
1470 * after the CP ring have chew one packet at least. Hence here we stop
1471 * and restart DPM after the radeon_ib_ring_tests().
1472 */
1473 if (rdev->pm.dpm_enabled &&
1474 (rdev->pm.pm_method == PM_METHOD_DPM) &&
1475 (rdev->family == CHIP_TURKS) &&
1476 (rdev->flags & RADEON_IS_MOBILITY)) {
1477 mutex_lock(&rdev->pm.mutex);
1478 radeon_dpm_disable(rdev);
1479 radeon_dpm_enable(rdev);
1480 mutex_unlock(&rdev->pm.mutex);
1481 }
1482
60a7e396 1483 if ((radeon_testing & 1)) {
4a1132a0
AD
1484 if (rdev->accel_working)
1485 radeon_test_moves(rdev);
1486 else
1487 DRM_INFO("radeon: acceleration disabled, skipping move tests\n");
ecc0b326 1488 }
60a7e396 1489 if ((radeon_testing & 2)) {
4a1132a0
AD
1490 if (rdev->accel_working)
1491 radeon_test_syncing(rdev);
1492 else
1493 DRM_INFO("radeon: acceleration disabled, skipping sync tests\n");
60a7e396 1494 }
771fe6b9 1495 if (radeon_benchmarking) {
4a1132a0
AD
1496 if (rdev->accel_working)
1497 radeon_benchmark(rdev, radeon_benchmarking);
1498 else
1499 DRM_INFO("radeon: acceleration disabled, skipping benchmarks\n");
771fe6b9 1500 }
6cf8a3f5 1501 return 0;
2e97140d
AD
1502
1503failed:
b8751946
LW
1504 /* balance pm_runtime_get_sync() in radeon_driver_unload_kms() */
1505 if (radeon_is_px(ddev))
1506 pm_runtime_put_noidle(ddev->dev);
2e97140d
AD
1507 if (runtime)
1508 vga_switcheroo_fini_domain_pm_ops(rdev->dev);
1509 return r;
771fe6b9
JG
1510}
1511
0c195119
AD
1512/**
1513 * radeon_device_fini - tear down the driver
1514 *
1515 * @rdev: radeon_device pointer
1516 *
1517 * Tear down the driver info (all asics).
1518 * Called at driver shutdown.
1519 */
771fe6b9
JG
1520void radeon_device_fini(struct radeon_device *rdev)
1521{
771fe6b9
JG
1522 DRM_INFO("radeon: finishing device.\n");
1523 rdev->shutdown = true;
90aca4d2
JG
1524 /* evict vram memory */
1525 radeon_bo_evict_vram(rdev);
62a8ea3f 1526 radeon_fini(rdev);
7ffb0ce3
LW
1527 if (!pci_is_thunderbolt_attached(rdev->pdev))
1528 vga_switcheroo_unregister_client(rdev->pdev);
2e97140d
AD
1529 if (rdev->flags & RADEON_IS_PX)
1530 vga_switcheroo_fini_domain_pm_ops(rdev->dev);
c1176d6f 1531 vga_client_register(rdev->pdev, NULL, NULL, NULL);
e0a2ca73
AD
1532 if (rdev->rio_mem)
1533 pci_iounmap(rdev->pdev, rdev->rio_mem);
351a52a2 1534 rdev->rio_mem = NULL;
771fe6b9
JG
1535 iounmap(rdev->rmmio);
1536 rdev->rmmio = NULL;
75efdee1
AD
1537 if (rdev->family >= CHIP_BONAIRE)
1538 radeon_doorbell_fini(rdev);
771fe6b9
JG
1539}
1540
1541
1542/*
1543 * Suspend & resume.
1544 */
0c195119
AD
1545/**
1546 * radeon_suspend_kms - initiate device suspend
1547 *
1548 * @pdev: drm dev pointer
1549 * @state: suspend state
1550 *
1551 * Puts the hw in the suspend state (all asics).
1552 * Returns 0 for success or an error on failure.
1553 * Called at driver suspend.
1554 */
274ad65c
JG
1555int radeon_suspend_kms(struct drm_device *dev, bool suspend,
1556 bool fbcon, bool freeze)
771fe6b9 1557{
875c1866 1558 struct radeon_device *rdev;
771fe6b9 1559 struct drm_crtc *crtc;
d8dcaa1d 1560 struct drm_connector *connector;
7465280c 1561 int i, r;
771fe6b9 1562
875c1866 1563 if (dev == NULL || dev->dev_private == NULL) {
771fe6b9
JG
1564 return -ENODEV;
1565 }
7473e830 1566
875c1866
DJ
1567 rdev = dev->dev_private;
1568
f2aba352 1569 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
6a9ee8af 1570 return 0;
d8dcaa1d 1571
86698c20
SF
1572 drm_kms_helper_poll_disable(dev);
1573
6adaed5b 1574 drm_modeset_lock_all(dev);
d8dcaa1d
AD
1575 /* turn off display hw */
1576 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1577 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
1578 }
6adaed5b 1579 drm_modeset_unlock_all(dev);
d8dcaa1d 1580
f3cbb17b 1581 /* unpin the front buffers and cursors */
771fe6b9 1582 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
f3cbb17b 1583 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
f4510a27 1584 struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->primary->fb);
4c788679 1585 struct radeon_bo *robj;
771fe6b9 1586
f3cbb17b
GG
1587 if (radeon_crtc->cursor_bo) {
1588 struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo);
1589 r = radeon_bo_reserve(robj, false);
1590 if (r == 0) {
1591 radeon_bo_unpin(robj);
1592 radeon_bo_unreserve(robj);
1593 }
1594 }
1595
771fe6b9
JG
1596 if (rfb == NULL || rfb->obj == NULL) {
1597 continue;
1598 }
7e4d15d9 1599 robj = gem_to_radeon_bo(rfb->obj);
38651674
DA
1600 /* don't unpin kernel fb objects */
1601 if (!radeon_fbdev_robj_is_fb(rdev, robj)) {
4c788679 1602 r = radeon_bo_reserve(robj, false);
38651674 1603 if (r == 0) {
4c788679
JG
1604 radeon_bo_unpin(robj);
1605 radeon_bo_unreserve(robj);
1606 }
771fe6b9
JG
1607 }
1608 }
1609 /* evict vram memory */
4c788679 1610 radeon_bo_evict_vram(rdev);
8a47cc9e 1611
771fe6b9 1612 /* wait for gpu to finish processing current batch */
5f8f635e 1613 for (i = 0; i < RADEON_NUM_RINGS; i++) {
37615527 1614 r = radeon_fence_wait_empty(rdev, i);
5f8f635e
JG
1615 if (r) {
1616 /* delay GPU reset to resume */
eb98c709 1617 radeon_fence_driver_force_completion(rdev, i);
5f8f635e
JG
1618 }
1619 }
771fe6b9 1620
f657c2a7
YZ
1621 radeon_save_bios_scratch_regs(rdev);
1622
62a8ea3f 1623 radeon_suspend(rdev);
d4877cf2 1624 radeon_hpd_fini(rdev);
ec9aaaff
AD
1625 /* evict remaining vram memory
1626 * This second call to evict vram is to evict the gart page table
1627 * using the CPU.
1628 */
4c788679 1629 radeon_bo_evict_vram(rdev);
771fe6b9 1630
10b06122
JG
1631 radeon_agp_suspend(rdev);
1632
771fe6b9 1633 pci_save_state(dev->pdev);
82060854 1634 if (freeze && rdev->family >= CHIP_CEDAR && !(rdev->flags & RADEON_IS_IGP)) {
274ad65c
JG
1635 rdev->asic->asic_reset(rdev, true);
1636 pci_restore_state(dev->pdev);
1637 } else if (suspend) {
771fe6b9
JG
1638 /* Shut down the device */
1639 pci_disable_device(dev->pdev);
1640 pci_set_power_state(dev->pdev, PCI_D3hot);
1641 }
10ebc0bc
DA
1642
1643 if (fbcon) {
1644 console_lock();
1645 radeon_fbdev_set_suspend(rdev, 1);
1646 console_unlock();
1647 }
771fe6b9
JG
1648 return 0;
1649}
1650
0c195119
AD
1651/**
1652 * radeon_resume_kms - initiate device resume
1653 *
1654 * @pdev: drm dev pointer
1655 *
1656 * Bring the hw back to operating state (all asics).
1657 * Returns 0 for success or an error on failure.
1658 * Called at driver resume.
1659 */
10ebc0bc 1660int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
771fe6b9 1661{
09bdf591 1662 struct drm_connector *connector;
771fe6b9 1663 struct radeon_device *rdev = dev->dev_private;
f3cbb17b 1664 struct drm_crtc *crtc;
04eb2206 1665 int r;
771fe6b9 1666
f2aba352 1667 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
6a9ee8af
DA
1668 return 0;
1669
10ebc0bc
DA
1670 if (fbcon) {
1671 console_lock();
1672 }
7473e830
DA
1673 if (resume) {
1674 pci_set_power_state(dev->pdev, PCI_D0);
1675 pci_restore_state(dev->pdev);
1676 if (pci_enable_device(dev->pdev)) {
10ebc0bc
DA
1677 if (fbcon)
1678 console_unlock();
7473e830
DA
1679 return -1;
1680 }
771fe6b9 1681 }
0ebf1717
DA
1682 /* resume AGP if in use */
1683 radeon_agp_resume(rdev);
62a8ea3f 1684 radeon_resume(rdev);
04eb2206
CK
1685
1686 r = radeon_ib_ring_tests(rdev);
1687 if (r)
1688 DRM_ERROR("ib ring test failed (%d).\n", r);
1689
bc6a6295 1690 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
6c7bccea
AD
1691 /* do dpm late init */
1692 r = radeon_pm_late_init(rdev);
1693 if (r) {
1694 rdev->pm.dpm_enabled = false;
1695 DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n");
1696 }
bc6a6295
AD
1697 } else {
1698 /* resume old pm late */
1699 radeon_pm_resume(rdev);
6c7bccea
AD
1700 }
1701
f657c2a7 1702 radeon_restore_bios_scratch_regs(rdev);
09bdf591 1703
f3cbb17b
GG
1704 /* pin cursors */
1705 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1706 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
1707
1708 if (radeon_crtc->cursor_bo) {
1709 struct radeon_bo *robj = gem_to_radeon_bo(radeon_crtc->cursor_bo);
1710 r = radeon_bo_reserve(robj, false);
1711 if (r == 0) {
1712 /* Only 27 bit offset for legacy cursor */
1713 r = radeon_bo_pin_restricted(robj,
1714 RADEON_GEM_DOMAIN_VRAM,
1715 ASIC_IS_AVIVO(rdev) ?
1716 0 : 1 << 27,
1717 &radeon_crtc->cursor_addr);
1718 if (r != 0)
1719 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
1720 radeon_bo_unreserve(robj);
1721 }
1722 }
1723 }
1724
3fa47d9e
AD
1725 /* init dig PHYs, disp eng pll */
1726 if (rdev->is_atom_bios) {
ac89af1e 1727 radeon_atom_encoder_init(rdev);
f3f1f03e 1728 radeon_atom_disp_eng_pll_init(rdev);
bced76f2
AD
1729 /* turn on the BL */
1730 if (rdev->mode_info.bl_encoder) {
1731 u8 bl_level = radeon_get_backlight_level(rdev,
1732 rdev->mode_info.bl_encoder);
1733 radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder,
1734 bl_level);
1735 }
3fa47d9e 1736 }
d4877cf2
AD
1737 /* reset hpd state */
1738 radeon_hpd_init(rdev);
771fe6b9 1739 /* blat the mode back in */
ec9954fc
DA
1740 if (fbcon) {
1741 drm_helper_resume_force_mode(dev);
1742 /* turn on display hw */
6adaed5b 1743 drm_modeset_lock_all(dev);
ec9954fc
DA
1744 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1745 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
1746 }
6adaed5b 1747 drm_modeset_unlock_all(dev);
a93f344d 1748 }
86698c20
SF
1749
1750 drm_kms_helper_poll_enable(dev);
18ee37a4 1751
3640da2f
AD
1752 /* set the power state here in case we are a PX system or headless */
1753 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
1754 radeon_pm_compute_clocks(rdev);
1755
18ee37a4
DV
1756 if (fbcon) {
1757 radeon_fbdev_set_suspend(rdev, 0);
1758 console_unlock();
1759 }
1760
771fe6b9
JG
1761 return 0;
1762}
1763
0c195119
AD
1764/**
1765 * radeon_gpu_reset - reset the asic
1766 *
1767 * @rdev: radeon device pointer
1768 *
1769 * Attempt the reset the GPU if it has hung (all asics).
1770 * Returns 0 for success or an error on failure.
1771 */
90aca4d2
JG
1772int radeon_gpu_reset(struct radeon_device *rdev)
1773{
55d7c221
CK
1774 unsigned ring_sizes[RADEON_NUM_RINGS];
1775 uint32_t *ring_data[RADEON_NUM_RINGS];
1776
1777 bool saved = false;
1778
1779 int i, r;
8fd1b84c 1780 int resched;
90aca4d2 1781
dee53e7f 1782 down_write(&rdev->exclusive_lock);
f9eaf9ae
CK
1783
1784 if (!rdev->needs_reset) {
1785 up_write(&rdev->exclusive_lock);
1786 return 0;
1787 }
1788
72b9076b
MO
1789 atomic_inc(&rdev->gpu_reset_counter);
1790
90aca4d2 1791 radeon_save_bios_scratch_regs(rdev);
8fd1b84c
DA
1792 /* block TTM */
1793 resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev);
90aca4d2 1794 radeon_suspend(rdev);
73ef0e0d 1795 radeon_hpd_fini(rdev);
90aca4d2 1796
55d7c221
CK
1797 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
1798 ring_sizes[i] = radeon_ring_backup(rdev, &rdev->ring[i],
1799 &ring_data[i]);
1800 if (ring_sizes[i]) {
1801 saved = true;
1802 dev_info(rdev->dev, "Saved %d dwords of commands "
1803 "on ring %d.\n", ring_sizes[i], i);
1804 }
1805 }
1806
90aca4d2
JG
1807 r = radeon_asic_reset(rdev);
1808 if (!r) {
55d7c221 1809 dev_info(rdev->dev, "GPU reset succeeded, trying to resume\n");
90aca4d2 1810 radeon_resume(rdev);
55d7c221 1811 }
04eb2206 1812
55d7c221 1813 radeon_restore_bios_scratch_regs(rdev);
04eb2206 1814
9bb39ff4
ML
1815 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
1816 if (!r && ring_data[i]) {
55d7c221
CK
1817 radeon_ring_restore(rdev, &rdev->ring[i],
1818 ring_sizes[i], ring_data[i]);
9bb39ff4 1819 } else {
eb98c709 1820 radeon_fence_driver_force_completion(rdev, i);
55d7c221
CK
1821 kfree(ring_data[i]);
1822 }
90aca4d2 1823 }
7a1619b9 1824
c940b447
AD
1825 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1826 /* do dpm late init */
1827 r = radeon_pm_late_init(rdev);
1828 if (r) {
1829 rdev->pm.dpm_enabled = false;
1830 DRM_ERROR("radeon_pm_late_init failed, disabling dpm\n");
1831 }
1832 } else {
1833 /* resume old pm late */
1834 radeon_pm_resume(rdev);
1835 }
1836
73ef0e0d
AD
1837 /* init dig PHYs, disp eng pll */
1838 if (rdev->is_atom_bios) {
1839 radeon_atom_encoder_init(rdev);
1840 radeon_atom_disp_eng_pll_init(rdev);
1841 /* turn on the BL */
1842 if (rdev->mode_info.bl_encoder) {
1843 u8 bl_level = radeon_get_backlight_level(rdev,
1844 rdev->mode_info.bl_encoder);
1845 radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder,
1846 bl_level);
1847 }
1848 }
1849 /* reset hpd state */
1850 radeon_hpd_init(rdev);
1851
9bb39ff4 1852 ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);
3c036389
CK
1853
1854 rdev->in_reset = true;
1855 rdev->needs_reset = false;
1856
9bb39ff4
ML
1857 downgrade_write(&rdev->exclusive_lock);
1858
d3493574
JG
1859 drm_helper_resume_force_mode(rdev->ddev);
1860
c940b447
AD
1861 /* set the power state here in case we are a PX system or headless */
1862 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled)
1863 radeon_pm_compute_clocks(rdev);
1864
9bb39ff4
ML
1865 if (!r) {
1866 r = radeon_ib_ring_tests(rdev);
1867 if (r && saved)
1868 r = -EAGAIN;
1869 } else {
7a1619b9
MD
1870 /* bad news, how to tell it to userspace ? */
1871 dev_info(rdev->dev, "GPU reset failed\n");
1872 }
1873
9bb39ff4
ML
1874 rdev->needs_reset = r == -EAGAIN;
1875 rdev->in_reset = false;
1876
1877 up_read(&rdev->exclusive_lock);
90aca4d2
JG
1878 return r;
1879}
1880
771fe6b9
JG
1881
1882/*
1883 * Debugfs
1884 */
771fe6b9
JG
1885int radeon_debugfs_add_files(struct radeon_device *rdev,
1886 struct drm_info_list *files,
1887 unsigned nfiles)
1888{
1889 unsigned i;
1890
4d8bf9ae
CK
1891 for (i = 0; i < rdev->debugfs_count; i++) {
1892 if (rdev->debugfs[i].files == files) {
771fe6b9
JG
1893 /* Already registered */
1894 return 0;
1895 }
1896 }
c245cb9e 1897
4d8bf9ae 1898 i = rdev->debugfs_count + 1;
c245cb9e
MW
1899 if (i > RADEON_DEBUGFS_MAX_COMPONENTS) {
1900 DRM_ERROR("Reached maximum number of debugfs components.\n");
1901 DRM_ERROR("Report so we increase "
3cf8bb1a 1902 "RADEON_DEBUGFS_MAX_COMPONENTS.\n");
771fe6b9
JG
1903 return -EINVAL;
1904 }
4d8bf9ae
CK
1905 rdev->debugfs[rdev->debugfs_count].files = files;
1906 rdev->debugfs[rdev->debugfs_count].num_files = nfiles;
1907 rdev->debugfs_count = i;
771fe6b9 1908#if defined(CONFIG_DEBUG_FS)
771fe6b9
JG
1909 drm_debugfs_create_files(files, nfiles,
1910 rdev->ddev->primary->debugfs_root,
1911 rdev->ddev->primary);
1912#endif
1913 return 0;
1914}