drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "nid.h"
  32 #include "atom.h"
  33 #include "ni_reg.h"
  34 #include "cayman_blit_shaders.h"
  35 #include "radeon_ucode.h"
  36 #include "clearstate_cayman.h"
  37
  38 static const u32 tn_rlc_save_restore_register_list[] =
  39 {
  40         0x98fc,
  41         0x98f0,
  42         0x9834,
  43         0x9838,
  44         0x9870,
  45         0x9874,
  46         0x8a14,
  47         0x8b24,
  48         0x8bcc,
  49         0x8b10,
  50         0x8c30,
  51         0x8d00,
  52         0x8d04,
  53         0x8c00,
  54         0x8c04,
  55         0x8c10,
  56         0x8c14,
  57         0x8d8c,
  58         0x8cf0,
  59         0x8e38,
  60         0x9508,
  61         0x9688,
  62         0x9608,
  63         0x960c,
  64         0x9610,
  65         0x9614,
  66         0x88c4,
  67         0x8978,
  68         0x88d4,
  69         0x900c,
  70         0x9100,
  71         0x913c,
  72         0x90e8,
  73         0x9354,
  74         0xa008,
  75         0x98f8,
  76         0x9148,
  77         0x914c,
  78         0x3f94,
  79         0x98f4,
  80         0x9b7c,
  81         0x3f8c,
  82         0x8950,
  83         0x8954,
  84         0x8a18,
  85         0x8b28,
  86         0x9144,
  87         0x3f90,
  88         0x915c,
  89         0x9160,
  90         0x9178,
  91         0x917c,
  92         0x9180,
  93         0x918c,
  94         0x9190,
  95         0x9194,
  96         0x9198,
  97         0x919c,
  98         0x91a8,
  99         0x91ac,
 100         0x91b0,
 101         0x91b4,
 102         0x91b8,
 103         0x91c4,
 104         0x91c8,
 105         0x91cc,
 106         0x91d0,
 107         0x91d4,
 108         0x91e0,
 109         0x91e4,
 110         0x91ec,
 111         0x91f0,
 112         0x91f4,
 113         0x9200,
 114         0x9204,
 115         0x929c,
 116         0x8030,
 117         0x9150,
 118         0x9a60,
 119         0x920c,
 120         0x9210,
 121         0x9228,
 122         0x922c,
 123         0x9244,
 124         0x9248,
 125         0x91e8,
 126         0x9294,
 127         0x9208,
 128         0x9224,
 129         0x9240,
 130         0x9220,
 131         0x923c,
 132         0x9258,
 133         0x9744,
 134         0xa200,
 135         0xa204,
 136         0xa208,
 137         0xa20c,
 138         0x8d58,
 139         0x9030,
 140         0x9034,
 141         0x9038,
 142         0x903c,
 143         0x9040,
 144         0x9654,
 145         0x897c,
 146         0xa210,
 147         0xa214,
 148         0x9868,
 149         0xa02c,
 150         0x9664,
 151         0x9698,
 152         0x949c,
 153         0x8e10,
 154         0x8e18,
 155         0x8c50,
 156         0x8c58,
 157         0x8c60,
 158         0x8c68,
 159         0x89b4,
 160         0x9830,
 161         0x802c,
 162 };
 163
 164 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 165 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 166 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 167 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 169 extern void evergreen_mc_program(struct radeon_device *rdev);
 170 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 171 extern int evergreen_mc_init(struct radeon_device *rdev);
 172 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 173 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 174 extern void evergreen_program_aspm(struct radeon_device *rdev);
 175 extern void sumo_rlc_fini(struct radeon_device *rdev);
 176 extern int sumo_rlc_init(struct radeon_device *rdev);
 177 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 178
 179 /* Firmware Names */
 180 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 181 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 183 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 184 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 185 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 188 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 189 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 192 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 193 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 197 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 198 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 200 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 201
 202
 203 static const u32 cayman_golden_registers2[] =
 204 {
 205         0x3e5c, 0xffffffff, 0x00000000,
 206         0x3e48, 0xffffffff, 0x00000000,
 207         0x3e4c, 0xffffffff, 0x00000000,
 208         0x3e64, 0xffffffff, 0x00000000,
 209         0x3e50, 0xffffffff, 0x00000000,
 210         0x3e60, 0xffffffff, 0x00000000
 211 };
 212
 213 static const u32 cayman_golden_registers[] =
 214 {
 215         0x5eb4, 0xffffffff, 0x00000002,
 216         0x5e78, 0x8f311ff1, 0x001000f0,
 217         0x3f90, 0xffff0000, 0xff000000,
 218         0x9148, 0xffff0000, 0xff000000,
 219         0x3f94, 0xffff0000, 0xff000000,
 220         0x914c, 0xffff0000, 0xff000000,
 221         0xc78, 0x00000080, 0x00000080,
 222         0xbd4, 0x70073777, 0x00011003,
 223         0xd02c, 0xbfffff1f, 0x08421000,
 224         0xd0b8, 0x73773777, 0x02011003,
 225         0x5bc0, 0x00200000, 0x50100000,
 226         0x98f8, 0x33773777, 0x02011003,
 227         0x98fc, 0xffffffff, 0x76541032,
 228         0x7030, 0x31000311, 0x00000011,
 229         0x2f48, 0x33773777, 0x42010001,
 230         0x6b28, 0x00000010, 0x00000012,
 231         0x7728, 0x00000010, 0x00000012,
 232         0x10328, 0x00000010, 0x00000012,
 233         0x10f28, 0x00000010, 0x00000012,
 234         0x11b28, 0x00000010, 0x00000012,
 235         0x12728, 0x00000010, 0x00000012,
 236         0x240c, 0x000007ff, 0x00000000,
 237         0x8a14, 0xf000001f, 0x00000007,
 238         0x8b24, 0x3fff3fff, 0x00ff0fff,
 239         0x8b10, 0x0000ff0f, 0x00000000,
 240         0x28a4c, 0x07ffffff, 0x06000000,
 241         0x10c, 0x00000001, 0x00010003,
 242         0xa02c, 0xffffffff, 0x0000009b,
 243         0x913c, 0x0000010f, 0x01000100,
 244         0x8c04, 0xf8ff00ff, 0x40600060,
 245         0x28350, 0x00000f01, 0x00000000,
 246         0x9508, 0x3700001f, 0x00000002,
 247         0x960c, 0xffffffff, 0x54763210,
 248         0x88c4, 0x001f3ae3, 0x00000082,
 249         0x88d0, 0xffffffff, 0x0f40df40,
 250         0x88d4, 0x0000001f, 0x00000010,
 251         0x8974, 0xffffffff, 0x00000000
 252 };
 253
 254 static const u32 dvst_golden_registers2[] =
 255 {
 256         0x8f8, 0xffffffff, 0,
 257         0x8fc, 0x00380000, 0,
 258         0x8f8, 0xffffffff, 1,
 259         0x8fc, 0x0e000000, 0
 260 };
 261
 262 static const u32 dvst_golden_registers[] =
 263 {
 264         0x690, 0x3fff3fff, 0x20c00033,
 265         0x918c, 0x0fff0fff, 0x00010006,
 266         0x91a8, 0x0fff0fff, 0x00010006,
 267         0x9150, 0xffffdfff, 0x6e944040,
 268         0x917c, 0x0fff0fff, 0x00030002,
 269         0x9198, 0x0fff0fff, 0x00030002,
 270         0x915c, 0x0fff0fff, 0x00010000,
 271         0x3f90, 0xffff0001, 0xff000000,
 272         0x9178, 0x0fff0fff, 0x00070000,
 273         0x9194, 0x0fff0fff, 0x00070000,
 274         0x9148, 0xffff0001, 0xff000000,
 275         0x9190, 0x0fff0fff, 0x00090008,
 276         0x91ac, 0x0fff0fff, 0x00090008,
 277         0x3f94, 0xffff0000, 0xff000000,
 278         0x914c, 0xffff0000, 0xff000000,
 279         0x929c, 0x00000fff, 0x00000001,
 280         0x55e4, 0xff607fff, 0xfc000100,
 281         0x8a18, 0xff000fff, 0x00000100,
 282         0x8b28, 0xff000fff, 0x00000100,
 283         0x9144, 0xfffc0fff, 0x00000100,
 284         0x6ed8, 0x00010101, 0x00010000,
 285         0x9830, 0xffffffff, 0x00000000,
 286         0x9834, 0xf00fffff, 0x00000400,
 287         0x9838, 0xfffffffe, 0x00000000,
 288         0xd0c0, 0xff000fff, 0x00000100,
 289         0xd02c, 0xbfffff1f, 0x08421000,
 290         0xd0b8, 0x73773777, 0x12010001,
 291         0x5bb0, 0x000000f0, 0x00000070,
 292         0x98f8, 0x73773777, 0x12010001,
 293         0x98fc, 0xffffffff, 0x00000010,
 294         0x9b7c, 0x00ff0000, 0x00fc0000,
 295         0x8030, 0x00001f0f, 0x0000100a,
 296         0x2f48, 0x73773777, 0x12010001,
 297         0x2408, 0x00030000, 0x000c007f,
 298         0x8a14, 0xf000003f, 0x00000007,
 299         0x8b24, 0x3fff3fff, 0x00ff0fff,
 300         0x8b10, 0x0000ff0f, 0x00000000,
 301         0x28a4c, 0x07ffffff, 0x06000000,
 302         0x4d8, 0x00000fff, 0x00000100,
 303         0xa008, 0xffffffff, 0x00010000,
 304         0x913c, 0xffff03ff, 0x01000100,
 305         0x8c00, 0x000000ff, 0x00000003,
 306         0x8c04, 0xf8ff00ff, 0x40600060,
 307         0x8cf0, 0x1fff1fff, 0x08e00410,
 308         0x28350, 0x00000f01, 0x00000000,
 309         0x9508, 0xf700071f, 0x00000002,
 310         0x960c, 0xffffffff, 0x54763210,
 311         0x20ef8, 0x01ff01ff, 0x00000002,
 312         0x20e98, 0xfffffbff, 0x00200000,
 313         0x2015c, 0xffffffff, 0x00000f40,
 314         0x88c4, 0x001f3ae3, 0x00000082,
 315         0x8978, 0x3fffffff, 0x04050140,
 316         0x88d4, 0x0000001f, 0x00000010,
 317         0x8974, 0xffffffff, 0x00000000
 318 };
 319
 320 static const u32 scrapper_golden_registers[] =
 321 {
 322         0x690, 0x3fff3fff, 0x20c00033,
 323         0x918c, 0x0fff0fff, 0x00010006,
 324         0x918c, 0x0fff0fff, 0x00010006,
 325         0x91a8, 0x0fff0fff, 0x00010006,
 326         0x91a8, 0x0fff0fff, 0x00010006,
 327         0x9150, 0xffffdfff, 0x6e944040,
 328         0x9150, 0xffffdfff, 0x6e944040,
 329         0x917c, 0x0fff0fff, 0x00030002,
 330         0x917c, 0x0fff0fff, 0x00030002,
 331         0x9198, 0x0fff0fff, 0x00030002,
 332         0x9198, 0x0fff0fff, 0x00030002,
 333         0x915c, 0x0fff0fff, 0x00010000,
 334         0x915c, 0x0fff0fff, 0x00010000,
 335         0x3f90, 0xffff0001, 0xff000000,
 336         0x3f90, 0xffff0001, 0xff000000,
 337         0x9178, 0x0fff0fff, 0x00070000,
 338         0x9178, 0x0fff0fff, 0x00070000,
 339         0x9194, 0x0fff0fff, 0x00070000,
 340         0x9194, 0x0fff0fff, 0x00070000,
 341         0x9148, 0xffff0001, 0xff000000,
 342         0x9148, 0xffff0001, 0xff000000,
 343         0x9190, 0x0fff0fff, 0x00090008,
 344         0x9190, 0x0fff0fff, 0x00090008,
 345         0x91ac, 0x0fff0fff, 0x00090008,
 346         0x91ac, 0x0fff0fff, 0x00090008,
 347         0x3f94, 0xffff0000, 0xff000000,
 348         0x3f94, 0xffff0000, 0xff000000,
 349         0x914c, 0xffff0000, 0xff000000,
 350         0x914c, 0xffff0000, 0xff000000,
 351         0x929c, 0x00000fff, 0x00000001,
 352         0x929c, 0x00000fff, 0x00000001,
 353         0x55e4, 0xff607fff, 0xfc000100,
 354         0x8a18, 0xff000fff, 0x00000100,
 355         0x8a18, 0xff000fff, 0x00000100,
 356         0x8b28, 0xff000fff, 0x00000100,
 357         0x8b28, 0xff000fff, 0x00000100,
 358         0x9144, 0xfffc0fff, 0x00000100,
 359         0x9144, 0xfffc0fff, 0x00000100,
 360         0x6ed8, 0x00010101, 0x00010000,
 361         0x9830, 0xffffffff, 0x00000000,
 362         0x9830, 0xffffffff, 0x00000000,
 363         0x9834, 0xf00fffff, 0x00000400,
 364         0x9834, 0xf00fffff, 0x00000400,
 365         0x9838, 0xfffffffe, 0x00000000,
 366         0x9838, 0xfffffffe, 0x00000000,
 367         0xd0c0, 0xff000fff, 0x00000100,
 368         0xd02c, 0xbfffff1f, 0x08421000,
 369         0xd02c, 0xbfffff1f, 0x08421000,
 370         0xd0b8, 0x73773777, 0x12010001,
 371         0xd0b8, 0x73773777, 0x12010001,
 372         0x5bb0, 0x000000f0, 0x00000070,
 373         0x98f8, 0x73773777, 0x12010001,
 374         0x98f8, 0x73773777, 0x12010001,
 375         0x98fc, 0xffffffff, 0x00000010,
 376         0x98fc, 0xffffffff, 0x00000010,
 377         0x9b7c, 0x00ff0000, 0x00fc0000,
 378         0x9b7c, 0x00ff0000, 0x00fc0000,
 379         0x8030, 0x00001f0f, 0x0000100a,
 380         0x8030, 0x00001f0f, 0x0000100a,
 381         0x2f48, 0x73773777, 0x12010001,
 382         0x2f48, 0x73773777, 0x12010001,
 383         0x2408, 0x00030000, 0x000c007f,
 384         0x8a14, 0xf000003f, 0x00000007,
 385         0x8a14, 0xf000003f, 0x00000007,
 386         0x8b24, 0x3fff3fff, 0x00ff0fff,
 387         0x8b24, 0x3fff3fff, 0x00ff0fff,
 388         0x8b10, 0x0000ff0f, 0x00000000,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x28a4c, 0x07ffffff, 0x06000000,
 391         0x28a4c, 0x07ffffff, 0x06000000,
 392         0x4d8, 0x00000fff, 0x00000100,
 393         0x4d8, 0x00000fff, 0x00000100,
 394         0xa008, 0xffffffff, 0x00010000,
 395         0xa008, 0xffffffff, 0x00010000,
 396         0x913c, 0xffff03ff, 0x01000100,
 397         0x913c, 0xffff03ff, 0x01000100,
 398         0x90e8, 0x001fffff, 0x010400c0,
 399         0x8c00, 0x000000ff, 0x00000003,
 400         0x8c00, 0x000000ff, 0x00000003,
 401         0x8c04, 0xf8ff00ff, 0x40600060,
 402         0x8c04, 0xf8ff00ff, 0x40600060,
 403         0x8c30, 0x0000000f, 0x00040005,
 404         0x8cf0, 0x1fff1fff, 0x08e00410,
 405         0x8cf0, 0x1fff1fff, 0x08e00410,
 406         0x900c, 0x00ffffff, 0x0017071f,
 407         0x28350, 0x00000f01, 0x00000000,
 408         0x28350, 0x00000f01, 0x00000000,
 409         0x9508, 0xf700071f, 0x00000002,
 410         0x9508, 0xf700071f, 0x00000002,
 411         0x9688, 0x00300000, 0x0017000f,
 412         0x960c, 0xffffffff, 0x54763210,
 413         0x960c, 0xffffffff, 0x54763210,
 414         0x20ef8, 0x01ff01ff, 0x00000002,
 415         0x20e98, 0xfffffbff, 0x00200000,
 416         0x2015c, 0xffffffff, 0x00000f40,
 417         0x88c4, 0x001f3ae3, 0x00000082,
 418         0x88c4, 0x001f3ae3, 0x00000082,
 419         0x8978, 0x3fffffff, 0x04050140,
 420         0x8978, 0x3fffffff, 0x04050140,
 421         0x88d4, 0x0000001f, 0x00000010,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x8974, 0xffffffff, 0x00000000,
 424         0x8974, 0xffffffff, 0x00000000
 425 };
 426
 427 static void ni_init_golden_registers(struct radeon_device *rdev)
 428 {
 429         switch (rdev->family) {
 430         case CHIP_CAYMAN:
 431                 radeon_program_register_sequence(rdev,
 432                                                  cayman_golden_registers,
 433                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 434                 radeon_program_register_sequence(rdev,
 435                                                  cayman_golden_registers2,
 436                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 437                 break;
 438         case CHIP_ARUBA:
 439                 if ((rdev->pdev->device == 0x9900) ||
 440                     (rdev->pdev->device == 0x9901) ||
 441                     (rdev->pdev->device == 0x9903) ||
 442                     (rdev->pdev->device == 0x9904) ||
 443                     (rdev->pdev->device == 0x9905) ||
 444                     (rdev->pdev->device == 0x9906) ||
 445                     (rdev->pdev->device == 0x9907) ||
 446                     (rdev->pdev->device == 0x9908) ||
 447                     (rdev->pdev->device == 0x9909) ||
 448                     (rdev->pdev->device == 0x990A) ||
 449                     (rdev->pdev->device == 0x990B) ||
 450                     (rdev->pdev->device == 0x990C) ||
 451                     (rdev->pdev->device == 0x990D) ||
 452                     (rdev->pdev->device == 0x990E) ||
 453                     (rdev->pdev->device == 0x990F) ||
 454                     (rdev->pdev->device == 0x9910) ||
 455                     (rdev->pdev->device == 0x9913) ||
 456                     (rdev->pdev->device == 0x9917) ||
 457                     (rdev->pdev->device == 0x9918)) {
 458                         radeon_program_register_sequence(rdev,
 459                                                          dvst_golden_registers,
 460                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 461                         radeon_program_register_sequence(rdev,
 462                                                          dvst_golden_registers2,
 463                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 464                 } else {
 465                         radeon_program_register_sequence(rdev,
 466                                                          scrapper_golden_registers,
 467                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 468                         radeon_program_register_sequence(rdev,
 469                                                          dvst_golden_registers2,
 470                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 471                 }
 472                 break;
 473         default:
 474                 break;
 475         }
 476 }
 477
 478 #define BTC_IO_MC_REGS_SIZE 29
 479
 480 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 481         {0x00000077, 0xff010100},
 482         {0x00000078, 0x00000000},
 483         {0x00000079, 0x00001434},
 484         {0x0000007a, 0xcc08ec08},
 485         {0x0000007b, 0x00040000},
 486         {0x0000007c, 0x000080c0},
 487         {0x0000007d, 0x09000000},
 488         {0x0000007e, 0x00210404},
 489         {0x00000081, 0x08a8e800},
 490         {0x00000082, 0x00030444},
 491         {0x00000083, 0x00000000},
 492         {0x00000085, 0x00000001},
 493         {0x00000086, 0x00000002},
 494         {0x00000087, 0x48490000},
 495         {0x00000088, 0x20244647},
 496         {0x00000089, 0x00000005},
 497         {0x0000008b, 0x66030000},
 498         {0x0000008c, 0x00006603},
 499         {0x0000008d, 0x00000100},
 500         {0x0000008f, 0x00001c0a},
 501         {0x00000090, 0xff000001},
 502         {0x00000094, 0x00101101},
 503         {0x00000095, 0x00000fff},
 504         {0x00000096, 0x00116fff},
 505         {0x00000097, 0x60010000},
 506         {0x00000098, 0x10010000},
 507         {0x00000099, 0x00006000},
 508         {0x0000009a, 0x00001000},
 509         {0x0000009f, 0x00946a00}
 510 };
 511
 512 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 513         {0x00000077, 0xff010100},
 514         {0x00000078, 0x00000000},
 515         {0x00000079, 0x00001434},
 516         {0x0000007a, 0xcc08ec08},
 517         {0x0000007b, 0x00040000},
 518         {0x0000007c, 0x000080c0},
 519         {0x0000007d, 0x09000000},
 520         {0x0000007e, 0x00210404},
 521         {0x00000081, 0x08a8e800},
 522         {0x00000082, 0x00030444},
 523         {0x00000083, 0x00000000},
 524         {0x00000085, 0x00000001},
 525         {0x00000086, 0x00000002},
 526         {0x00000087, 0x48490000},
 527         {0x00000088, 0x20244647},
 528         {0x00000089, 0x00000005},
 529         {0x0000008b, 0x66030000},
 530         {0x0000008c, 0x00006603},
 531         {0x0000008d, 0x00000100},
 532         {0x0000008f, 0x00001c0a},
 533         {0x00000090, 0xff000001},
 534         {0x00000094, 0x00101101},
 535         {0x00000095, 0x00000fff},
 536         {0x00000096, 0x00116fff},
 537         {0x00000097, 0x60010000},
 538         {0x00000098, 0x10010000},
 539         {0x00000099, 0x00006000},
 540         {0x0000009a, 0x00001000},
 541         {0x0000009f, 0x00936a00}
 542 };
 543
 544 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 545         {0x00000077, 0xff010100},
 546         {0x00000078, 0x00000000},
 547         {0x00000079, 0x00001434},
 548         {0x0000007a, 0xcc08ec08},
 549         {0x0000007b, 0x00040000},
 550         {0x0000007c, 0x000080c0},
 551         {0x0000007d, 0x09000000},
 552         {0x0000007e, 0x00210404},
 553         {0x00000081, 0x08a8e800},
 554         {0x00000082, 0x00030444},
 555         {0x00000083, 0x00000000},
 556         {0x00000085, 0x00000001},
 557         {0x00000086, 0x00000002},
 558         {0x00000087, 0x48490000},
 559         {0x00000088, 0x20244647},
 560         {0x00000089, 0x00000005},
 561         {0x0000008b, 0x66030000},
 562         {0x0000008c, 0x00006603},
 563         {0x0000008d, 0x00000100},
 564         {0x0000008f, 0x00001c0a},
 565         {0x00000090, 0xff000001},
 566         {0x00000094, 0x00101101},
 567         {0x00000095, 0x00000fff},
 568         {0x00000096, 0x00116fff},
 569         {0x00000097, 0x60010000},
 570         {0x00000098, 0x10010000},
 571         {0x00000099, 0x00006000},
 572         {0x0000009a, 0x00001000},
 573         {0x0000009f, 0x00916a00}
 574 };
 575
 576 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 577         {0x00000077, 0xff010100},
 578         {0x00000078, 0x00000000},
 579         {0x00000079, 0x00001434},
 580         {0x0000007a, 0xcc08ec08},
 581         {0x0000007b, 0x00040000},
 582         {0x0000007c, 0x000080c0},
 583         {0x0000007d, 0x09000000},
 584         {0x0000007e, 0x00210404},
 585         {0x00000081, 0x08a8e800},
 586         {0x00000082, 0x00030444},
 587         {0x00000083, 0x00000000},
 588         {0x00000085, 0x00000001},
 589         {0x00000086, 0x00000002},
 590         {0x00000087, 0x48490000},
 591         {0x00000088, 0x20244647},
 592         {0x00000089, 0x00000005},
 593         {0x0000008b, 0x66030000},
 594         {0x0000008c, 0x00006603},
 595         {0x0000008d, 0x00000100},
 596         {0x0000008f, 0x00001c0a},
 597         {0x00000090, 0xff000001},
 598         {0x00000094, 0x00101101},
 599         {0x00000095, 0x00000fff},
 600         {0x00000096, 0x00116fff},
 601         {0x00000097, 0x60010000},
 602         {0x00000098, 0x10010000},
 603         {0x00000099, 0x00006000},
 604         {0x0000009a, 0x00001000},
 605         {0x0000009f, 0x00976b00}
 606 };
 607
 608 int ni_mc_load_microcode(struct radeon_device *rdev)
 609 {
 610         const __be32 *fw_data;
 611         u32 mem_type, running, blackout = 0;
 612         u32 *io_mc_regs;
 613         int i, ucode_size, regs_size;
 614
 615         if (!rdev->mc_fw)
 616                 return -EINVAL;
 617
 618         switch (rdev->family) {
 619         case CHIP_BARTS:
 620                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 621                 ucode_size = BTC_MC_UCODE_SIZE;
 622                 regs_size = BTC_IO_MC_REGS_SIZE;
 623                 break;
 624         case CHIP_TURKS:
 625                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 626                 ucode_size = BTC_MC_UCODE_SIZE;
 627                 regs_size = BTC_IO_MC_REGS_SIZE;
 628                 break;
 629         case CHIP_CAICOS:
 630         default:
 631                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 632                 ucode_size = BTC_MC_UCODE_SIZE;
 633                 regs_size = BTC_IO_MC_REGS_SIZE;
 634                 break;
 635         case CHIP_CAYMAN:
 636                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 637                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 638                 regs_size = BTC_IO_MC_REGS_SIZE;
 639                 break;
 640         }
 641
 642         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 643         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 644
 645         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 646                 if (running) {
 647                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 648                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 649                 }
 650
 651                 /* reset the engine and set to writable */
 652                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 653                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 654
 655                 /* load mc io regs */
 656                 for (i = 0; i < regs_size; i++) {
 657                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 658                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 659                 }
 660                 /* load the MC ucode */
 661                 fw_data = (const __be32 *)rdev->mc_fw->data;
 662                 for (i = 0; i < ucode_size; i++)
 663                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 664
 665                 /* put the engine back into the active state */
 666                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 668                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 669
 670                 /* wait for training to complete */
 671                 for (i = 0; i < rdev->usec_timeout; i++) {
 672                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 673                                 break;
 674                         udelay(1);
 675                 }
 676
 677                 if (running)
 678                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 679         }
 680
 681         return 0;
 682 }
 683
 684 int ni_init_microcode(struct radeon_device *rdev)
 685 {
 686         const char *chip_name;
 687         const char *rlc_chip_name;
 688         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 689         size_t smc_req_size = 0;
 690         char fw_name[30];
 691         int err;
 692
 693         DRM_DEBUG("\n");
 694
 695         switch (rdev->family) {
 696         case CHIP_BARTS:
 697                 chip_name = "BARTS";
 698                 rlc_chip_name = "BTC";
 699                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 700                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 701                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 702                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 703                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 704                 break;
 705         case CHIP_TURKS:
 706                 chip_name = "TURKS";
 707                 rlc_chip_name = "BTC";
 708                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 709                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 710                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 711                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 712                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 713                 break;
 714         case CHIP_CAICOS:
 715                 chip_name = "CAICOS";
 716                 rlc_chip_name = "BTC";
 717                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 718                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 719                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 720                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 721                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 722                 break;
 723         case CHIP_CAYMAN:
 724                 chip_name = "CAYMAN";
 725                 rlc_chip_name = "CAYMAN";
 726                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 727                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 728                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 729                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 730                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 731                 break;
 732         case CHIP_ARUBA:
 733                 chip_name = "ARUBA";
 734                 rlc_chip_name = "ARUBA";
 735                 /* pfp/me same size as CAYMAN */
 736                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 737                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 738                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 739                 mc_req_size = 0;
 740                 break;
 741         default: BUG();
 742         }
 743
 744         DRM_INFO("Loading %s Microcode\n", chip_name);
 745
 746         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 747         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 748         if (err)
 749                 goto out;
 750         if (rdev->pfp_fw->size != pfp_req_size) {
 751                 printk(KERN_ERR
 752                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 753                        rdev->pfp_fw->size, fw_name);
 754                 err = -EINVAL;
 755                 goto out;
 756         }
 757
 758         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 759         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 760         if (err)
 761                 goto out;
 762         if (rdev->me_fw->size != me_req_size) {
 763                 printk(KERN_ERR
 764                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 765                        rdev->me_fw->size, fw_name);
 766                 err = -EINVAL;
 767         }
 768
 769         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 770         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 771         if (err)
 772                 goto out;
 773         if (rdev->rlc_fw->size != rlc_req_size) {
 774                 printk(KERN_ERR
 775                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 776                        rdev->rlc_fw->size, fw_name);
 777                 err = -EINVAL;
 778         }
 779
 780         /* no MC ucode on TN */
 781         if (!(rdev->flags & RADEON_IS_IGP)) {
 782                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 783                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 784                 if (err)
 785                         goto out;
 786                 if (rdev->mc_fw->size != mc_req_size) {
 787                         printk(KERN_ERR
 788                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 789                                rdev->mc_fw->size, fw_name);
 790                         err = -EINVAL;
 791                 }
 792         }
 793
 794         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 795                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 796                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 797                 if (err) {
 798                         printk(KERN_ERR
 799                                "smc: error loading firmware \"%s\"\n",
 800                                fw_name);
 801                         release_firmware(rdev->smc_fw);
 802                         rdev->smc_fw = NULL;
 803                         err = 0;
 804                 } else if (rdev->smc_fw->size != smc_req_size) {
 805                         printk(KERN_ERR
 806                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 807                                rdev->mc_fw->size, fw_name);
 808                         err = -EINVAL;
 809                 }
 810         }
 811
 812 out:
 813         if (err) {
 814                 if (err != -EINVAL)
 815                         printk(KERN_ERR
 816                                "ni_cp: Failed to load firmware \"%s\"\n",
 817                                fw_name);
 818                 release_firmware(rdev->pfp_fw);
 819                 rdev->pfp_fw = NULL;
 820                 release_firmware(rdev->me_fw);
 821                 rdev->me_fw = NULL;
 822                 release_firmware(rdev->rlc_fw);
 823                 rdev->rlc_fw = NULL;
 824                 release_firmware(rdev->mc_fw);
 825                 rdev->mc_fw = NULL;
 826         }
 827         return err;
 828 }
 829
 830 int tn_get_temp(struct radeon_device *rdev)
 831 {
 832         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 833         int actual_temp = (temp / 8) - 49;
 834
 835         return actual_temp * 1000;
 836 }
 837
 838 /*
 839  * Core functions
 840  */
 841 static void cayman_gpu_init(struct radeon_device *rdev)
 842 {
 843         u32 gb_addr_config = 0;
 844         u32 mc_shared_chmap, mc_arb_ramcfg;
 845         u32 cgts_tcc_disable;
 846         u32 sx_debug_1;
 847         u32 smx_dc_ctl0;
 848         u32 cgts_sm_ctrl_reg;
 849         u32 hdp_host_path_cntl;
 850         u32 tmp;
 851         u32 disabled_rb_mask;
 852         int i, j;
 853
 854         switch (rdev->family) {
 855         case CHIP_CAYMAN:
 856                 rdev->config.cayman.max_shader_engines = 2;
 857                 rdev->config.cayman.max_pipes_per_simd = 4;
 858                 rdev->config.cayman.max_tile_pipes = 8;
 859                 rdev->config.cayman.max_simds_per_se = 12;
 860                 rdev->config.cayman.max_backends_per_se = 4;
 861                 rdev->config.cayman.max_texture_channel_caches = 8;
 862                 rdev->config.cayman.max_gprs = 256;
 863                 rdev->config.cayman.max_threads = 256;
 864                 rdev->config.cayman.max_gs_threads = 32;
 865                 rdev->config.cayman.max_stack_entries = 512;
 866                 rdev->config.cayman.sx_num_of_sets = 8;
 867                 rdev->config.cayman.sx_max_export_size = 256;
 868                 rdev->config.cayman.sx_max_export_pos_size = 64;
 869                 rdev->config.cayman.sx_max_export_smx_size = 192;
 870                 rdev->config.cayman.max_hw_contexts = 8;
 871                 rdev->config.cayman.sq_num_cf_insts = 2;
 872
 873                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 874                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 875                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 876                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 877                 break;
 878         case CHIP_ARUBA:
 879         default:
 880                 rdev->config.cayman.max_shader_engines = 1;
 881                 rdev->config.cayman.max_pipes_per_simd = 4;
 882                 rdev->config.cayman.max_tile_pipes = 2;
 883                 if ((rdev->pdev->device == 0x9900) ||
 884                     (rdev->pdev->device == 0x9901) ||
 885                     (rdev->pdev->device == 0x9905) ||
 886                     (rdev->pdev->device == 0x9906) ||
 887                     (rdev->pdev->device == 0x9907) ||
 888                     (rdev->pdev->device == 0x9908) ||
 889                     (rdev->pdev->device == 0x9909) ||
 890                     (rdev->pdev->device == 0x990B) ||
 891                     (rdev->pdev->device == 0x990C) ||
 892                     (rdev->pdev->device == 0x990F) ||
 893                     (rdev->pdev->device == 0x9910) ||
 894                     (rdev->pdev->device == 0x9917) ||
 895                     (rdev->pdev->device == 0x9999) ||
 896                     (rdev->pdev->device == 0x999C)) {
 897                         rdev->config.cayman.max_simds_per_se = 6;
 898                         rdev->config.cayman.max_backends_per_se = 2;
 899                         rdev->config.cayman.max_hw_contexts = 8;
 900                         rdev->config.cayman.sx_max_export_size = 256;
 901                         rdev->config.cayman.sx_max_export_pos_size = 64;
 902                         rdev->config.cayman.sx_max_export_smx_size = 192;
 903                 } else if ((rdev->pdev->device == 0x9903) ||
 904                            (rdev->pdev->device == 0x9904) ||
 905                            (rdev->pdev->device == 0x990A) ||
 906                            (rdev->pdev->device == 0x990D) ||
 907                            (rdev->pdev->device == 0x990E) ||
 908                            (rdev->pdev->device == 0x9913) ||
 909                            (rdev->pdev->device == 0x9918) ||
 910                            (rdev->pdev->device == 0x999D)) {
 911                         rdev->config.cayman.max_simds_per_se = 4;
 912                         rdev->config.cayman.max_backends_per_se = 2;
 913                         rdev->config.cayman.max_hw_contexts = 8;
 914                         rdev->config.cayman.sx_max_export_size = 256;
 915                         rdev->config.cayman.sx_max_export_pos_size = 64;
 916                         rdev->config.cayman.sx_max_export_smx_size = 192;
 917                 } else if ((rdev->pdev->device == 0x9919) ||
 918                            (rdev->pdev->device == 0x9990) ||
 919                            (rdev->pdev->device == 0x9991) ||
 920                            (rdev->pdev->device == 0x9994) ||
 921                            (rdev->pdev->device == 0x9995) ||
 922                            (rdev->pdev->device == 0x9996) ||
 923                            (rdev->pdev->device == 0x999A) ||
 924                            (rdev->pdev->device == 0x99A0)) {
 925                         rdev->config.cayman.max_simds_per_se = 3;
 926                         rdev->config.cayman.max_backends_per_se = 1;
 927                         rdev->config.cayman.max_hw_contexts = 4;
 928                         rdev->config.cayman.sx_max_export_size = 128;
 929                         rdev->config.cayman.sx_max_export_pos_size = 32;
 930                         rdev->config.cayman.sx_max_export_smx_size = 96;
 931                 } else {
 932                         rdev->config.cayman.max_simds_per_se = 2;
 933                         rdev->config.cayman.max_backends_per_se = 1;
 934                         rdev->config.cayman.max_hw_contexts = 4;
 935                         rdev->config.cayman.sx_max_export_size = 128;
 936                         rdev->config.cayman.sx_max_export_pos_size = 32;
 937                         rdev->config.cayman.sx_max_export_smx_size = 96;
 938                 }
 939                 rdev->config.cayman.max_texture_channel_caches = 2;
 940                 rdev->config.cayman.max_gprs = 256;
 941                 rdev->config.cayman.max_threads = 256;
 942                 rdev->config.cayman.max_gs_threads = 32;
 943                 rdev->config.cayman.max_stack_entries = 512;
 944                 rdev->config.cayman.sx_num_of_sets = 8;
 945                 rdev->config.cayman.sq_num_cf_insts = 2;
 946
 947                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 948                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 949                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 950                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 951                 break;
 952         }
 953
 954         /* Initialize HDP */
 955         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 956                 WREG32((0x2c14 + j), 0x00000000);
 957                 WREG32((0x2c18 + j), 0x00000000);
 958                 WREG32((0x2c1c + j), 0x00000000);
 959                 WREG32((0x2c20 + j), 0x00000000);
 960                 WREG32((0x2c24 + j), 0x00000000);
 961         }
 962
 963         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 964
 965         evergreen_fix_pci_max_read_req_size(rdev);
 966
 967         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 968         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 969
 970         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 971         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 972         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 973                 rdev->config.cayman.mem_row_size_in_kb = 4;
 974         /* XXX use MC settings? */
 975         rdev->config.cayman.shader_engine_tile_size = 32;
 976         rdev->config.cayman.num_gpus = 1;
 977         rdev->config.cayman.multi_gpu_tile_size = 64;
 978
 979         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 980         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 981         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 982         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 983         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 984         rdev->config.cayman.num_shader_engines = tmp + 1;
 985         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 986         rdev->config.cayman.num_gpus = tmp + 1;
 987         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 988         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 989         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 990         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 991
 992
 993         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 994          * not have bank info, so create a custom tiling dword.
 995          * bits 3:0   num_pipes
 996          * bits 7:4   num_banks
 997          * bits 11:8  group_size
 998          * bits 15:12 row_size
 999          */
1000         rdev->config.cayman.tile_config = 0;
1001         switch (rdev->config.cayman.num_tile_pipes) {
1002         case 1:
1003         default:
1004                 rdev->config.cayman.tile_config |= (0 << 0);
1005                 break;
1006         case 2:
1007                 rdev->config.cayman.tile_config |= (1 << 0);
1008                 break;
1009         case 4:
1010                 rdev->config.cayman.tile_config |= (2 << 0);
1011                 break;
1012         case 8:
1013                 rdev->config.cayman.tile_config |= (3 << 0);
1014                 break;
1015         }
1016
1017         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1018         if (rdev->flags & RADEON_IS_IGP)
1019                 rdev->config.cayman.tile_config |= 1 << 4;
1020         else {
1021                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1022                 case 0: /* four banks */
1023                         rdev->config.cayman.tile_config |= 0 << 4;
1024                         break;
1025                 case 1: /* eight banks */
1026                         rdev->config.cayman.tile_config |= 1 << 4;
1027                         break;
1028                 case 2: /* sixteen banks */
1029                 default:
1030                         rdev->config.cayman.tile_config |= 2 << 4;
1031                         break;
1032                 }
1033         }
1034         rdev->config.cayman.tile_config |=
1035                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1036         rdev->config.cayman.tile_config |=
1037                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1038
1039         tmp = 0;
1040         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1041                 u32 rb_disable_bitmap;
1042
1043                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1044                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1045                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1046                 tmp <<= 4;
1047                 tmp |= rb_disable_bitmap;
1048         }
1049         /* enabled rb are just the one not disabled :) */
1050         disabled_rb_mask = tmp;
1051         tmp = 0;
1052         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1053                 tmp |= (1 << i);
1054         /* if all the backends are disabled, fix it up here */
1055         if ((disabled_rb_mask & tmp) == tmp) {
1056                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1057                         disabled_rb_mask &= ~(1 << i);
1058         }
1059
1060         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1061         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1062
1063         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1064         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1065         if (ASIC_IS_DCE6(rdev))
1066                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1067         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1068         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1069         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1070         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1071         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1072         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1073
1074         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1075             (rdev->flags & RADEON_IS_IGP)) {
1076                 if ((disabled_rb_mask & 3) == 1) {
1077                         /* RB0 disabled, RB1 enabled */
1078                         tmp = 0x11111111;
1079                 } else {
1080                         /* RB1 disabled, RB0 enabled */
1081                         tmp = 0x00000000;
1082                 }
1083         } else {
1084                 tmp = gb_addr_config & NUM_PIPES_MASK;
1085                 tmp = r6xx_remap_render_backend(rdev, tmp,
1086                                                 rdev->config.cayman.max_backends_per_se *
1087                                                 rdev->config.cayman.max_shader_engines,
1088                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1089         }
1090         WREG32(GB_BACKEND_MAP, tmp);
1091
1092         cgts_tcc_disable = 0xffff0000;
1093         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1094                 cgts_tcc_disable &= ~(1 << (16 + i));
1095         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1096         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1097         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1098         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1099
1100         /* reprogram the shader complex */
1101         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1102         for (i = 0; i < 16; i++)
1103                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1104         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1105
1106         /* set HW defaults for 3D engine */
1107         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1108
1109         sx_debug_1 = RREG32(SX_DEBUG_1);
1110         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1111         WREG32(SX_DEBUG_1, sx_debug_1);
1112
1113         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1114         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1115         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1116         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1117
1118         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1119
1120         /* need to be explicitly zero-ed */
1121         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1122         WREG32(SQ_LSTMP_RING_BASE, 0);
1123         WREG32(SQ_HSTMP_RING_BASE, 0);
1124         WREG32(SQ_ESTMP_RING_BASE, 0);
1125         WREG32(SQ_GSTMP_RING_BASE, 0);
1126         WREG32(SQ_VSTMP_RING_BASE, 0);
1127         WREG32(SQ_PSTMP_RING_BASE, 0);
1128
1129         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1130
1131         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1132                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1133                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1134
1135         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1136                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1137                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1138
1139
1140         WREG32(VGT_NUM_INSTANCES, 1);
1141
1142         WREG32(CP_PERFMON_CNTL, 0);
1143
1144         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1145                                   FETCH_FIFO_HIWATER(0x4) |
1146                                   DONE_FIFO_HIWATER(0xe0) |
1147                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1148
1149         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1150         WREG32(SQ_CONFIG, (VC_ENABLE |
1151                            EXPORT_SRC_C |
1152                            GFX_PRIO(0) |
1153                            CS1_PRIO(0) |
1154                            CS2_PRIO(1)));
1155         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1156
1157         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1158                                           FORCE_EOV_MAX_REZ_CNT(255)));
1159
1160         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1161                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1162
1163         WREG32(VGT_GS_VERTEX_REUSE, 16);
1164         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1165
1166         WREG32(CB_PERF_CTR0_SEL_0, 0);
1167         WREG32(CB_PERF_CTR0_SEL_1, 0);
1168         WREG32(CB_PERF_CTR1_SEL_0, 0);
1169         WREG32(CB_PERF_CTR1_SEL_1, 0);
1170         WREG32(CB_PERF_CTR2_SEL_0, 0);
1171         WREG32(CB_PERF_CTR2_SEL_1, 0);
1172         WREG32(CB_PERF_CTR3_SEL_0, 0);
1173         WREG32(CB_PERF_CTR3_SEL_1, 0);
1174
1175         tmp = RREG32(HDP_MISC_CNTL);
1176         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1177         WREG32(HDP_MISC_CNTL, tmp);
1178
1179         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1180         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1181
1182         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1183
1184         udelay(50);
1185
1186         /* set clockgating golden values on TN */
1187         if (rdev->family == CHIP_ARUBA) {
1188                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1189                 tmp &= ~0x00380000;
1190                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1191                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1192                 tmp &= ~0x0e000000;
1193                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1194         }
1195 }
1196
1197 /*
1198  * GART
1199  */
1200 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1201 {
1202         /* flush hdp cache */
1203         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1204
1205         /* bits 0-7 are the VM contexts0-7 */
1206         WREG32(VM_INVALIDATE_REQUEST, 1);
1207 }
1208
1209 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1210 {
1211         int i, r;
1212
1213         if (rdev->gart.robj == NULL) {
1214                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1215                 return -EINVAL;
1216         }
1217         r = radeon_gart_table_vram_pin(rdev);
1218         if (r)
1219                 return r;
1220         radeon_gart_restore(rdev);
1221         /* Setup TLB control */
1222         WREG32(MC_VM_MX_L1_TLB_CNTL,
1223                (0xA << 7) |
1224                ENABLE_L1_TLB |
1225                ENABLE_L1_FRAGMENT_PROCESSING |
1226                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1227                ENABLE_ADVANCED_DRIVER_MODEL |
1228                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1229         /* Setup L2 cache */
1230         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1231                ENABLE_L2_FRAGMENT_PROCESSING |
1232                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1233                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1234                EFFECTIVE_L2_QUEUE_SIZE(7) |
1235                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1236         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1237         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1238                BANK_SELECT(6) |
1239                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1240         /* setup context0 */
1241         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1242         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1243         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1244         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1245                         (u32)(rdev->dummy_page.addr >> 12));
1246         WREG32(VM_CONTEXT0_CNTL2, 0);
1247         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1248                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1249
1250         WREG32(0x15D4, 0);
1251         WREG32(0x15D8, 0);
1252         WREG32(0x15DC, 0);
1253
1254         /* empty context1-7 */
1255         /* Assign the pt base to something valid for now; the pts used for
1256          * the VMs are determined by the application and setup and assigned
1257          * on the fly in the vm part of radeon_gart.c
1258          */
1259         for (i = 1; i < 8; i++) {
1260                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1261                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1262                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1263                         rdev->gart.table_addr >> 12);
1264         }
1265
1266         /* enable context1-7 */
1267         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1268                (u32)(rdev->dummy_page.addr >> 12));
1269         WREG32(VM_CONTEXT1_CNTL2, 4);
1270         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1271                                 PAGE_TABLE_BLOCK_SIZE(RADEON_VM_BLOCK_SIZE - 9) |
1272                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1273                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1274                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1275                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1276                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1277                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1278                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1279                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1280                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1281                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1282                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1283                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1284
1285         cayman_pcie_gart_tlb_flush(rdev);
1286         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1287                  (unsigned)(rdev->mc.gtt_size >> 20),
1288                  (unsigned long long)rdev->gart.table_addr);
1289         rdev->gart.ready = true;
1290         return 0;
1291 }
1292
1293 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1294 {
1295         /* Disable all tables */
1296         WREG32(VM_CONTEXT0_CNTL, 0);
1297         WREG32(VM_CONTEXT1_CNTL, 0);
1298         /* Setup TLB control */
1299         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1300                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1301                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1302         /* Setup L2 cache */
1303         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1304                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1305                EFFECTIVE_L2_QUEUE_SIZE(7) |
1306                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1307         WREG32(VM_L2_CNTL2, 0);
1308         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1309                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1310         radeon_gart_table_vram_unpin(rdev);
1311 }
1312
1313 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1314 {
1315         cayman_pcie_gart_disable(rdev);
1316         radeon_gart_table_vram_free(rdev);
1317         radeon_gart_fini(rdev);
1318 }
1319
1320 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1321                               int ring, u32 cp_int_cntl)
1322 {
1323         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1324
1325         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1326         WREG32(CP_INT_CNTL, cp_int_cntl);
1327 }
1328
1329 /*
1330  * CP.
1331  */
1332 void cayman_fence_ring_emit(struct radeon_device *rdev,
1333                             struct radeon_fence *fence)
1334 {
1335         struct radeon_ring *ring = &rdev->ring[fence->ring];
1336         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1337         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1338                 PACKET3_SH_ACTION_ENA;
1339
1340         /* flush read cache over gart for this vmid */
1341         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1342         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1343         radeon_ring_write(ring, 0xFFFFFFFF);
1344         radeon_ring_write(ring, 0);
1345         radeon_ring_write(ring, 10); /* poll interval */
1346         /* EVENT_WRITE_EOP - flush caches, send int */
1347         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1348         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1349         radeon_ring_write(ring, addr & 0xffffffff);
1350         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1351         radeon_ring_write(ring, fence->seq);
1352         radeon_ring_write(ring, 0);
1353 }
1354
1355 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1356 {
1357         struct radeon_ring *ring = &rdev->ring[ib->ring];
1358         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1359                 PACKET3_SH_ACTION_ENA;
1360
1361         /* set to DX10/11 mode */
1362         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1363         radeon_ring_write(ring, 1);
1364
1365         if (ring->rptr_save_reg) {
1366                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1367                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1368                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1369                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1370                 radeon_ring_write(ring, next_rptr);
1371         }
1372
1373         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1374         radeon_ring_write(ring,
1375 #ifdef __BIG_ENDIAN
1376                           (2 << 0) |
1377 #endif
1378                           (ib->gpu_addr & 0xFFFFFFFC));
1379         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1380         radeon_ring_write(ring, ib->length_dw |
1381                           (ib->vm ? (ib->vm->id << 24) : 0));
1382
1383         /* flush read cache over gart for this vmid */
1384         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1385         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1386         radeon_ring_write(ring, 0xFFFFFFFF);
1387         radeon_ring_write(ring, 0);
1388         radeon_ring_write(ring, ((ib->vm ? ib->vm->id : 0) << 24) | 10); /* poll interval */
1389 }
1390
1391 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1392 {
1393         if (enable)
1394                 WREG32(CP_ME_CNTL, 0);
1395         else {
1396                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1397                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1398                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1399                 WREG32(SCRATCH_UMSK, 0);
1400                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1401         }
1402 }
1403
1404 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1405                         struct radeon_ring *ring)
1406 {
1407         u32 rptr;
1408
1409         if (rdev->wb.enabled)
1410                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1411         else {
1412                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1413                         rptr = RREG32(CP_RB0_RPTR);
1414                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1415                         rptr = RREG32(CP_RB1_RPTR);
1416                 else
1417                         rptr = RREG32(CP_RB2_RPTR);
1418         }
1419
1420         return rptr;
1421 }
1422
1423 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1424                         struct radeon_ring *ring)
1425 {
1426         u32 wptr;
1427
1428         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1429                 wptr = RREG32(CP_RB0_WPTR);
1430         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1431                 wptr = RREG32(CP_RB1_WPTR);
1432         else
1433                 wptr = RREG32(CP_RB2_WPTR);
1434
1435         return wptr;
1436 }
1437
1438 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1439                          struct radeon_ring *ring)
1440 {
1441         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1442                 WREG32(CP_RB0_WPTR, ring->wptr);
1443                 (void)RREG32(CP_RB0_WPTR);
1444         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1445                 WREG32(CP_RB1_WPTR, ring->wptr);
1446                 (void)RREG32(CP_RB1_WPTR);
1447         } else {
1448                 WREG32(CP_RB2_WPTR, ring->wptr);
1449                 (void)RREG32(CP_RB2_WPTR);
1450         }
1451 }
1452
1453 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1454 {
1455         const __be32 *fw_data;
1456         int i;
1457
1458         if (!rdev->me_fw || !rdev->pfp_fw)
1459                 return -EINVAL;
1460
1461         cayman_cp_enable(rdev, false);
1462
1463         fw_data = (const __be32 *)rdev->pfp_fw->data;
1464         WREG32(CP_PFP_UCODE_ADDR, 0);
1465         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1466                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1467         WREG32(CP_PFP_UCODE_ADDR, 0);
1468
1469         fw_data = (const __be32 *)rdev->me_fw->data;
1470         WREG32(CP_ME_RAM_WADDR, 0);
1471         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1472                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1473
1474         WREG32(CP_PFP_UCODE_ADDR, 0);
1475         WREG32(CP_ME_RAM_WADDR, 0);
1476         WREG32(CP_ME_RAM_RADDR, 0);
1477         return 0;
1478 }
1479
1480 static int cayman_cp_start(struct radeon_device *rdev)
1481 {
1482         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1483         int r, i;
1484
1485         r = radeon_ring_lock(rdev, ring, 7);
1486         if (r) {
1487                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1488                 return r;
1489         }
1490         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1491         radeon_ring_write(ring, 0x1);
1492         radeon_ring_write(ring, 0x0);
1493         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1494         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1495         radeon_ring_write(ring, 0);
1496         radeon_ring_write(ring, 0);
1497         radeon_ring_unlock_commit(rdev, ring);
1498
1499         cayman_cp_enable(rdev, true);
1500
1501         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1502         if (r) {
1503                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1504                 return r;
1505         }
1506
1507         /* setup clear context state */
1508         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1509         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1510
1511         for (i = 0; i < cayman_default_size; i++)
1512                 radeon_ring_write(ring, cayman_default_state[i]);
1513
1514         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1515         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1516
1517         /* set clear context state */
1518         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1519         radeon_ring_write(ring, 0);
1520
1521         /* SQ_VTX_BASE_VTX_LOC */
1522         radeon_ring_write(ring, 0xc0026f00);
1523         radeon_ring_write(ring, 0x00000000);
1524         radeon_ring_write(ring, 0x00000000);
1525         radeon_ring_write(ring, 0x00000000);
1526
1527         /* Clear consts */
1528         radeon_ring_write(ring, 0xc0036f00);
1529         radeon_ring_write(ring, 0x00000bc4);
1530         radeon_ring_write(ring, 0xffffffff);
1531         radeon_ring_write(ring, 0xffffffff);
1532         radeon_ring_write(ring, 0xffffffff);
1533
1534         radeon_ring_write(ring, 0xc0026900);
1535         radeon_ring_write(ring, 0x00000316);
1536         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1537         radeon_ring_write(ring, 0x00000010); /*  */
1538
1539         radeon_ring_unlock_commit(rdev, ring);
1540
1541         /* XXX init other rings */
1542
1543         return 0;
1544 }
1545
1546 static void cayman_cp_fini(struct radeon_device *rdev)
1547 {
1548         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1549         cayman_cp_enable(rdev, false);
1550         radeon_ring_fini(rdev, ring);
1551         radeon_scratch_free(rdev, ring->rptr_save_reg);
1552 }
1553
1554 static int cayman_cp_resume(struct radeon_device *rdev)
1555 {
1556         static const int ridx[] = {
1557                 RADEON_RING_TYPE_GFX_INDEX,
1558                 CAYMAN_RING_TYPE_CP1_INDEX,
1559                 CAYMAN_RING_TYPE_CP2_INDEX
1560         };
1561         static const unsigned cp_rb_cntl[] = {
1562                 CP_RB0_CNTL,
1563                 CP_RB1_CNTL,
1564                 CP_RB2_CNTL,
1565         };
1566         static const unsigned cp_rb_rptr_addr[] = {
1567                 CP_RB0_RPTR_ADDR,
1568                 CP_RB1_RPTR_ADDR,
1569                 CP_RB2_RPTR_ADDR
1570         };
1571         static const unsigned cp_rb_rptr_addr_hi[] = {
1572                 CP_RB0_RPTR_ADDR_HI,
1573                 CP_RB1_RPTR_ADDR_HI,
1574                 CP_RB2_RPTR_ADDR_HI
1575         };
1576         static const unsigned cp_rb_base[] = {
1577                 CP_RB0_BASE,
1578                 CP_RB1_BASE,
1579                 CP_RB2_BASE
1580         };
1581         static const unsigned cp_rb_rptr[] = {
1582                 CP_RB0_RPTR,
1583                 CP_RB1_RPTR,
1584                 CP_RB2_RPTR
1585         };
1586         static const unsigned cp_rb_wptr[] = {
1587                 CP_RB0_WPTR,
1588                 CP_RB1_WPTR,
1589                 CP_RB2_WPTR
1590         };
1591         struct radeon_ring *ring;
1592         int i, r;
1593
1594         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1595         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1596                                  SOFT_RESET_PA |
1597                                  SOFT_RESET_SH |
1598                                  SOFT_RESET_VGT |
1599                                  SOFT_RESET_SPI |
1600                                  SOFT_RESET_SX));
1601         RREG32(GRBM_SOFT_RESET);
1602         mdelay(15);
1603         WREG32(GRBM_SOFT_RESET, 0);
1604         RREG32(GRBM_SOFT_RESET);
1605
1606         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1607         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1608
1609         /* Set the write pointer delay */
1610         WREG32(CP_RB_WPTR_DELAY, 0);
1611
1612         WREG32(CP_DEBUG, (1 << 27));
1613
1614         /* set the wb address whether it's enabled or not */
1615         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1616         WREG32(SCRATCH_UMSK, 0xff);
1617
1618         for (i = 0; i < 3; ++i) {
1619                 uint32_t rb_cntl;
1620                 uint64_t addr;
1621
1622                 /* Set ring buffer size */
1623                 ring = &rdev->ring[ridx[i]];
1624                 rb_cntl = order_base_2(ring->ring_size / 8);
1625                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1626 #ifdef __BIG_ENDIAN
1627                 rb_cntl |= BUF_SWAP_32BIT;
1628 #endif
1629                 WREG32(cp_rb_cntl[i], rb_cntl);
1630
1631                 /* set the wb address whether it's enabled or not */
1632                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1633                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1634                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1635         }
1636
1637         /* set the rb base addr, this causes an internal reset of ALL rings */
1638         for (i = 0; i < 3; ++i) {
1639                 ring = &rdev->ring[ridx[i]];
1640                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1641         }
1642
1643         for (i = 0; i < 3; ++i) {
1644                 /* Initialize the ring buffer's read and write pointers */
1645                 ring = &rdev->ring[ridx[i]];
1646                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1647
1648                 ring->wptr = 0;
1649                 WREG32(cp_rb_rptr[i], 0);
1650                 WREG32(cp_rb_wptr[i], ring->wptr);
1651
1652                 mdelay(1);
1653                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1654         }
1655
1656         /* start the rings */
1657         cayman_cp_start(rdev);
1658         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1659         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1660         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1661         /* this only test cp0 */
1662         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1663         if (r) {
1664                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1665                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1666                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1667                 return r;
1668         }
1669
1670         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1671                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1672
1673         return 0;
1674 }
1675
1676 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1677 {
1678         u32 reset_mask = 0;
1679         u32 tmp;
1680
1681         /* GRBM_STATUS */
1682         tmp = RREG32(GRBM_STATUS);
1683         if (tmp & (PA_BUSY | SC_BUSY |
1684                    SH_BUSY | SX_BUSY |
1685                    TA_BUSY | VGT_BUSY |
1686                    DB_BUSY | CB_BUSY |
1687                    GDS_BUSY | SPI_BUSY |
1688                    IA_BUSY | IA_BUSY_NO_DMA))
1689                 reset_mask |= RADEON_RESET_GFX;
1690
1691         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1692                    CP_BUSY | CP_COHERENCY_BUSY))
1693                 reset_mask |= RADEON_RESET_CP;
1694
1695         if (tmp & GRBM_EE_BUSY)
1696                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1697
1698         /* DMA_STATUS_REG 0 */
1699         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1700         if (!(tmp & DMA_IDLE))
1701                 reset_mask |= RADEON_RESET_DMA;
1702
1703         /* DMA_STATUS_REG 1 */
1704         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1705         if (!(tmp & DMA_IDLE))
1706                 reset_mask |= RADEON_RESET_DMA1;
1707
1708         /* SRBM_STATUS2 */
1709         tmp = RREG32(SRBM_STATUS2);
1710         if (tmp & DMA_BUSY)
1711                 reset_mask |= RADEON_RESET_DMA;
1712
1713         if (tmp & DMA1_BUSY)
1714                 reset_mask |= RADEON_RESET_DMA1;
1715
1716         /* SRBM_STATUS */
1717         tmp = RREG32(SRBM_STATUS);
1718         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1719                 reset_mask |= RADEON_RESET_RLC;
1720
1721         if (tmp & IH_BUSY)
1722                 reset_mask |= RADEON_RESET_IH;
1723
1724         if (tmp & SEM_BUSY)
1725                 reset_mask |= RADEON_RESET_SEM;
1726
1727         if (tmp & GRBM_RQ_PENDING)
1728                 reset_mask |= RADEON_RESET_GRBM;
1729
1730         if (tmp & VMC_BUSY)
1731                 reset_mask |= RADEON_RESET_VMC;
1732
1733         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1734                    MCC_BUSY | MCD_BUSY))
1735                 reset_mask |= RADEON_RESET_MC;
1736
1737         if (evergreen_is_display_hung(rdev))
1738                 reset_mask |= RADEON_RESET_DISPLAY;
1739
1740         /* VM_L2_STATUS */
1741         tmp = RREG32(VM_L2_STATUS);
1742         if (tmp & L2_BUSY)
1743                 reset_mask |= RADEON_RESET_VMC;
1744
1745         /* Skip MC reset as it's mostly likely not hung, just busy */
1746         if (reset_mask & RADEON_RESET_MC) {
1747                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1748                 reset_mask &= ~RADEON_RESET_MC;
1749         }
1750
1751         return reset_mask;
1752 }
1753
1754 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1755 {
1756         struct evergreen_mc_save save;
1757         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1758         u32 tmp;
1759
1760         if (reset_mask == 0)
1761                 return;
1762
1763         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1764
1765         evergreen_print_gpu_status_regs(rdev);
1766         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1767                  RREG32(0x14F8));
1768         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1769                  RREG32(0x14D8));
1770         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1771                  RREG32(0x14FC));
1772         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1773                  RREG32(0x14DC));
1774
1775         /* Disable CP parsing/prefetching */
1776         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1777
1778         if (reset_mask & RADEON_RESET_DMA) {
1779                 /* dma0 */
1780                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1781                 tmp &= ~DMA_RB_ENABLE;
1782                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1783         }
1784
1785         if (reset_mask & RADEON_RESET_DMA1) {
1786                 /* dma1 */
1787                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1788                 tmp &= ~DMA_RB_ENABLE;
1789                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1790         }
1791
1792         udelay(50);
1793
1794         evergreen_mc_stop(rdev, &save);
1795         if (evergreen_mc_wait_for_idle(rdev)) {
1796                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1797         }
1798
1799         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1800                 grbm_soft_reset = SOFT_RESET_CB |
1801                         SOFT_RESET_DB |
1802                         SOFT_RESET_GDS |
1803                         SOFT_RESET_PA |
1804                         SOFT_RESET_SC |
1805                         SOFT_RESET_SPI |
1806                         SOFT_RESET_SH |
1807                         SOFT_RESET_SX |
1808                         SOFT_RESET_TC |
1809                         SOFT_RESET_TA |
1810                         SOFT_RESET_VGT |
1811                         SOFT_RESET_IA;
1812         }
1813
1814         if (reset_mask & RADEON_RESET_CP) {
1815                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1816
1817                 srbm_soft_reset |= SOFT_RESET_GRBM;
1818         }
1819
1820         if (reset_mask & RADEON_RESET_DMA)
1821                 srbm_soft_reset |= SOFT_RESET_DMA;
1822
1823         if (reset_mask & RADEON_RESET_DMA1)
1824                 srbm_soft_reset |= SOFT_RESET_DMA1;
1825
1826         if (reset_mask & RADEON_RESET_DISPLAY)
1827                 srbm_soft_reset |= SOFT_RESET_DC;
1828
1829         if (reset_mask & RADEON_RESET_RLC)
1830                 srbm_soft_reset |= SOFT_RESET_RLC;
1831
1832         if (reset_mask & RADEON_RESET_SEM)
1833                 srbm_soft_reset |= SOFT_RESET_SEM;
1834
1835         if (reset_mask & RADEON_RESET_IH)
1836                 srbm_soft_reset |= SOFT_RESET_IH;
1837
1838         if (reset_mask & RADEON_RESET_GRBM)
1839                 srbm_soft_reset |= SOFT_RESET_GRBM;
1840
1841         if (reset_mask & RADEON_RESET_VMC)
1842                 srbm_soft_reset |= SOFT_RESET_VMC;
1843
1844         if (!(rdev->flags & RADEON_IS_IGP)) {
1845                 if (reset_mask & RADEON_RESET_MC)
1846                         srbm_soft_reset |= SOFT_RESET_MC;
1847         }
1848
1849         if (grbm_soft_reset) {
1850                 tmp = RREG32(GRBM_SOFT_RESET);
1851                 tmp |= grbm_soft_reset;
1852                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1853                 WREG32(GRBM_SOFT_RESET, tmp);
1854                 tmp = RREG32(GRBM_SOFT_RESET);
1855
1856                 udelay(50);
1857
1858                 tmp &= ~grbm_soft_reset;
1859                 WREG32(GRBM_SOFT_RESET, tmp);
1860                 tmp = RREG32(GRBM_SOFT_RESET);
1861         }
1862
1863         if (srbm_soft_reset) {
1864                 tmp = RREG32(SRBM_SOFT_RESET);
1865                 tmp |= srbm_soft_reset;
1866                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1867                 WREG32(SRBM_SOFT_RESET, tmp);
1868                 tmp = RREG32(SRBM_SOFT_RESET);
1869
1870                 udelay(50);
1871
1872                 tmp &= ~srbm_soft_reset;
1873                 WREG32(SRBM_SOFT_RESET, tmp);
1874                 tmp = RREG32(SRBM_SOFT_RESET);
1875         }
1876
1877         /* Wait a little for things to settle down */
1878         udelay(50);
1879
1880         evergreen_mc_resume(rdev, &save);
1881         udelay(50);
1882
1883         evergreen_print_gpu_status_regs(rdev);
1884 }
1885
1886 int cayman_asic_reset(struct radeon_device *rdev)
1887 {
1888         u32 reset_mask;
1889
1890         reset_mask = cayman_gpu_check_soft_reset(rdev);
1891
1892         if (reset_mask)
1893                 r600_set_bios_scratch_engine_hung(rdev, true);
1894
1895         cayman_gpu_soft_reset(rdev, reset_mask);
1896
1897         reset_mask = cayman_gpu_check_soft_reset(rdev);
1898
1899         if (reset_mask)
1900                 evergreen_gpu_pci_config_reset(rdev);
1901
1902         r600_set_bios_scratch_engine_hung(rdev, false);
1903
1904         return 0;
1905 }
1906
1907 /**
1908  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1909  *
1910  * @rdev: radeon_device pointer
1911  * @ring: radeon_ring structure holding ring information
1912  *
1913  * Check if the GFX engine is locked up.
1914  * Returns true if the engine appears to be locked up, false if not.
1915  */
1916 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1917 {
1918         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1919
1920         if (!(reset_mask & (RADEON_RESET_GFX |
1921                             RADEON_RESET_COMPUTE |
1922                             RADEON_RESET_CP))) {
1923                 radeon_ring_lockup_update(rdev, ring);
1924                 return false;
1925         }
1926         return radeon_ring_test_lockup(rdev, ring);
1927 }
1928
1929 static int cayman_startup(struct radeon_device *rdev)
1930 {
1931         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1932         int r;
1933
1934         /* enable pcie gen2 link */
1935         evergreen_pcie_gen2_enable(rdev);
1936         /* enable aspm */
1937         evergreen_program_aspm(rdev);
1938
1939         /* scratch needs to be initialized before MC */
1940         r = r600_vram_scratch_init(rdev);
1941         if (r)
1942                 return r;
1943
1944         evergreen_mc_program(rdev);
1945
1946         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1947                 r = ni_mc_load_microcode(rdev);
1948                 if (r) {
1949                         DRM_ERROR("Failed to load MC firmware!\n");
1950                         return r;
1951                 }
1952         }
1953
1954         r = cayman_pcie_gart_enable(rdev);
1955         if (r)
1956                 return r;
1957         cayman_gpu_init(rdev);
1958
1959         /* allocate rlc buffers */
1960         if (rdev->flags & RADEON_IS_IGP) {
1961                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1962                 rdev->rlc.reg_list_size =
1963                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1964                 rdev->rlc.cs_data = cayman_cs_data;
1965                 r = sumo_rlc_init(rdev);
1966                 if (r) {
1967                         DRM_ERROR("Failed to init rlc BOs!\n");
1968                         return r;
1969                 }
1970         }
1971
1972         /* allocate wb buffer */
1973         r = radeon_wb_init(rdev);
1974         if (r)
1975                 return r;
1976
1977         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1978         if (r) {
1979                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1980                 return r;
1981         }
1982
1983         r = uvd_v2_2_resume(rdev);
1984         if (!r) {
1985                 r = radeon_fence_driver_start_ring(rdev,
1986                                                    R600_RING_TYPE_UVD_INDEX);
1987                 if (r)
1988                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1989         }
1990         if (r)
1991                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1992
1993         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
1994         if (r) {
1995                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1996                 return r;
1997         }
1998
1999         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2000         if (r) {
2001                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2002                 return r;
2003         }
2004
2005         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2006         if (r) {
2007                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2008                 return r;
2009         }
2010
2011         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2012         if (r) {
2013                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2014                 return r;
2015         }
2016
2017         /* Enable IRQ */
2018         if (!rdev->irq.installed) {
2019                 r = radeon_irq_kms_init(rdev);
2020                 if (r)
2021                         return r;
2022         }
2023
2024         r = r600_irq_init(rdev);
2025         if (r) {
2026                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2027                 radeon_irq_kms_fini(rdev);
2028                 return r;
2029         }
2030         evergreen_irq_set(rdev);
2031
2032         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2033                              RADEON_CP_PACKET2);
2034         if (r)
2035                 return r;
2036
2037         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2038         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2039                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2040         if (r)
2041                 return r;
2042
2043         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2044         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2045                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2046         if (r)
2047                 return r;
2048
2049         r = cayman_cp_load_microcode(rdev);
2050         if (r)
2051                 return r;
2052         r = cayman_cp_resume(rdev);
2053         if (r)
2054                 return r;
2055
2056         r = cayman_dma_resume(rdev);
2057         if (r)
2058                 return r;
2059
2060         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2061         if (ring->ring_size) {
2062                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2063                                      RADEON_CP_PACKET2);
2064                 if (!r)
2065                         r = uvd_v1_0_init(rdev);
2066                 if (r)
2067                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2068         }
2069
2070         r = radeon_ib_pool_init(rdev);
2071         if (r) {
2072                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2073                 return r;
2074         }
2075
2076         r = radeon_vm_manager_init(rdev);
2077         if (r) {
2078                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2079                 return r;
2080         }
2081
2082         if (ASIC_IS_DCE6(rdev)) {
2083                 r = dce6_audio_init(rdev);
2084                 if (r)
2085                         return r;
2086         } else {
2087                 r = r600_audio_init(rdev);
2088                 if (r)
2089                         return r;
2090         }
2091
2092         return 0;
2093 }
2094
2095 int cayman_resume(struct radeon_device *rdev)
2096 {
2097         int r;
2098
2099         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2100          * posting will perform necessary task to bring back GPU into good
2101          * shape.
2102          */
2103         /* post card */
2104         atom_asic_init(rdev->mode_info.atom_context);
2105
2106         /* init golden registers */
2107         ni_init_golden_registers(rdev);
2108
2109         if (rdev->pm.pm_method == PM_METHOD_DPM)
2110                 radeon_pm_resume(rdev);
2111
2112         rdev->accel_working = true;
2113         r = cayman_startup(rdev);
2114         if (r) {
2115                 DRM_ERROR("cayman startup failed on resume\n");
2116                 rdev->accel_working = false;
2117                 return r;
2118         }
2119         return r;
2120 }
2121
2122 int cayman_suspend(struct radeon_device *rdev)
2123 {
2124         radeon_pm_suspend(rdev);
2125         if (ASIC_IS_DCE6(rdev))
2126                 dce6_audio_fini(rdev);
2127         else
2128                 r600_audio_fini(rdev);
2129         radeon_vm_manager_fini(rdev);
2130         cayman_cp_enable(rdev, false);
2131         cayman_dma_stop(rdev);
2132         uvd_v1_0_fini(rdev);
2133         radeon_uvd_suspend(rdev);
2134         evergreen_irq_suspend(rdev);
2135         radeon_wb_disable(rdev);
2136         cayman_pcie_gart_disable(rdev);
2137         return 0;
2138 }
2139
2140 /* Plan is to move initialization in that function and use
2141  * helper function so that radeon_device_init pretty much
2142  * do nothing more than calling asic specific function. This
2143  * should also allow to remove a bunch of callback function
2144  * like vram_info.
2145  */
2146 int cayman_init(struct radeon_device *rdev)
2147 {
2148         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2149         int r;
2150
2151         /* Read BIOS */
2152         if (!radeon_get_bios(rdev)) {
2153                 if (ASIC_IS_AVIVO(rdev))
2154                         return -EINVAL;
2155         }
2156         /* Must be an ATOMBIOS */
2157         if (!rdev->is_atom_bios) {
2158                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2159                 return -EINVAL;
2160         }
2161         r = radeon_atombios_init(rdev);
2162         if (r)
2163                 return r;
2164
2165         /* Post card if necessary */
2166         if (!radeon_card_posted(rdev)) {
2167                 if (!rdev->bios) {
2168                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2169                         return -EINVAL;
2170                 }
2171                 DRM_INFO("GPU not posted. posting now...\n");
2172                 atom_asic_init(rdev->mode_info.atom_context);
2173         }
2174         /* init golden registers */
2175         ni_init_golden_registers(rdev);
2176         /* Initialize scratch registers */
2177         r600_scratch_init(rdev);
2178         /* Initialize surface registers */
2179         radeon_surface_init(rdev);
2180         /* Initialize clocks */
2181         radeon_get_clock_info(rdev->ddev);
2182         /* Fence driver */
2183         r = radeon_fence_driver_init(rdev);
2184         if (r)
2185                 return r;
2186         /* initialize memory controller */
2187         r = evergreen_mc_init(rdev);
2188         if (r)
2189                 return r;
2190         /* Memory manager */
2191         r = radeon_bo_init(rdev);
2192         if (r)
2193                 return r;
2194
2195         if (rdev->flags & RADEON_IS_IGP) {
2196                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2197                         r = ni_init_microcode(rdev);
2198                         if (r) {
2199                                 DRM_ERROR("Failed to load firmware!\n");
2200                                 return r;
2201                         }
2202                 }
2203         } else {
2204                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2205                         r = ni_init_microcode(rdev);
2206                         if (r) {
2207                                 DRM_ERROR("Failed to load firmware!\n");
2208                                 return r;
2209                         }
2210                 }
2211         }
2212
2213         /* Initialize power management */
2214         radeon_pm_init(rdev);
2215
2216         ring->ring_obj = NULL;
2217         r600_ring_init(rdev, ring, 1024 * 1024);
2218
2219         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2220         ring->ring_obj = NULL;
2221         r600_ring_init(rdev, ring, 64 * 1024);
2222
2223         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2224         ring->ring_obj = NULL;
2225         r600_ring_init(rdev, ring, 64 * 1024);
2226
2227         r = radeon_uvd_init(rdev);
2228         if (!r) {
2229                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2230                 ring->ring_obj = NULL;
2231                 r600_ring_init(rdev, ring, 4096);
2232         }
2233
2234         rdev->ih.ring_obj = NULL;
2235         r600_ih_ring_init(rdev, 64 * 1024);
2236
2237         r = r600_pcie_gart_init(rdev);
2238         if (r)
2239                 return r;
2240
2241         rdev->accel_working = true;
2242         r = cayman_startup(rdev);
2243         if (r) {
2244                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2245                 cayman_cp_fini(rdev);
2246                 cayman_dma_fini(rdev);
2247                 r600_irq_fini(rdev);
2248                 if (rdev->flags & RADEON_IS_IGP)
2249                         sumo_rlc_fini(rdev);
2250                 radeon_wb_fini(rdev);
2251                 radeon_ib_pool_fini(rdev);
2252                 radeon_vm_manager_fini(rdev);
2253                 radeon_irq_kms_fini(rdev);
2254                 cayman_pcie_gart_fini(rdev);
2255                 rdev->accel_working = false;
2256         }
2257
2258         /* Don't start up if the MC ucode is missing.
2259          * The default clocks and voltages before the MC ucode
2260          * is loaded are not suffient for advanced operations.
2261          *
2262          * We can skip this check for TN, because there is no MC
2263          * ucode.
2264          */
2265         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2266                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2267                 return -EINVAL;
2268         }
2269
2270         return 0;
2271 }
2272
2273 void cayman_fini(struct radeon_device *rdev)
2274 {
2275         radeon_pm_fini(rdev);
2276         cayman_cp_fini(rdev);
2277         cayman_dma_fini(rdev);
2278         r600_irq_fini(rdev);
2279         if (rdev->flags & RADEON_IS_IGP)
2280                 sumo_rlc_fini(rdev);
2281         radeon_wb_fini(rdev);
2282         radeon_vm_manager_fini(rdev);
2283         radeon_ib_pool_fini(rdev);
2284         radeon_irq_kms_fini(rdev);
2285         uvd_v1_0_fini(rdev);
2286         radeon_uvd_fini(rdev);
2287         cayman_pcie_gart_fini(rdev);
2288         r600_vram_scratch_fini(rdev);
2289         radeon_gem_fini(rdev);
2290         radeon_fence_driver_fini(rdev);
2291         radeon_bo_fini(rdev);
2292         radeon_atombios_fini(rdev);
2293         kfree(rdev->bios);
2294         rdev->bios = NULL;
2295 }
2296
2297 /*
2298  * vm
2299  */
2300 int cayman_vm_init(struct radeon_device *rdev)
2301 {
2302         /* number of VMs */
2303         rdev->vm_manager.nvm = 8;
2304         /* base offset of vram pages */
2305         if (rdev->flags & RADEON_IS_IGP) {
2306                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2307                 tmp <<= 22;
2308                 rdev->vm_manager.vram_base_offset = tmp;
2309         } else
2310                 rdev->vm_manager.vram_base_offset = 0;
2311         return 0;
2312 }
2313
2314 void cayman_vm_fini(struct radeon_device *rdev)
2315 {
2316 }
2317
2318 /**
2319  * cayman_vm_decode_fault - print human readable fault info
2320  *
2321  * @rdev: radeon_device pointer
2322  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2323  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2324  *
2325  * Print human readable fault information (cayman/TN).
2326  */
2327 void cayman_vm_decode_fault(struct radeon_device *rdev,
2328                             u32 status, u32 addr)
2329 {
2330         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2331         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2332         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2333         char *block;
2334
2335         switch (mc_id) {
2336         case 32:
2337         case 16:
2338         case 96:
2339         case 80:
2340         case 160:
2341         case 144:
2342         case 224:
2343         case 208:
2344                 block = "CB";
2345                 break;
2346         case 33:
2347         case 17:
2348         case 97:
2349         case 81:
2350         case 161:
2351         case 145:
2352         case 225:
2353         case 209:
2354                 block = "CB_FMASK";
2355                 break;
2356         case 34:
2357         case 18:
2358         case 98:
2359         case 82:
2360         case 162:
2361         case 146:
2362         case 226:
2363         case 210:
2364                 block = "CB_CMASK";
2365                 break;
2366         case 35:
2367         case 19:
2368         case 99:
2369         case 83:
2370         case 163:
2371         case 147:
2372         case 227:
2373         case 211:
2374                 block = "CB_IMMED";
2375                 break;
2376         case 36:
2377         case 20:
2378         case 100:
2379         case 84:
2380         case 164:
2381         case 148:
2382         case 228:
2383         case 212:
2384                 block = "DB";
2385                 break;
2386         case 37:
2387         case 21:
2388         case 101:
2389         case 85:
2390         case 165:
2391         case 149:
2392         case 229:
2393         case 213:
2394                 block = "DB_HTILE";
2395                 break;
2396         case 38:
2397         case 22:
2398         case 102:
2399         case 86:
2400         case 166:
2401         case 150:
2402         case 230:
2403         case 214:
2404                 block = "SX";
2405                 break;
2406         case 39:
2407         case 23:
2408         case 103:
2409         case 87:
2410         case 167:
2411         case 151:
2412         case 231:
2413         case 215:
2414                 block = "DB_STEN";
2415                 break;
2416         case 40:
2417         case 24:
2418         case 104:
2419         case 88:
2420         case 232:
2421         case 216:
2422         case 168:
2423         case 152:
2424                 block = "TC_TFETCH";
2425                 break;
2426         case 41:
2427         case 25:
2428         case 105:
2429         case 89:
2430         case 233:
2431         case 217:
2432         case 169:
2433         case 153:
2434                 block = "TC_VFETCH";
2435                 break;
2436         case 42:
2437         case 26:
2438         case 106:
2439         case 90:
2440         case 234:
2441         case 218:
2442         case 170:
2443         case 154:
2444                 block = "VC";
2445                 break;
2446         case 112:
2447                 block = "CP";
2448                 break;
2449         case 113:
2450         case 114:
2451                 block = "SH";
2452                 break;
2453         case 115:
2454                 block = "VGT";
2455                 break;
2456         case 178:
2457                 block = "IH";
2458                 break;
2459         case 51:
2460                 block = "RLC";
2461                 break;
2462         case 55:
2463                 block = "DMA";
2464                 break;
2465         case 56:
2466                 block = "HDP";
2467                 break;
2468         default:
2469                 block = "unknown";
2470                 break;
2471         }
2472
2473         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2474                protections, vmid, addr,
2475                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2476                block, mc_id);
2477 }
2478
2479 /**
2480  * cayman_vm_flush - vm flush using the CP
2481  *
2482  * @rdev: radeon_device pointer
2483  *
2484  * Update the page table base and flush the VM TLB
2485  * using the CP (cayman-si).
2486  */
2487 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2488 {
2489         struct radeon_ring *ring = &rdev->ring[ridx];
2490
2491         if (vm == NULL)
2492                 return;
2493
2494         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2495         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2496
2497         /* flush hdp cache */
2498         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2499         radeon_ring_write(ring, 0x1);
2500
2501         /* bits 0-7 are the VM contexts0-7 */
2502         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2503         radeon_ring_write(ring, 1 << vm->id);
2504
2505         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2506         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2507         radeon_ring_write(ring, 0x0);
2508 }