drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/slab.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/firmware.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/module.h>
  33 #include <drm/drmP.h>
  34 #include <drm/radeon_drm.h>
  35 #include "radeon.h"
  36 #include "radeon_asic.h"
  37 #include "radeon_mode.h"
  38 #include "r600d.h"
  39 #include "atom.h"
  40 #include "avivod.h"
  41
  42 #define PFP_UCODE_SIZE 576
  43 #define PM4_UCODE_SIZE 1792
  44 #define RLC_UCODE_SIZE 768
  45 #define R700_PFP_UCODE_SIZE 848
  46 #define R700_PM4_UCODE_SIZE 1360
  47 #define R700_RLC_UCODE_SIZE 1024
  48 #define EVERGREEN_PFP_UCODE_SIZE 1120
  49 #define EVERGREEN_PM4_UCODE_SIZE 1376
  50 #define EVERGREEN_RLC_UCODE_SIZE 768
  51 #define CAYMAN_RLC_UCODE_SIZE 1024
  52 #define ARUBA_RLC_UCODE_SIZE 1536
  53
  54 /* Firmware Names */
  55 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  56 MODULE_FIRMWARE("radeon/R600_me.bin");
  57 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  58 MODULE_FIRMWARE("radeon/RV610_me.bin");
  59 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  60 MODULE_FIRMWARE("radeon/RV630_me.bin");
  61 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  62 MODULE_FIRMWARE("radeon/RV620_me.bin");
  63 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  64 MODULE_FIRMWARE("radeon/RV635_me.bin");
  65 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  66 MODULE_FIRMWARE("radeon/RV670_me.bin");
  67 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  68 MODULE_FIRMWARE("radeon/RS780_me.bin");
  69 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  70 MODULE_FIRMWARE("radeon/RV770_me.bin");
  71 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  72 MODULE_FIRMWARE("radeon/RV730_me.bin");
  73 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  74 MODULE_FIRMWARE("radeon/RV710_me.bin");
  75 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  76 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  77 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  78 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  79 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  80 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  81 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  82 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  83 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  84 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  85 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  86 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  87 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  88 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  89 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
  90 MODULE_FIRMWARE("radeon/PALM_me.bin");
  91 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
  92 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
  93 MODULE_FIRMWARE("radeon/SUMO_me.bin");
  94 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
  95 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
  96
  97 static const u32 crtc_offsets[2] =
  98 {
  99         0,
 100         AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
 101 };
 102
 103 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
 104
 105 /* r600,rv610,rv630,rv620,rv635,rv670 */
 106 int r600_mc_wait_for_idle(struct radeon_device *rdev);
 107 static void r600_gpu_init(struct radeon_device *rdev);
 108 void r600_fini(struct radeon_device *rdev);
 109 void r600_irq_disable(struct radeon_device *rdev);
 110 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 111
 112 /* get temperature in millidegrees */
 113 int rv6xx_get_temp(struct radeon_device *rdev)
 114 {
 115         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
 116                 ASIC_T_SHIFT;
 117         int actual_temp = temp & 0xff;
 118
 119         if (temp & 0x100)
 120                 actual_temp -= 256;
 121
 122         return actual_temp * 1000;
 123 }
 124
 125 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
 126 {
 127         int i;
 128
 129         rdev->pm.dynpm_can_upclock = true;
 130         rdev->pm.dynpm_can_downclock = true;
 131
 132         /* power state array is low to high, default is first */
 133         if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
 134                 int min_power_state_index = 0;
 135
 136                 if (rdev->pm.num_power_states > 2)
 137                         min_power_state_index = 1;
 138
 139                 switch (rdev->pm.dynpm_planned_action) {
 140                 case DYNPM_ACTION_MINIMUM:
 141                         rdev->pm.requested_power_state_index = min_power_state_index;
 142                         rdev->pm.requested_clock_mode_index = 0;
 143                         rdev->pm.dynpm_can_downclock = false;
 144                         break;
 145                 case DYNPM_ACTION_DOWNCLOCK:
 146                         if (rdev->pm.current_power_state_index == min_power_state_index) {
 147                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 148                                 rdev->pm.dynpm_can_downclock = false;
 149                         } else {
 150                                 if (rdev->pm.active_crtc_count > 1) {
 151                                         for (i = 0; i < rdev->pm.num_power_states; i++) {
 152                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 153                                                         continue;
 154                                                 else if (i >= rdev->pm.current_power_state_index) {
 155                                                         rdev->pm.requested_power_state_index =
 156                                                                 rdev->pm.current_power_state_index;
 157                                                         break;
 158                                                 } else {
 159                                                         rdev->pm.requested_power_state_index = i;
 160                                                         break;
 161                                                 }
 162                                         }
 163                                 } else {
 164                                         if (rdev->pm.current_power_state_index == 0)
 165                                                 rdev->pm.requested_power_state_index =
 166                                                         rdev->pm.num_power_states - 1;
 167                                         else
 168                                                 rdev->pm.requested_power_state_index =
 169                                                         rdev->pm.current_power_state_index - 1;
 170                                 }
 171                         }
 172                         rdev->pm.requested_clock_mode_index = 0;
 173                         /* don't use the power state if crtcs are active and no display flag is set */
 174                         if ((rdev->pm.active_crtc_count > 0) &&
 175                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 176                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 177                              RADEON_PM_MODE_NO_DISPLAY)) {
 178                                 rdev->pm.requested_power_state_index++;
 179                         }
 180                         break;
 181                 case DYNPM_ACTION_UPCLOCK:
 182                         if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
 183                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 184                                 rdev->pm.dynpm_can_upclock = false;
 185                         } else {
 186                                 if (rdev->pm.active_crtc_count > 1) {
 187                                         for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
 188                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 189                                                         continue;
 190                                                 else if (i <= rdev->pm.current_power_state_index) {
 191                                                         rdev->pm.requested_power_state_index =
 192                                                                 rdev->pm.current_power_state_index;
 193                                                         break;
 194                                                 } else {
 195                                                         rdev->pm.requested_power_state_index = i;
 196                                                         break;
 197                                                 }
 198                                         }
 199                                 } else
 200                                         rdev->pm.requested_power_state_index =
 201                                                 rdev->pm.current_power_state_index + 1;
 202                         }
 203                         rdev->pm.requested_clock_mode_index = 0;
 204                         break;
 205                 case DYNPM_ACTION_DEFAULT:
 206                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 207                         rdev->pm.requested_clock_mode_index = 0;
 208                         rdev->pm.dynpm_can_upclock = false;
 209                         break;
 210                 case DYNPM_ACTION_NONE:
 211                 default:
 212                         DRM_ERROR("Requested mode for not defined action\n");
 213                         return;
 214                 }
 215         } else {
 216                 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
 217                 /* for now just select the first power state and switch between clock modes */
 218                 /* power state array is low to high, default is first (0) */
 219                 if (rdev->pm.active_crtc_count > 1) {
 220                         rdev->pm.requested_power_state_index = -1;
 221                         /* start at 1 as we don't want the default mode */
 222                         for (i = 1; i < rdev->pm.num_power_states; i++) {
 223                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 224                                         continue;
 225                                 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
 226                                          (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
 227                                         rdev->pm.requested_power_state_index = i;
 228                                         break;
 229                                 }
 230                         }
 231                         /* if nothing selected, grab the default state. */
 232                         if (rdev->pm.requested_power_state_index == -1)
 233                                 rdev->pm.requested_power_state_index = 0;
 234                 } else
 235                         rdev->pm.requested_power_state_index = 1;
 236
 237                 switch (rdev->pm.dynpm_planned_action) {
 238                 case DYNPM_ACTION_MINIMUM:
 239                         rdev->pm.requested_clock_mode_index = 0;
 240                         rdev->pm.dynpm_can_downclock = false;
 241                         break;
 242                 case DYNPM_ACTION_DOWNCLOCK:
 243                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 244                                 if (rdev->pm.current_clock_mode_index == 0) {
 245                                         rdev->pm.requested_clock_mode_index = 0;
 246                                         rdev->pm.dynpm_can_downclock = false;
 247                                 } else
 248                                         rdev->pm.requested_clock_mode_index =
 249                                                 rdev->pm.current_clock_mode_index - 1;
 250                         } else {
 251                                 rdev->pm.requested_clock_mode_index = 0;
 252                                 rdev->pm.dynpm_can_downclock = false;
 253                         }
 254                         /* don't use the power state if crtcs are active and no display flag is set */
 255                         if ((rdev->pm.active_crtc_count > 0) &&
 256                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 257                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 258                              RADEON_PM_MODE_NO_DISPLAY)) {
 259                                 rdev->pm.requested_clock_mode_index++;
 260                         }
 261                         break;
 262                 case DYNPM_ACTION_UPCLOCK:
 263                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 264                                 if (rdev->pm.current_clock_mode_index ==
 265                                     (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
 266                                         rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
 267                                         rdev->pm.dynpm_can_upclock = false;
 268                                 } else
 269                                         rdev->pm.requested_clock_mode_index =
 270                                                 rdev->pm.current_clock_mode_index + 1;
 271                         } else {
 272                                 rdev->pm.requested_clock_mode_index =
 273                                         rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
 274                                 rdev->pm.dynpm_can_upclock = false;
 275                         }
 276                         break;
 277                 case DYNPM_ACTION_DEFAULT:
 278                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 279                         rdev->pm.requested_clock_mode_index = 0;
 280                         rdev->pm.dynpm_can_upclock = false;
 281                         break;
 282                 case DYNPM_ACTION_NONE:
 283                 default:
 284                         DRM_ERROR("Requested mode for not defined action\n");
 285                         return;
 286                 }
 287         }
 288
 289         DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
 290                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 291                   clock_info[rdev->pm.requested_clock_mode_index].sclk,
 292                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 293                   clock_info[rdev->pm.requested_clock_mode_index].mclk,
 294                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 295                   pcie_lanes);
 296 }
 297
 298 void rs780_pm_init_profile(struct radeon_device *rdev)
 299 {
 300         if (rdev->pm.num_power_states == 2) {
 301                 /* default */
 302                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 303                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 304                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 305                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 306                 /* low sh */
 307                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
 308                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
 309                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 310                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 311                 /* mid sh */
 312                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
 313                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
 314                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 315                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 316                 /* high sh */
 317                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
 318                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 319                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 320                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 321                 /* low mh */
 322                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
 323                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 324                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 325                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 326                 /* mid mh */
 327                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
 328                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 329                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 330                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 331                 /* high mh */
 332                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
 333                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
 334                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 335                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 336         } else if (rdev->pm.num_power_states == 3) {
 337                 /* default */
 338                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 339                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 340                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 341                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 342                 /* low sh */
 343                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 344                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 345                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 346                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 347                 /* mid sh */
 348                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 349                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 350                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 351                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 352                 /* high sh */
 353                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 354                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
 355                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 356                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 357                 /* low mh */
 358                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
 359                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
 360                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 361                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 362                 /* mid mh */
 363                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
 364                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
 365                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 366                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 367                 /* high mh */
 368                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
 369                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 370                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 371                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 372         } else {
 373                 /* default */
 374                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 375                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 376                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 377                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 378                 /* low sh */
 379                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
 380                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
 381                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 382                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 383                 /* mid sh */
 384                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
 385                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
 386                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 387                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 388                 /* high sh */
 389                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
 390                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
 391                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 392                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 393                 /* low mh */
 394                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 395                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 396                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 397                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 398                 /* mid mh */
 399                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 400                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 401                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 402                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 403                 /* high mh */
 404                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 405                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
 406                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 407                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 408         }
 409 }
 410
 411 void r600_pm_init_profile(struct radeon_device *rdev)
 412 {
 413         int idx;
 414
 415         if (rdev->family == CHIP_R600) {
 416                 /* XXX */
 417                 /* default */
 418                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 419                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 420                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 421                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 422                 /* low sh */
 423                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 424                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 425                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 426                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 427                 /* mid sh */
 428                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 429                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 430                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 431                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 432                 /* high sh */
 433                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 434                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 435                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 436                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 437                 /* low mh */
 438                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 439                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 440                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 441                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 442                 /* mid mh */
 443                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 444                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 445                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 446                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 447                 /* high mh */
 448                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 449                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 450                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 451                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 452         } else {
 453                 if (rdev->pm.num_power_states < 4) {
 454                         /* default */
 455                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 456                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 457                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 458                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 459                         /* low sh */
 460                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 461                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 462                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 463                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 464                         /* mid sh */
 465                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 466                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 467                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 468                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 469                         /* high sh */
 470                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 471                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 472                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 473                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 474                         /* low mh */
 475                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 476                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
 477                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 478                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 479                         /* low mh */
 480                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 481                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
 482                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 483                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 484                         /* high mh */
 485                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 486                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 487                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 488                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 489                 } else {
 490                         /* default */
 491                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 492                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 493                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 494                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 495                         /* low sh */
 496                         if (rdev->flags & RADEON_IS_MOBILITY)
 497                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
 498                         else
 499                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 500                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
 501                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
 502                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 503                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 504                         /* mid sh */
 505                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
 506                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
 507                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 508                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 509                         /* high sh */
 510                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 511                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
 512                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
 513                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 514                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 515                         /* low mh */
 516                         if (rdev->flags & RADEON_IS_MOBILITY)
 517                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
 518                         else
 519                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 520                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
 521                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
 522                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 523                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 524                         /* mid mh */
 525                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
 526                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
 527                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 528                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 529                         /* high mh */
 530                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 531                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
 532                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
 533                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 534                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 535                 }
 536         }
 537 }
 538
 539 void r600_pm_misc(struct radeon_device *rdev)
 540 {
 541         int req_ps_idx = rdev->pm.requested_power_state_index;
 542         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 543         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 544         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 545
 546         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 547                 /* 0xff01 is a flag rather then an actual voltage */
 548                 if (voltage->voltage == 0xff01)
 549                         return;
 550                 if (voltage->voltage != rdev->pm.current_vddc) {
 551                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 552                         rdev->pm.current_vddc = voltage->voltage;
 553                         DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
 554                 }
 555         }
 556 }
 557
 558 bool r600_gui_idle(struct radeon_device *rdev)
 559 {
 560         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 561                 return false;
 562         else
 563                 return true;
 564 }
 565
 566 /* hpd for digital panel detect/disconnect */
 567 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 568 {
 569         bool connected = false;
 570
 571         if (ASIC_IS_DCE3(rdev)) {
 572                 switch (hpd) {
 573                 case RADEON_HPD_1:
 574                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 575                                 connected = true;
 576                         break;
 577                 case RADEON_HPD_2:
 578                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 579                                 connected = true;
 580                         break;
 581                 case RADEON_HPD_3:
 582                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 583                                 connected = true;
 584                         break;
 585                 case RADEON_HPD_4:
 586                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 587                                 connected = true;
 588                         break;
 589                         /* DCE 3.2 */
 590                 case RADEON_HPD_5:
 591                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 592                                 connected = true;
 593                         break;
 594                 case RADEON_HPD_6:
 595                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 596                                 connected = true;
 597                         break;
 598                 default:
 599                         break;
 600                 }
 601         } else {
 602                 switch (hpd) {
 603                 case RADEON_HPD_1:
 604                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 605                                 connected = true;
 606                         break;
 607                 case RADEON_HPD_2:
 608                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 609                                 connected = true;
 610                         break;
 611                 case RADEON_HPD_3:
 612                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 613                                 connected = true;
 614                         break;
 615                 default:
 616                         break;
 617                 }
 618         }
 619         return connected;
 620 }
 621
 622 void r600_hpd_set_polarity(struct radeon_device *rdev,
 623                            enum radeon_hpd_id hpd)
 624 {
 625         u32 tmp;
 626         bool connected = r600_hpd_sense(rdev, hpd);
 627
 628         if (ASIC_IS_DCE3(rdev)) {
 629                 switch (hpd) {
 630                 case RADEON_HPD_1:
 631                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 632                         if (connected)
 633                                 tmp &= ~DC_HPDx_INT_POLARITY;
 634                         else
 635                                 tmp |= DC_HPDx_INT_POLARITY;
 636                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 637                         break;
 638                 case RADEON_HPD_2:
 639                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 640                         if (connected)
 641                                 tmp &= ~DC_HPDx_INT_POLARITY;
 642                         else
 643                                 tmp |= DC_HPDx_INT_POLARITY;
 644                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 645                         break;
 646                 case RADEON_HPD_3:
 647                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 648                         if (connected)
 649                                 tmp &= ~DC_HPDx_INT_POLARITY;
 650                         else
 651                                 tmp |= DC_HPDx_INT_POLARITY;
 652                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 653                         break;
 654                 case RADEON_HPD_4:
 655                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 656                         if (connected)
 657                                 tmp &= ~DC_HPDx_INT_POLARITY;
 658                         else
 659                                 tmp |= DC_HPDx_INT_POLARITY;
 660                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 661                         break;
 662                 case RADEON_HPD_5:
 663                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 664                         if (connected)
 665                                 tmp &= ~DC_HPDx_INT_POLARITY;
 666                         else
 667                                 tmp |= DC_HPDx_INT_POLARITY;
 668                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 669                         break;
 670                         /* DCE 3.2 */
 671                 case RADEON_HPD_6:
 672                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 673                         if (connected)
 674                                 tmp &= ~DC_HPDx_INT_POLARITY;
 675                         else
 676                                 tmp |= DC_HPDx_INT_POLARITY;
 677                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 678                         break;
 679                 default:
 680                         break;
 681                 }
 682         } else {
 683                 switch (hpd) {
 684                 case RADEON_HPD_1:
 685                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 686                         if (connected)
 687                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 688                         else
 689                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 690                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 691                         break;
 692                 case RADEON_HPD_2:
 693                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 694                         if (connected)
 695                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 696                         else
 697                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 698                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 699                         break;
 700                 case RADEON_HPD_3:
 701                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 702                         if (connected)
 703                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 704                         else
 705                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 706                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 707                         break;
 708                 default:
 709                         break;
 710                 }
 711         }
 712 }
 713
 714 void r600_hpd_init(struct radeon_device *rdev)
 715 {
 716         struct drm_device *dev = rdev->ddev;
 717         struct drm_connector *connector;
 718         unsigned enable = 0;
 719
 720         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 721                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 722
 723                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 724                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
 725                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
 726                          * aux dp channel on imac and help (but not completely fix)
 727                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 728                          */
 729                         continue;
 730                 }
 731                 if (ASIC_IS_DCE3(rdev)) {
 732                         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 733                         if (ASIC_IS_DCE32(rdev))
 734                                 tmp |= DC_HPDx_EN;
 735
 736                         switch (radeon_connector->hpd.hpd) {
 737                         case RADEON_HPD_1:
 738                                 WREG32(DC_HPD1_CONTROL, tmp);
 739                                 break;
 740                         case RADEON_HPD_2:
 741                                 WREG32(DC_HPD2_CONTROL, tmp);
 742                                 break;
 743                         case RADEON_HPD_3:
 744                                 WREG32(DC_HPD3_CONTROL, tmp);
 745                                 break;
 746                         case RADEON_HPD_4:
 747                                 WREG32(DC_HPD4_CONTROL, tmp);
 748                                 break;
 749                                 /* DCE 3.2 */
 750                         case RADEON_HPD_5:
 751                                 WREG32(DC_HPD5_CONTROL, tmp);
 752                                 break;
 753                         case RADEON_HPD_6:
 754                                 WREG32(DC_HPD6_CONTROL, tmp);
 755                                 break;
 756                         default:
 757                                 break;
 758                         }
 759                 } else {
 760                         switch (radeon_connector->hpd.hpd) {
 761                         case RADEON_HPD_1:
 762                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 763                                 break;
 764                         case RADEON_HPD_2:
 765                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 766                                 break;
 767                         case RADEON_HPD_3:
 768                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 769                                 break;
 770                         default:
 771                                 break;
 772                         }
 773                 }
 774                 enable |= 1 << radeon_connector->hpd.hpd;
 775                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
 776         }
 777         radeon_irq_kms_enable_hpd(rdev, enable);
 778 }
 779
 780 void r600_hpd_fini(struct radeon_device *rdev)
 781 {
 782         struct drm_device *dev = rdev->ddev;
 783         struct drm_connector *connector;
 784         unsigned disable = 0;
 785
 786         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 787                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 788                 if (ASIC_IS_DCE3(rdev)) {
 789                         switch (radeon_connector->hpd.hpd) {
 790                         case RADEON_HPD_1:
 791                                 WREG32(DC_HPD1_CONTROL, 0);
 792                                 break;
 793                         case RADEON_HPD_2:
 794                                 WREG32(DC_HPD2_CONTROL, 0);
 795                                 break;
 796                         case RADEON_HPD_3:
 797                                 WREG32(DC_HPD3_CONTROL, 0);
 798                                 break;
 799                         case RADEON_HPD_4:
 800                                 WREG32(DC_HPD4_CONTROL, 0);
 801                                 break;
 802                                 /* DCE 3.2 */
 803                         case RADEON_HPD_5:
 804                                 WREG32(DC_HPD5_CONTROL, 0);
 805                                 break;
 806                         case RADEON_HPD_6:
 807                                 WREG32(DC_HPD6_CONTROL, 0);
 808                                 break;
 809                         default:
 810                                 break;
 811                         }
 812                 } else {
 813                         switch (radeon_connector->hpd.hpd) {
 814                         case RADEON_HPD_1:
 815                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 816                                 break;
 817                         case RADEON_HPD_2:
 818                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 819                                 break;
 820                         case RADEON_HPD_3:
 821                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 822                                 break;
 823                         default:
 824                                 break;
 825                         }
 826                 }
 827                 disable |= 1 << radeon_connector->hpd.hpd;
 828         }
 829         radeon_irq_kms_disable_hpd(rdev, disable);
 830 }
 831
 832 /*
 833  * R600 PCIE GART
 834  */
 835 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 836 {
 837         unsigned i;
 838         u32 tmp;
 839
 840         /* flush hdp cache so updates hit vram */
 841         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
 842             !(rdev->flags & RADEON_IS_AGP)) {
 843                 void __iomem *ptr = (void *)rdev->gart.ptr;
 844                 u32 tmp;
 845
 846                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
 847                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
 848                  * This seems to cause problems on some AGP cards. Just use the old
 849                  * method for them.
 850                  */
 851                 WREG32(HDP_DEBUG1, 0);
 852                 tmp = readl((void __iomem *)ptr);
 853         } else
 854                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 855
 856         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 857         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 858         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 859         for (i = 0; i < rdev->usec_timeout; i++) {
 860                 /* read MC_STATUS */
 861                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 862                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 863                 if (tmp == 2) {
 864                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 865                         return;
 866                 }
 867                 if (tmp) {
 868                         return;
 869                 }
 870                 udelay(1);
 871         }
 872 }
 873
 874 int r600_pcie_gart_init(struct radeon_device *rdev)
 875 {
 876         int r;
 877
 878         if (rdev->gart.robj) {
 879                 WARN(1, "R600 PCIE GART already initialized\n");
 880                 return 0;
 881         }
 882         /* Initialize common gart structure */
 883         r = radeon_gart_init(rdev);
 884         if (r)
 885                 return r;
 886         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 887         return radeon_gart_table_vram_alloc(rdev);
 888 }
 889
 890 static int r600_pcie_gart_enable(struct radeon_device *rdev)
 891 {
 892         u32 tmp;
 893         int r, i;
 894
 895         if (rdev->gart.robj == NULL) {
 896                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 897                 return -EINVAL;
 898         }
 899         r = radeon_gart_table_vram_pin(rdev);
 900         if (r)
 901                 return r;
 902         radeon_gart_restore(rdev);
 903
 904         /* Setup L2 cache */
 905         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 906                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 907                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 908         WREG32(VM_L2_CNTL2, 0);
 909         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 910         /* Setup TLB control */
 911         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 912                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 913                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 914                 ENABLE_WAIT_L2_QUERY;
 915         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 916         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 917         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 918         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 919         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 920         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 921         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 922         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 923         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 924         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 925         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 926         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 927         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 928         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 929         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 930         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 931         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 932         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 933                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 934         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 935                         (u32)(rdev->dummy_page.addr >> 12));
 936         for (i = 1; i < 7; i++)
 937                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 938
 939         r600_pcie_gart_tlb_flush(rdev);
 940         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 941                  (unsigned)(rdev->mc.gtt_size >> 20),
 942                  (unsigned long long)rdev->gart.table_addr);
 943         rdev->gart.ready = true;
 944         return 0;
 945 }
 946
 947 static void r600_pcie_gart_disable(struct radeon_device *rdev)
 948 {
 949         u32 tmp;
 950         int i;
 951
 952         /* Disable all tables */
 953         for (i = 0; i < 7; i++)
 954                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 955
 956         /* Disable L2 cache */
 957         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 958                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 959         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 960         /* Setup L1 TLB control */
 961         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 962                 ENABLE_WAIT_L2_QUERY;
 963         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 964         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 965         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 966         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 967         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 968         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 969         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 970         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 971         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 972         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 973         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 974         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 975         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 976         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 977         radeon_gart_table_vram_unpin(rdev);
 978 }
 979
 980 static void r600_pcie_gart_fini(struct radeon_device *rdev)
 981 {
 982         radeon_gart_fini(rdev);
 983         r600_pcie_gart_disable(rdev);
 984         radeon_gart_table_vram_free(rdev);
 985 }
 986
 987 static void r600_agp_enable(struct radeon_device *rdev)
 988 {
 989         u32 tmp;
 990         int i;
 991
 992         /* Setup L2 cache */
 993         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 994                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 995                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 996         WREG32(VM_L2_CNTL2, 0);
 997         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 998         /* Setup TLB control */
 999         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1000                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1001                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1002                 ENABLE_WAIT_L2_QUERY;
1003         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1004         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1005         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1006         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1007         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1008         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1009         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1010         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1011         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1012         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1013         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1014         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1015         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1016         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1017         for (i = 0; i < 7; i++)
1018                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1019 }
1020
1021 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1022 {
1023         unsigned i;
1024         u32 tmp;
1025
1026         for (i = 0; i < rdev->usec_timeout; i++) {
1027                 /* read MC_STATUS */
1028                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1029                 if (!tmp)
1030                         return 0;
1031                 udelay(1);
1032         }
1033         return -1;
1034 }
1035
1036 static void r600_mc_program(struct radeon_device *rdev)
1037 {
1038         struct rv515_mc_save save;
1039         u32 tmp;
1040         int i, j;
1041
1042         /* Initialize HDP */
1043         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1044                 WREG32((0x2c14 + j), 0x00000000);
1045                 WREG32((0x2c18 + j), 0x00000000);
1046                 WREG32((0x2c1c + j), 0x00000000);
1047                 WREG32((0x2c20 + j), 0x00000000);
1048                 WREG32((0x2c24 + j), 0x00000000);
1049         }
1050         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1051
1052         rv515_mc_stop(rdev, &save);
1053         if (r600_mc_wait_for_idle(rdev)) {
1054                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1055         }
1056         /* Lockout access through VGA aperture (doesn't exist before R600) */
1057         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1058         /* Update configuration */
1059         if (rdev->flags & RADEON_IS_AGP) {
1060                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1061                         /* VRAM before AGP */
1062                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1063                                 rdev->mc.vram_start >> 12);
1064                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1065                                 rdev->mc.gtt_end >> 12);
1066                 } else {
1067                         /* VRAM after AGP */
1068                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1069                                 rdev->mc.gtt_start >> 12);
1070                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1071                                 rdev->mc.vram_end >> 12);
1072                 }
1073         } else {
1074                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1075                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1076         }
1077         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1078         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1079         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1080         WREG32(MC_VM_FB_LOCATION, tmp);
1081         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1082         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1083         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1084         if (rdev->flags & RADEON_IS_AGP) {
1085                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1086                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1087                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1088         } else {
1089                 WREG32(MC_VM_AGP_BASE, 0);
1090                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1091                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1092         }
1093         if (r600_mc_wait_for_idle(rdev)) {
1094                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1095         }
1096         rv515_mc_resume(rdev, &save);
1097         /* we need to own VRAM, so turn off the VGA renderer here
1098          * to stop it overwriting our objects */
1099         rv515_vga_render_disable(rdev);
1100 }
1101
1102 /**
1103  * r600_vram_gtt_location - try to find VRAM & GTT location
1104  * @rdev: radeon device structure holding all necessary informations
1105  * @mc: memory controller structure holding memory informations
1106  *
1107  * Function will place try to place VRAM at same place as in CPU (PCI)
1108  * address space as some GPU seems to have issue when we reprogram at
1109  * different address space.
1110  *
1111  * If there is not enough space to fit the unvisible VRAM after the
1112  * aperture then we limit the VRAM size to the aperture.
1113  *
1114  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1115  * them to be in one from GPU point of view so that we can program GPU to
1116  * catch access outside them (weird GPU policy see ??).
1117  *
1118  * This function will never fails, worst case are limiting VRAM or GTT.
1119  *
1120  * Note: GTT start, end, size should be initialized before calling this
1121  * function on AGP platform.
1122  */
1123 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1124 {
1125         u64 size_bf, size_af;
1126
1127         if (mc->mc_vram_size > 0xE0000000) {
1128                 /* leave room for at least 512M GTT */
1129                 dev_warn(rdev->dev, "limiting VRAM\n");
1130                 mc->real_vram_size = 0xE0000000;
1131                 mc->mc_vram_size = 0xE0000000;
1132         }
1133         if (rdev->flags & RADEON_IS_AGP) {
1134                 size_bf = mc->gtt_start;
1135                 size_af = 0xFFFFFFFF - mc->gtt_end;
1136                 if (size_bf > size_af) {
1137                         if (mc->mc_vram_size > size_bf) {
1138                                 dev_warn(rdev->dev, "limiting VRAM\n");
1139                                 mc->real_vram_size = size_bf;
1140                                 mc->mc_vram_size = size_bf;
1141                         }
1142                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1143                 } else {
1144                         if (mc->mc_vram_size > size_af) {
1145                                 dev_warn(rdev->dev, "limiting VRAM\n");
1146                                 mc->real_vram_size = size_af;
1147                                 mc->mc_vram_size = size_af;
1148                         }
1149                         mc->vram_start = mc->gtt_end + 1;
1150                 }
1151                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1152                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1153                                 mc->mc_vram_size >> 20, mc->vram_start,
1154                                 mc->vram_end, mc->real_vram_size >> 20);
1155         } else {
1156                 u64 base = 0;
1157                 if (rdev->flags & RADEON_IS_IGP) {
1158                         base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1159                         base <<= 24;
1160                 }
1161                 radeon_vram_location(rdev, &rdev->mc, base);
1162                 rdev->mc.gtt_base_align = 0;
1163                 radeon_gtt_location(rdev, mc);
1164         }
1165 }
1166
1167 static int r600_mc_init(struct radeon_device *rdev)
1168 {
1169         u32 tmp;
1170         int chansize, numchan;
1171
1172         /* Get VRAM informations */
1173         rdev->mc.vram_is_ddr = true;
1174         tmp = RREG32(RAMCFG);
1175         if (tmp & CHANSIZE_OVERRIDE) {
1176                 chansize = 16;
1177         } else if (tmp & CHANSIZE_MASK) {
1178                 chansize = 64;
1179         } else {
1180                 chansize = 32;
1181         }
1182         tmp = RREG32(CHMAP);
1183         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1184         case 0:
1185         default:
1186                 numchan = 1;
1187                 break;
1188         case 1:
1189                 numchan = 2;
1190                 break;
1191         case 2:
1192                 numchan = 4;
1193                 break;
1194         case 3:
1195                 numchan = 8;
1196                 break;
1197         }
1198         rdev->mc.vram_width = numchan * chansize;
1199         /* Could aper size report 0 ? */
1200         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1201         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1202         /* Setup GPU memory space */
1203         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1204         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1205         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1206         r600_vram_gtt_location(rdev, &rdev->mc);
1207
1208         if (rdev->flags & RADEON_IS_IGP) {
1209                 rs690_pm_info(rdev);
1210                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1211         }
1212         radeon_update_bandwidth_info(rdev);
1213         return 0;
1214 }
1215
1216 int r600_vram_scratch_init(struct radeon_device *rdev)
1217 {
1218         int r;
1219
1220         if (rdev->vram_scratch.robj == NULL) {
1221                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1222                                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1223                                      NULL, &rdev->vram_scratch.robj);
1224                 if (r) {
1225                         return r;
1226                 }
1227         }
1228
1229         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1230         if (unlikely(r != 0))
1231                 return r;
1232         r = radeon_bo_pin(rdev->vram_scratch.robj,
1233                           RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1234         if (r) {
1235                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1236                 return r;
1237         }
1238         r = radeon_bo_kmap(rdev->vram_scratch.robj,
1239                                 (void **)&rdev->vram_scratch.ptr);
1240         if (r)
1241                 radeon_bo_unpin(rdev->vram_scratch.robj);
1242         radeon_bo_unreserve(rdev->vram_scratch.robj);
1243
1244         return r;
1245 }
1246
1247 void r600_vram_scratch_fini(struct radeon_device *rdev)
1248 {
1249         int r;
1250
1251         if (rdev->vram_scratch.robj == NULL) {
1252                 return;
1253         }
1254         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1255         if (likely(r == 0)) {
1256                 radeon_bo_kunmap(rdev->vram_scratch.robj);
1257                 radeon_bo_unpin(rdev->vram_scratch.robj);
1258                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1259         }
1260         radeon_bo_unref(&rdev->vram_scratch.robj);
1261 }
1262
1263 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1264 {
1265         u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1266
1267         if (hung)
1268                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1269         else
1270                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1271
1272         WREG32(R600_BIOS_3_SCRATCH, tmp);
1273 }
1274
1275 static void r600_print_gpu_status_regs(struct radeon_device *rdev)
1276 {
1277         dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",
1278                  RREG32(R_008010_GRBM_STATUS));
1279         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",
1280                  RREG32(R_008014_GRBM_STATUS2));
1281         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",
1282                  RREG32(R_000E50_SRBM_STATUS));
1283         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1284                  RREG32(CP_STALLED_STAT1));
1285         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1286                  RREG32(CP_STALLED_STAT2));
1287         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1288                  RREG32(CP_BUSY_STAT));
1289         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1290                  RREG32(CP_STAT));
1291         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1292                 RREG32(DMA_STATUS_REG));
1293 }
1294
1295 static bool r600_is_display_hung(struct radeon_device *rdev)
1296 {
1297         u32 crtc_hung = 0;
1298         u32 crtc_status[2];
1299         u32 i, j, tmp;
1300
1301         for (i = 0; i < rdev->num_crtc; i++) {
1302                 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1303                         crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1304                         crtc_hung |= (1 << i);
1305                 }
1306         }
1307
1308         for (j = 0; j < 10; j++) {
1309                 for (i = 0; i < rdev->num_crtc; i++) {
1310                         if (crtc_hung & (1 << i)) {
1311                                 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1312                                 if (tmp != crtc_status[i])
1313                                         crtc_hung &= ~(1 << i);
1314                         }
1315                 }
1316                 if (crtc_hung == 0)
1317                         return false;
1318                 udelay(100);
1319         }
1320
1321         return true;
1322 }
1323
1324 static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1325 {
1326         u32 reset_mask = 0;
1327         u32 tmp;
1328
1329         /* GRBM_STATUS */
1330         tmp = RREG32(R_008010_GRBM_STATUS);
1331         if (rdev->family >= CHIP_RV770) {
1332                 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1333                     G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1334                     G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1335                     G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1336                     G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1337                         reset_mask |= RADEON_RESET_GFX;
1338         } else {
1339                 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1340                     G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1341                     G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1342                     G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1343                     G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1344                         reset_mask |= RADEON_RESET_GFX;
1345         }
1346
1347         if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1348             G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1349                 reset_mask |= RADEON_RESET_CP;
1350
1351         if (G_008010_GRBM_EE_BUSY(tmp))
1352                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1353
1354         /* DMA_STATUS_REG */
1355         tmp = RREG32(DMA_STATUS_REG);
1356         if (!(tmp & DMA_IDLE))
1357                 reset_mask |= RADEON_RESET_DMA;
1358
1359         /* SRBM_STATUS */
1360         tmp = RREG32(R_000E50_SRBM_STATUS);
1361         if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1362                 reset_mask |= RADEON_RESET_RLC;
1363
1364         if (G_000E50_IH_BUSY(tmp))
1365                 reset_mask |= RADEON_RESET_IH;
1366
1367         if (G_000E50_SEM_BUSY(tmp))
1368                 reset_mask |= RADEON_RESET_SEM;
1369
1370         if (G_000E50_GRBM_RQ_PENDING(tmp))
1371                 reset_mask |= RADEON_RESET_GRBM;
1372
1373         if (G_000E50_VMC_BUSY(tmp))
1374                 reset_mask |= RADEON_RESET_VMC;
1375
1376         if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1377             G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1378             G_000E50_MCDW_BUSY(tmp))
1379                 reset_mask |= RADEON_RESET_MC;
1380
1381         if (r600_is_display_hung(rdev))
1382                 reset_mask |= RADEON_RESET_DISPLAY;
1383
1384         return reset_mask;
1385 }
1386
1387 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1388 {
1389         struct rv515_mc_save save;
1390         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1391         u32 tmp;
1392
1393         if (reset_mask == 0)
1394                 return;
1395
1396         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1397
1398         r600_print_gpu_status_regs(rdev);
1399
1400         /* Disable CP parsing/prefetching */
1401         if (rdev->family >= CHIP_RV770)
1402                 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1403         else
1404                 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1405
1406         /* disable the RLC */
1407         WREG32(RLC_CNTL, 0);
1408
1409         if (reset_mask & RADEON_RESET_DMA) {
1410                 /* Disable DMA */
1411                 tmp = RREG32(DMA_RB_CNTL);
1412                 tmp &= ~DMA_RB_ENABLE;
1413                 WREG32(DMA_RB_CNTL, tmp);
1414         }
1415
1416         mdelay(50);
1417
1418         rv515_mc_stop(rdev, &save);
1419         if (r600_mc_wait_for_idle(rdev)) {
1420                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1421         }
1422
1423         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1424                 if (rdev->family >= CHIP_RV770)
1425                         grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1426                                 S_008020_SOFT_RESET_CB(1) |
1427                                 S_008020_SOFT_RESET_PA(1) |
1428                                 S_008020_SOFT_RESET_SC(1) |
1429                                 S_008020_SOFT_RESET_SPI(1) |
1430                                 S_008020_SOFT_RESET_SX(1) |
1431                                 S_008020_SOFT_RESET_SH(1) |
1432                                 S_008020_SOFT_RESET_TC(1) |
1433                                 S_008020_SOFT_RESET_TA(1) |
1434                                 S_008020_SOFT_RESET_VC(1) |
1435                                 S_008020_SOFT_RESET_VGT(1);
1436                 else
1437                         grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1438                                 S_008020_SOFT_RESET_DB(1) |
1439                                 S_008020_SOFT_RESET_CB(1) |
1440                                 S_008020_SOFT_RESET_PA(1) |
1441                                 S_008020_SOFT_RESET_SC(1) |
1442                                 S_008020_SOFT_RESET_SMX(1) |
1443                                 S_008020_SOFT_RESET_SPI(1) |
1444                                 S_008020_SOFT_RESET_SX(1) |
1445                                 S_008020_SOFT_RESET_SH(1) |
1446                                 S_008020_SOFT_RESET_TC(1) |
1447                                 S_008020_SOFT_RESET_TA(1) |
1448                                 S_008020_SOFT_RESET_VC(1) |
1449                                 S_008020_SOFT_RESET_VGT(1);
1450         }
1451
1452         if (reset_mask & RADEON_RESET_CP) {
1453                 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1454                         S_008020_SOFT_RESET_VGT(1);
1455
1456                 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1457         }
1458
1459         if (reset_mask & RADEON_RESET_DMA) {
1460                 if (rdev->family >= CHIP_RV770)
1461                         srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1462                 else
1463                         srbm_soft_reset |= SOFT_RESET_DMA;
1464         }
1465
1466         if (reset_mask & RADEON_RESET_RLC)
1467                 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1468
1469         if (reset_mask & RADEON_RESET_SEM)
1470                 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1471
1472         if (reset_mask & RADEON_RESET_IH)
1473                 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1474
1475         if (reset_mask & RADEON_RESET_GRBM)
1476                 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1477
1478         if (reset_mask & RADEON_RESET_MC)
1479                 srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1480
1481         if (reset_mask & RADEON_RESET_VMC)
1482                 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1483
1484         if (grbm_soft_reset) {
1485                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1486                 tmp |= grbm_soft_reset;
1487                 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1488                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1489                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1490
1491                 udelay(50);
1492
1493                 tmp &= ~grbm_soft_reset;
1494                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1495                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1496         }
1497
1498         if (srbm_soft_reset) {
1499                 tmp = RREG32(SRBM_SOFT_RESET);
1500                 tmp |= srbm_soft_reset;
1501                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1502                 WREG32(SRBM_SOFT_RESET, tmp);
1503                 tmp = RREG32(SRBM_SOFT_RESET);
1504
1505                 udelay(50);
1506
1507                 tmp &= ~srbm_soft_reset;
1508                 WREG32(SRBM_SOFT_RESET, tmp);
1509                 tmp = RREG32(SRBM_SOFT_RESET);
1510         }
1511
1512         /* Wait a little for things to settle down */
1513         mdelay(1);
1514
1515         rv515_mc_resume(rdev, &save);
1516         udelay(50);
1517
1518         r600_print_gpu_status_regs(rdev);
1519 }
1520
1521 int r600_asic_reset(struct radeon_device *rdev)
1522 {
1523         u32 reset_mask;
1524
1525         reset_mask = r600_gpu_check_soft_reset(rdev);
1526
1527         if (reset_mask)
1528                 r600_set_bios_scratch_engine_hung(rdev, true);
1529
1530         r600_gpu_soft_reset(rdev, reset_mask);
1531
1532         reset_mask = r600_gpu_check_soft_reset(rdev);
1533
1534         if (!reset_mask)
1535                 r600_set_bios_scratch_engine_hung(rdev, false);
1536
1537         return 0;
1538 }
1539
1540 /**
1541  * r600_gfx_is_lockup - Check if the GFX engine is locked up
1542  *
1543  * @rdev: radeon_device pointer
1544  * @ring: radeon_ring structure holding ring information
1545  *
1546  * Check if the GFX engine is locked up.
1547  * Returns true if the engine appears to be locked up, false if not.
1548  */
1549 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1550 {
1551         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1552
1553         if (!(reset_mask & (RADEON_RESET_GFX |
1554                             RADEON_RESET_COMPUTE |
1555                             RADEON_RESET_CP))) {
1556                 radeon_ring_lockup_update(ring);
1557                 return false;
1558         }
1559         /* force CP activities */
1560         radeon_ring_force_activity(rdev, ring);
1561         return radeon_ring_test_lockup(rdev, ring);
1562 }
1563
1564 /**
1565  * r600_dma_is_lockup - Check if the DMA engine is locked up
1566  *
1567  * @rdev: radeon_device pointer
1568  * @ring: radeon_ring structure holding ring information
1569  *
1570  * Check if the async DMA engine is locked up.
1571  * Returns true if the engine appears to be locked up, false if not.
1572  */
1573 bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1574 {
1575         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1576
1577         if (!(reset_mask & RADEON_RESET_DMA)) {
1578                 radeon_ring_lockup_update(ring);
1579                 return false;
1580         }
1581         /* force ring activities */
1582         radeon_ring_force_activity(rdev, ring);
1583         return radeon_ring_test_lockup(rdev, ring);
1584 }
1585
1586 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1587                               u32 tiling_pipe_num,
1588                               u32 max_rb_num,
1589                               u32 total_max_rb_num,
1590                               u32 disabled_rb_mask)
1591 {
1592         u32 rendering_pipe_num, rb_num_width, req_rb_num;
1593         u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1594         u32 data = 0, mask = 1 << (max_rb_num - 1);
1595         unsigned i, j;
1596
1597         /* mask out the RBs that don't exist on that asic */
1598         tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1599         /* make sure at least one RB is available */
1600         if ((tmp & 0xff) != 0xff)
1601                 disabled_rb_mask = tmp;
1602
1603         rendering_pipe_num = 1 << tiling_pipe_num;
1604         req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1605         BUG_ON(rendering_pipe_num < req_rb_num);
1606
1607         pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1608         pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1609
1610         if (rdev->family <= CHIP_RV740) {
1611                 /* r6xx/r7xx */
1612                 rb_num_width = 2;
1613         } else {
1614                 /* eg+ */
1615                 rb_num_width = 4;
1616         }
1617
1618         for (i = 0; i < max_rb_num; i++) {
1619                 if (!(mask & disabled_rb_mask)) {
1620                         for (j = 0; j < pipe_rb_ratio; j++) {
1621                                 data <<= rb_num_width;
1622                                 data |= max_rb_num - i - 1;
1623                         }
1624                         if (pipe_rb_remain) {
1625                                 data <<= rb_num_width;
1626                                 data |= max_rb_num - i - 1;
1627                                 pipe_rb_remain--;
1628                         }
1629                 }
1630                 mask >>= 1;
1631         }
1632
1633         return data;
1634 }
1635
1636 int r600_count_pipe_bits(uint32_t val)
1637 {
1638         return hweight32(val);
1639 }
1640
1641 static void r600_gpu_init(struct radeon_device *rdev)
1642 {
1643         u32 tiling_config;
1644         u32 ramcfg;
1645         u32 cc_rb_backend_disable;
1646         u32 cc_gc_shader_pipe_config;
1647         u32 tmp;
1648         int i, j;
1649         u32 sq_config;
1650         u32 sq_gpr_resource_mgmt_1 = 0;
1651         u32 sq_gpr_resource_mgmt_2 = 0;
1652         u32 sq_thread_resource_mgmt = 0;
1653         u32 sq_stack_resource_mgmt_1 = 0;
1654         u32 sq_stack_resource_mgmt_2 = 0;
1655         u32 disabled_rb_mask;
1656
1657         rdev->config.r600.tiling_group_size = 256;
1658         switch (rdev->family) {
1659         case CHIP_R600:
1660                 rdev->config.r600.max_pipes = 4;
1661                 rdev->config.r600.max_tile_pipes = 8;
1662                 rdev->config.r600.max_simds = 4;
1663                 rdev->config.r600.max_backends = 4;
1664                 rdev->config.r600.max_gprs = 256;
1665                 rdev->config.r600.max_threads = 192;
1666                 rdev->config.r600.max_stack_entries = 256;
1667                 rdev->config.r600.max_hw_contexts = 8;
1668                 rdev->config.r600.max_gs_threads = 16;
1669                 rdev->config.r600.sx_max_export_size = 128;
1670                 rdev->config.r600.sx_max_export_pos_size = 16;
1671                 rdev->config.r600.sx_max_export_smx_size = 128;
1672                 rdev->config.r600.sq_num_cf_insts = 2;
1673                 break;
1674         case CHIP_RV630:
1675         case CHIP_RV635:
1676                 rdev->config.r600.max_pipes = 2;
1677                 rdev->config.r600.max_tile_pipes = 2;
1678                 rdev->config.r600.max_simds = 3;
1679                 rdev->config.r600.max_backends = 1;
1680                 rdev->config.r600.max_gprs = 128;
1681                 rdev->config.r600.max_threads = 192;
1682                 rdev->config.r600.max_stack_entries = 128;
1683                 rdev->config.r600.max_hw_contexts = 8;
1684                 rdev->config.r600.max_gs_threads = 4;
1685                 rdev->config.r600.sx_max_export_size = 128;
1686                 rdev->config.r600.sx_max_export_pos_size = 16;
1687                 rdev->config.r600.sx_max_export_smx_size = 128;
1688                 rdev->config.r600.sq_num_cf_insts = 2;
1689                 break;
1690         case CHIP_RV610:
1691         case CHIP_RV620:
1692         case CHIP_RS780:
1693         case CHIP_RS880:
1694                 rdev->config.r600.max_pipes = 1;
1695                 rdev->config.r600.max_tile_pipes = 1;
1696                 rdev->config.r600.max_simds = 2;
1697                 rdev->config.r600.max_backends = 1;
1698                 rdev->config.r600.max_gprs = 128;
1699                 rdev->config.r600.max_threads = 192;
1700                 rdev->config.r600.max_stack_entries = 128;
1701                 rdev->config.r600.max_hw_contexts = 4;
1702                 rdev->config.r600.max_gs_threads = 4;
1703                 rdev->config.r600.sx_max_export_size = 128;
1704                 rdev->config.r600.sx_max_export_pos_size = 16;
1705                 rdev->config.r600.sx_max_export_smx_size = 128;
1706                 rdev->config.r600.sq_num_cf_insts = 1;
1707                 break;
1708         case CHIP_RV670:
1709                 rdev->config.r600.max_pipes = 4;
1710                 rdev->config.r600.max_tile_pipes = 4;
1711                 rdev->config.r600.max_simds = 4;
1712                 rdev->config.r600.max_backends = 4;
1713                 rdev->config.r600.max_gprs = 192;
1714                 rdev->config.r600.max_threads = 192;
1715                 rdev->config.r600.max_stack_entries = 256;
1716                 rdev->config.r600.max_hw_contexts = 8;
1717                 rdev->config.r600.max_gs_threads = 16;
1718                 rdev->config.r600.sx_max_export_size = 128;
1719                 rdev->config.r600.sx_max_export_pos_size = 16;
1720                 rdev->config.r600.sx_max_export_smx_size = 128;
1721                 rdev->config.r600.sq_num_cf_insts = 2;
1722                 break;
1723         default:
1724                 break;
1725         }
1726
1727         /* Initialize HDP */
1728         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1729                 WREG32((0x2c14 + j), 0x00000000);
1730                 WREG32((0x2c18 + j), 0x00000000);
1731                 WREG32((0x2c1c + j), 0x00000000);
1732                 WREG32((0x2c20 + j), 0x00000000);
1733                 WREG32((0x2c24 + j), 0x00000000);
1734         }
1735
1736         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1737
1738         /* Setup tiling */
1739         tiling_config = 0;
1740         ramcfg = RREG32(RAMCFG);
1741         switch (rdev->config.r600.max_tile_pipes) {
1742         case 1:
1743                 tiling_config |= PIPE_TILING(0);
1744                 break;
1745         case 2:
1746                 tiling_config |= PIPE_TILING(1);
1747                 break;
1748         case 4:
1749                 tiling_config |= PIPE_TILING(2);
1750                 break;
1751         case 8:
1752                 tiling_config |= PIPE_TILING(3);
1753                 break;
1754         default:
1755                 break;
1756         }
1757         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1758         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1759         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1760         tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1761
1762         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1763         if (tmp > 3) {
1764                 tiling_config |= ROW_TILING(3);
1765                 tiling_config |= SAMPLE_SPLIT(3);
1766         } else {
1767                 tiling_config |= ROW_TILING(tmp);
1768                 tiling_config |= SAMPLE_SPLIT(tmp);
1769         }
1770         tiling_config |= BANK_SWAPS(1);
1771
1772         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1773         tmp = R6XX_MAX_BACKENDS -
1774                 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1775         if (tmp < rdev->config.r600.max_backends) {
1776                 rdev->config.r600.max_backends = tmp;
1777         }
1778
1779         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1780         tmp = R6XX_MAX_PIPES -
1781                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1782         if (tmp < rdev->config.r600.max_pipes) {
1783                 rdev->config.r600.max_pipes = tmp;
1784         }
1785         tmp = R6XX_MAX_SIMDS -
1786                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1787         if (tmp < rdev->config.r600.max_simds) {
1788                 rdev->config.r600.max_simds = tmp;
1789         }
1790
1791         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1792         tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1793         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1794                                         R6XX_MAX_BACKENDS, disabled_rb_mask);
1795         tiling_config |= tmp << 16;
1796         rdev->config.r600.backend_map = tmp;
1797
1798         rdev->config.r600.tile_config = tiling_config;
1799         WREG32(GB_TILING_CONFIG, tiling_config);
1800         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1801         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1802         WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1803
1804         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1805         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1806         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1807
1808         /* Setup some CP states */
1809         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1810         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1811
1812         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1813                              SYNC_WALKER | SYNC_ALIGNER));
1814         /* Setup various GPU states */
1815         if (rdev->family == CHIP_RV670)
1816                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1817
1818         tmp = RREG32(SX_DEBUG_1);
1819         tmp |= SMX_EVENT_RELEASE;
1820         if ((rdev->family > CHIP_R600))
1821                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1822         WREG32(SX_DEBUG_1, tmp);
1823
1824         if (((rdev->family) == CHIP_R600) ||
1825             ((rdev->family) == CHIP_RV630) ||
1826             ((rdev->family) == CHIP_RV610) ||
1827             ((rdev->family) == CHIP_RV620) ||
1828             ((rdev->family) == CHIP_RS780) ||
1829             ((rdev->family) == CHIP_RS880)) {
1830                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1831         } else {
1832                 WREG32(DB_DEBUG, 0);
1833         }
1834         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1835                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1836
1837         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1838         WREG32(VGT_NUM_INSTANCES, 0);
1839
1840         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1841         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1842
1843         tmp = RREG32(SQ_MS_FIFO_SIZES);
1844         if (((rdev->family) == CHIP_RV610) ||
1845             ((rdev->family) == CHIP_RV620) ||
1846             ((rdev->family) == CHIP_RS780) ||
1847             ((rdev->family) == CHIP_RS880)) {
1848                 tmp = (CACHE_FIFO_SIZE(0xa) |
1849                        FETCH_FIFO_HIWATER(0xa) |
1850                        DONE_FIFO_HIWATER(0xe0) |
1851                        ALU_UPDATE_FIFO_HIWATER(0x8));
1852         } else if (((rdev->family) == CHIP_R600) ||
1853                    ((rdev->family) == CHIP_RV630)) {
1854                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1855                 tmp |= DONE_FIFO_HIWATER(0x4);
1856         }
1857         WREG32(SQ_MS_FIFO_SIZES, tmp);
1858
1859         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1860          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1861          */
1862         sq_config = RREG32(SQ_CONFIG);
1863         sq_config &= ~(PS_PRIO(3) |
1864                        VS_PRIO(3) |
1865                        GS_PRIO(3) |
1866                        ES_PRIO(3));
1867         sq_config |= (DX9_CONSTS |
1868                       VC_ENABLE |
1869                       PS_PRIO(0) |
1870                       VS_PRIO(1) |
1871                       GS_PRIO(2) |
1872                       ES_PRIO(3));
1873
1874         if ((rdev->family) == CHIP_R600) {
1875                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1876                                           NUM_VS_GPRS(124) |
1877                                           NUM_CLAUSE_TEMP_GPRS(4));
1878                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1879                                           NUM_ES_GPRS(0));
1880                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1881                                            NUM_VS_THREADS(48) |
1882                                            NUM_GS_THREADS(4) |
1883                                            NUM_ES_THREADS(4));
1884                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1885                                             NUM_VS_STACK_ENTRIES(128));
1886                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1887                                             NUM_ES_STACK_ENTRIES(0));
1888         } else if (((rdev->family) == CHIP_RV610) ||
1889                    ((rdev->family) == CHIP_RV620) ||
1890                    ((rdev->family) == CHIP_RS780) ||
1891                    ((rdev->family) == CHIP_RS880)) {
1892                 /* no vertex cache */
1893                 sq_config &= ~VC_ENABLE;
1894
1895                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1896                                           NUM_VS_GPRS(44) |
1897                                           NUM_CLAUSE_TEMP_GPRS(2));
1898                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1899                                           NUM_ES_GPRS(17));
1900                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1901                                            NUM_VS_THREADS(78) |
1902                                            NUM_GS_THREADS(4) |
1903                                            NUM_ES_THREADS(31));
1904                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1905                                             NUM_VS_STACK_ENTRIES(40));
1906                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1907                                             NUM_ES_STACK_ENTRIES(16));
1908         } else if (((rdev->family) == CHIP_RV630) ||
1909                    ((rdev->family) == CHIP_RV635)) {
1910                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1911                                           NUM_VS_GPRS(44) |
1912                                           NUM_CLAUSE_TEMP_GPRS(2));
1913                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1914                                           NUM_ES_GPRS(18));
1915                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1916                                            NUM_VS_THREADS(78) |
1917                                            NUM_GS_THREADS(4) |
1918                                            NUM_ES_THREADS(31));
1919                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1920                                             NUM_VS_STACK_ENTRIES(40));
1921                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1922                                             NUM_ES_STACK_ENTRIES(16));
1923         } else if ((rdev->family) == CHIP_RV670) {
1924                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1925                                           NUM_VS_GPRS(44) |
1926                                           NUM_CLAUSE_TEMP_GPRS(2));
1927                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1928                                           NUM_ES_GPRS(17));
1929                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1930                                            NUM_VS_THREADS(78) |
1931                                            NUM_GS_THREADS(4) |
1932                                            NUM_ES_THREADS(31));
1933                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1934                                             NUM_VS_STACK_ENTRIES(64));
1935                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1936                                             NUM_ES_STACK_ENTRIES(64));
1937         }
1938
1939         WREG32(SQ_CONFIG, sq_config);
1940         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1941         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1942         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1943         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1944         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1945
1946         if (((rdev->family) == CHIP_RV610) ||
1947             ((rdev->family) == CHIP_RV620) ||
1948             ((rdev->family) == CHIP_RS780) ||
1949             ((rdev->family) == CHIP_RS880)) {
1950                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1951         } else {
1952                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1953         }
1954
1955         /* More default values. 2D/3D driver should adjust as needed */
1956         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1957                                          S1_X(0x4) | S1_Y(0xc)));
1958         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1959                                          S1_X(0x2) | S1_Y(0x2) |
1960                                          S2_X(0xa) | S2_Y(0x6) |
1961                                          S3_X(0x6) | S3_Y(0xa)));
1962         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1963                                              S1_X(0x4) | S1_Y(0xc) |
1964                                              S2_X(0x1) | S2_Y(0x6) |
1965                                              S3_X(0xa) | S3_Y(0xe)));
1966         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1967                                              S5_X(0x0) | S5_Y(0x0) |
1968                                              S6_X(0xb) | S6_Y(0x4) |
1969                                              S7_X(0x7) | S7_Y(0x8)));
1970
1971         WREG32(VGT_STRMOUT_EN, 0);
1972         tmp = rdev->config.r600.max_pipes * 16;
1973         switch (rdev->family) {
1974         case CHIP_RV610:
1975         case CHIP_RV620:
1976         case CHIP_RS780:
1977         case CHIP_RS880:
1978                 tmp += 32;
1979                 break;
1980         case CHIP_RV670:
1981                 tmp += 128;
1982                 break;
1983         default:
1984                 break;
1985         }
1986         if (tmp > 256) {
1987                 tmp = 256;
1988         }
1989         WREG32(VGT_ES_PER_GS, 128);
1990         WREG32(VGT_GS_PER_ES, tmp);
1991         WREG32(VGT_GS_PER_VS, 2);
1992         WREG32(VGT_GS_VERTEX_REUSE, 16);
1993
1994         /* more default values. 2D/3D driver should adjust as needed */
1995         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1996         WREG32(VGT_STRMOUT_EN, 0);
1997         WREG32(SX_MISC, 0);
1998         WREG32(PA_SC_MODE_CNTL, 0);
1999         WREG32(PA_SC_AA_CONFIG, 0);
2000         WREG32(PA_SC_LINE_STIPPLE, 0);
2001         WREG32(SPI_INPUT_Z, 0);
2002         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2003         WREG32(CB_COLOR7_FRAG, 0);
2004
2005         /* Clear render buffer base addresses */
2006         WREG32(CB_COLOR0_BASE, 0);
2007         WREG32(CB_COLOR1_BASE, 0);
2008         WREG32(CB_COLOR2_BASE, 0);
2009         WREG32(CB_COLOR3_BASE, 0);
2010         WREG32(CB_COLOR4_BASE, 0);
2011         WREG32(CB_COLOR5_BASE, 0);
2012         WREG32(CB_COLOR6_BASE, 0);
2013         WREG32(CB_COLOR7_BASE, 0);
2014         WREG32(CB_COLOR7_FRAG, 0);
2015
2016         switch (rdev->family) {
2017         case CHIP_RV610:
2018         case CHIP_RV620:
2019         case CHIP_RS780:
2020         case CHIP_RS880:
2021                 tmp = TC_L2_SIZE(8);
2022                 break;
2023         case CHIP_RV630:
2024         case CHIP_RV635:
2025                 tmp = TC_L2_SIZE(4);
2026                 break;
2027         case CHIP_R600:
2028                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2029                 break;
2030         default:
2031                 tmp = TC_L2_SIZE(0);
2032                 break;
2033         }
2034         WREG32(TC_CNTL, tmp);
2035
2036         tmp = RREG32(HDP_HOST_PATH_CNTL);
2037         WREG32(HDP_HOST_PATH_CNTL, tmp);
2038
2039         tmp = RREG32(ARB_POP);
2040         tmp |= ENABLE_TC128;
2041         WREG32(ARB_POP, tmp);
2042
2043         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2044         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2045                                NUM_CLIP_SEQ(3)));
2046         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2047         WREG32(VC_ENHANCE, 0);
2048 }
2049
2050
2051 /*
2052  * Indirect registers accessor
2053  */
2054 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2055 {
2056         u32 r;
2057
2058         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2059         (void)RREG32(PCIE_PORT_INDEX);
2060         r = RREG32(PCIE_PORT_DATA);
2061         return r;
2062 }
2063
2064 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2065 {
2066         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2067         (void)RREG32(PCIE_PORT_INDEX);
2068         WREG32(PCIE_PORT_DATA, (v));
2069         (void)RREG32(PCIE_PORT_DATA);
2070 }
2071
2072 /*
2073  * CP & Ring
2074  */
2075 void r600_cp_stop(struct radeon_device *rdev)
2076 {
2077         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2078         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
2079         WREG32(SCRATCH_UMSK, 0);
2080         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2081 }
2082
2083 int r600_init_microcode(struct radeon_device *rdev)
2084 {
2085         struct platform_device *pdev;
2086         const char *chip_name;
2087         const char *rlc_chip_name;
2088         size_t pfp_req_size, me_req_size, rlc_req_size;
2089         char fw_name[30];
2090         int err;
2091
2092         DRM_DEBUG("\n");
2093
2094         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
2095         err = IS_ERR(pdev);
2096         if (err) {
2097                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
2098                 return -EINVAL;
2099         }
2100
2101         switch (rdev->family) {
2102         case CHIP_R600:
2103                 chip_name = "R600";
2104                 rlc_chip_name = "R600";
2105                 break;
2106         case CHIP_RV610:
2107                 chip_name = "RV610";
2108                 rlc_chip_name = "R600";
2109                 break;
2110         case CHIP_RV630:
2111                 chip_name = "RV630";
2112                 rlc_chip_name = "R600";
2113                 break;
2114         case CHIP_RV620:
2115                 chip_name = "RV620";
2116                 rlc_chip_name = "R600";
2117                 break;
2118         case CHIP_RV635:
2119                 chip_name = "RV635";
2120                 rlc_chip_name = "R600";
2121                 break;
2122         case CHIP_RV670:
2123                 chip_name = "RV670";
2124                 rlc_chip_name = "R600";
2125                 break;
2126         case CHIP_RS780:
2127         case CHIP_RS880:
2128                 chip_name = "RS780";
2129                 rlc_chip_name = "R600";
2130                 break;
2131         case CHIP_RV770:
2132                 chip_name = "RV770";
2133                 rlc_chip_name = "R700";
2134                 break;
2135         case CHIP_RV730:
2136         case CHIP_RV740:
2137                 chip_name = "RV730";
2138                 rlc_chip_name = "R700";
2139                 break;
2140         case CHIP_RV710:
2141                 chip_name = "RV710";
2142                 rlc_chip_name = "R700";
2143                 break;
2144         case CHIP_CEDAR:
2145                 chip_name = "CEDAR";
2146                 rlc_chip_name = "CEDAR";
2147                 break;
2148         case CHIP_REDWOOD:
2149                 chip_name = "REDWOOD";
2150                 rlc_chip_name = "REDWOOD";
2151                 break;
2152         case CHIP_JUNIPER:
2153                 chip_name = "JUNIPER";
2154                 rlc_chip_name = "JUNIPER";
2155                 break;
2156         case CHIP_CYPRESS:
2157         case CHIP_HEMLOCK:
2158                 chip_name = "CYPRESS";
2159                 rlc_chip_name = "CYPRESS";
2160                 break;
2161         case CHIP_PALM:
2162                 chip_name = "PALM";
2163                 rlc_chip_name = "SUMO";
2164                 break;
2165         case CHIP_SUMO:
2166                 chip_name = "SUMO";
2167                 rlc_chip_name = "SUMO";
2168                 break;
2169         case CHIP_SUMO2:
2170                 chip_name = "SUMO2";
2171                 rlc_chip_name = "SUMO";
2172                 break;
2173         default: BUG();
2174         }
2175
2176         if (rdev->family >= CHIP_CEDAR) {
2177                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2178                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2179                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2180         } else if (rdev->family >= CHIP_RV770) {
2181                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2182                 me_req_size = R700_PM4_UCODE_SIZE * 4;
2183                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2184         } else {
2185                 pfp_req_size = PFP_UCODE_SIZE * 4;
2186                 me_req_size = PM4_UCODE_SIZE * 12;
2187                 rlc_req_size = RLC_UCODE_SIZE * 4;
2188         }
2189
2190         DRM_INFO("Loading %s Microcode\n", chip_name);
2191
2192         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2193         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
2194         if (err)
2195                 goto out;
2196         if (rdev->pfp_fw->size != pfp_req_size) {
2197                 printk(KERN_ERR
2198                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2199                        rdev->pfp_fw->size, fw_name);
2200                 err = -EINVAL;
2201                 goto out;
2202         }
2203
2204         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2205         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
2206         if (err)
2207                 goto out;
2208         if (rdev->me_fw->size != me_req_size) {
2209                 printk(KERN_ERR
2210                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2211                        rdev->me_fw->size, fw_name);
2212                 err = -EINVAL;
2213         }
2214
2215         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2216         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
2217         if (err)
2218                 goto out;
2219         if (rdev->rlc_fw->size != rlc_req_size) {
2220                 printk(KERN_ERR
2221                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2222                        rdev->rlc_fw->size, fw_name);
2223                 err = -EINVAL;
2224         }
2225
2226 out:
2227         platform_device_unregister(pdev);
2228
2229         if (err) {
2230                 if (err != -EINVAL)
2231                         printk(KERN_ERR
2232                                "r600_cp: Failed to load firmware \"%s\"\n",
2233                                fw_name);
2234                 release_firmware(rdev->pfp_fw);
2235                 rdev->pfp_fw = NULL;
2236                 release_firmware(rdev->me_fw);
2237                 rdev->me_fw = NULL;
2238                 release_firmware(rdev->rlc_fw);
2239                 rdev->rlc_fw = NULL;
2240         }
2241         return err;
2242 }
2243
2244 static int r600_cp_load_microcode(struct radeon_device *rdev)
2245 {
2246         const __be32 *fw_data;
2247         int i;
2248
2249         if (!rdev->me_fw || !rdev->pfp_fw)
2250                 return -EINVAL;
2251
2252         r600_cp_stop(rdev);
2253
2254         WREG32(CP_RB_CNTL,
2255 #ifdef __BIG_ENDIAN
2256                BUF_SWAP_32BIT |
2257 #endif
2258                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2259
2260         /* Reset cp */
2261         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2262         RREG32(GRBM_SOFT_RESET);
2263         mdelay(15);
2264         WREG32(GRBM_SOFT_RESET, 0);
2265
2266         WREG32(CP_ME_RAM_WADDR, 0);
2267
2268         fw_data = (const __be32 *)rdev->me_fw->data;
2269         WREG32(CP_ME_RAM_WADDR, 0);
2270         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
2271                 WREG32(CP_ME_RAM_DATA,
2272                        be32_to_cpup(fw_data++));
2273
2274         fw_data = (const __be32 *)rdev->pfp_fw->data;
2275         WREG32(CP_PFP_UCODE_ADDR, 0);
2276         for (i = 0; i < PFP_UCODE_SIZE; i++)
2277                 WREG32(CP_PFP_UCODE_DATA,
2278                        be32_to_cpup(fw_data++));
2279
2280         WREG32(CP_PFP_UCODE_ADDR, 0);
2281         WREG32(CP_ME_RAM_WADDR, 0);
2282         WREG32(CP_ME_RAM_RADDR, 0);
2283         return 0;
2284 }
2285
2286 int r600_cp_start(struct radeon_device *rdev)
2287 {
2288         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2289         int r;
2290         uint32_t cp_me;
2291
2292         r = radeon_ring_lock(rdev, ring, 7);
2293         if (r) {
2294                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2295                 return r;
2296         }
2297         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2298         radeon_ring_write(ring, 0x1);
2299         if (rdev->family >= CHIP_RV770) {
2300                 radeon_ring_write(ring, 0x0);
2301                 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2302         } else {
2303                 radeon_ring_write(ring, 0x3);
2304                 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2305         }
2306         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2307         radeon_ring_write(ring, 0);
2308         radeon_ring_write(ring, 0);
2309         radeon_ring_unlock_commit(rdev, ring);
2310
2311         cp_me = 0xff;
2312         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2313         return 0;
2314 }
2315
2316 int r600_cp_resume(struct radeon_device *rdev)
2317 {
2318         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2319         u32 tmp;
2320         u32 rb_bufsz;
2321         int r;
2322
2323         /* Reset cp */
2324         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2325         RREG32(GRBM_SOFT_RESET);
2326         mdelay(15);
2327         WREG32(GRBM_SOFT_RESET, 0);
2328
2329         /* Set ring buffer size */
2330         rb_bufsz = drm_order(ring->ring_size / 8);
2331         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2332 #ifdef __BIG_ENDIAN
2333         tmp |= BUF_SWAP_32BIT;
2334 #endif
2335         WREG32(CP_RB_CNTL, tmp);
2336         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2337
2338         /* Set the write pointer delay */
2339         WREG32(CP_RB_WPTR_DELAY, 0);
2340
2341         /* Initialize the ring buffer's read and write pointers */
2342         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2343         WREG32(CP_RB_RPTR_WR, 0);
2344         ring->wptr = 0;
2345         WREG32(CP_RB_WPTR, ring->wptr);
2346
2347         /* set the wb address whether it's enabled or not */
2348         WREG32(CP_RB_RPTR_ADDR,
2349                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2350         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2351         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2352
2353         if (rdev->wb.enabled)
2354                 WREG32(SCRATCH_UMSK, 0xff);
2355         else {
2356                 tmp |= RB_NO_UPDATE;
2357                 WREG32(SCRATCH_UMSK, 0);
2358         }
2359
2360         mdelay(1);
2361         WREG32(CP_RB_CNTL, tmp);
2362
2363         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2364         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2365
2366         ring->rptr = RREG32(CP_RB_RPTR);
2367
2368         r600_cp_start(rdev);
2369         ring->ready = true;
2370         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2371         if (r) {
2372                 ring->ready = false;
2373                 return r;
2374         }
2375         return 0;
2376 }
2377
2378 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2379 {
2380         u32 rb_bufsz;
2381         int r;
2382
2383         /* Align ring size */
2384         rb_bufsz = drm_order(ring_size / 8);
2385         ring_size = (1 << (rb_bufsz + 1)) * 4;
2386         ring->ring_size = ring_size;
2387         ring->align_mask = 16 - 1;
2388
2389         if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2390                 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2391                 if (r) {
2392                         DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2393                         ring->rptr_save_reg = 0;
2394                 }
2395         }
2396 }
2397
2398 void r600_cp_fini(struct radeon_device *rdev)
2399 {
2400         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2401         r600_cp_stop(rdev);
2402         radeon_ring_fini(rdev, ring);
2403         radeon_scratch_free(rdev, ring->rptr_save_reg);
2404 }
2405
2406 /*
2407  * DMA
2408  * Starting with R600, the GPU has an asynchronous
2409  * DMA engine.  The programming model is very similar
2410  * to the 3D engine (ring buffer, IBs, etc.), but the
2411  * DMA controller has it's own packet format that is
2412  * different form the PM4 format used by the 3D engine.
2413  * It supports copying data, writing embedded data,
2414  * solid fills, and a number of other things.  It also
2415  * has support for tiling/detiling of buffers.
2416  */
2417 /**
2418  * r600_dma_stop - stop the async dma engine
2419  *
2420  * @rdev: radeon_device pointer
2421  *
2422  * Stop the async dma engine (r6xx-evergreen).
2423  */
2424 void r600_dma_stop(struct radeon_device *rdev)
2425 {
2426         u32 rb_cntl = RREG32(DMA_RB_CNTL);
2427
2428         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2429
2430         rb_cntl &= ~DMA_RB_ENABLE;
2431         WREG32(DMA_RB_CNTL, rb_cntl);
2432
2433         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2434 }
2435
2436 /**
2437  * r600_dma_resume - setup and start the async dma engine
2438  *
2439  * @rdev: radeon_device pointer
2440  *
2441  * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2442  * Returns 0 for success, error for failure.
2443  */
2444 int r600_dma_resume(struct radeon_device *rdev)
2445 {
2446         struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2447         u32 rb_cntl, dma_cntl, ib_cntl;
2448         u32 rb_bufsz;
2449         int r;
2450
2451         /* Reset dma */
2452         if (rdev->family >= CHIP_RV770)
2453                 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2454         else
2455                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2456         RREG32(SRBM_SOFT_RESET);
2457         udelay(50);
2458         WREG32(SRBM_SOFT_RESET, 0);
2459
2460         WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2461         WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2462
2463         /* Set ring buffer size in dwords */
2464         rb_bufsz = drm_order(ring->ring_size / 4);
2465         rb_cntl = rb_bufsz << 1;
2466 #ifdef __BIG_ENDIAN
2467         rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2468 #endif
2469         WREG32(DMA_RB_CNTL, rb_cntl);
2470
2471         /* Initialize the ring buffer's read and write pointers */
2472         WREG32(DMA_RB_RPTR, 0);
2473         WREG32(DMA_RB_WPTR, 0);
2474
2475         /* set the wb address whether it's enabled or not */
2476         WREG32(DMA_RB_RPTR_ADDR_HI,
2477                upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2478         WREG32(DMA_RB_RPTR_ADDR_LO,
2479                ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2480
2481         if (rdev->wb.enabled)
2482                 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2483
2484         WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2485
2486         /* enable DMA IBs */
2487         ib_cntl = DMA_IB_ENABLE;
2488 #ifdef __BIG_ENDIAN
2489         ib_cntl |= DMA_IB_SWAP_ENABLE;
2490 #endif
2491         WREG32(DMA_IB_CNTL, ib_cntl);
2492
2493         dma_cntl = RREG32(DMA_CNTL);
2494         dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2495         WREG32(DMA_CNTL, dma_cntl);
2496
2497         if (rdev->family >= CHIP_RV770)
2498                 WREG32(DMA_MODE, 1);
2499
2500         ring->wptr = 0;
2501         WREG32(DMA_RB_WPTR, ring->wptr << 2);
2502
2503         ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2504
2505         WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2506
2507         ring->ready = true;
2508
2509         r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2510         if (r) {
2511                 ring->ready = false;
2512                 return r;
2513         }
2514
2515         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2516
2517         return 0;
2518 }
2519
2520 /**
2521  * r600_dma_fini - tear down the async dma engine
2522  *
2523  * @rdev: radeon_device pointer
2524  *
2525  * Stop the async dma engine and free the ring (r6xx-evergreen).
2526  */
2527 void r600_dma_fini(struct radeon_device *rdev)
2528 {
2529         r600_dma_stop(rdev);
2530         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2531 }
2532
2533 /*
2534  * GPU scratch registers helpers function.
2535  */
2536 void r600_scratch_init(struct radeon_device *rdev)
2537 {
2538         int i;
2539
2540         rdev->scratch.num_reg = 7;
2541         rdev->scratch.reg_base = SCRATCH_REG0;
2542         for (i = 0; i < rdev->scratch.num_reg; i++) {
2543                 rdev->scratch.free[i] = true;
2544                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2545         }
2546 }
2547
2548 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2549 {
2550         uint32_t scratch;
2551         uint32_t tmp = 0;
2552         unsigned i;
2553         int r;
2554
2555         r = radeon_scratch_get(rdev, &scratch);
2556         if (r) {
2557                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2558                 return r;
2559         }
2560         WREG32(scratch, 0xCAFEDEAD);
2561         r = radeon_ring_lock(rdev, ring, 3);
2562         if (r) {
2563                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2564                 radeon_scratch_free(rdev, scratch);
2565                 return r;
2566         }
2567         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2568         radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2569         radeon_ring_write(ring, 0xDEADBEEF);
2570         radeon_ring_unlock_commit(rdev, ring);
2571         for (i = 0; i < rdev->usec_timeout; i++) {
2572                 tmp = RREG32(scratch);
2573                 if (tmp == 0xDEADBEEF)
2574                         break;
2575                 DRM_UDELAY(1);
2576         }
2577         if (i < rdev->usec_timeout) {
2578                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2579         } else {
2580                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2581                           ring->idx, scratch, tmp);
2582                 r = -EINVAL;
2583         }
2584         radeon_scratch_free(rdev, scratch);
2585         return r;
2586 }
2587
2588 /**
2589  * r600_dma_ring_test - simple async dma engine test
2590  *
2591  * @rdev: radeon_device pointer
2592  * @ring: radeon_ring structure holding ring information
2593  *
2594  * Test the DMA engine by writing using it to write an
2595  * value to memory. (r6xx-SI).
2596  * Returns 0 for success, error for failure.
2597  */
2598 int r600_dma_ring_test(struct radeon_device *rdev,
2599                        struct radeon_ring *ring)
2600 {
2601         unsigned i;
2602         int r;
2603         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
2604         u32 tmp;
2605
2606         if (!ptr) {
2607                 DRM_ERROR("invalid vram scratch pointer\n");
2608                 return -EINVAL;
2609         }
2610
2611         tmp = 0xCAFEDEAD;
2612         writel(tmp, ptr);
2613
2614         r = radeon_ring_lock(rdev, ring, 4);
2615         if (r) {
2616                 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2617                 return r;
2618         }
2619         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2620         radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2621         radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2622         radeon_ring_write(ring, 0xDEADBEEF);
2623         radeon_ring_unlock_commit(rdev, ring);
2624
2625         for (i = 0; i < rdev->usec_timeout; i++) {
2626                 tmp = readl(ptr);
2627                 if (tmp == 0xDEADBEEF)
2628                         break;
2629                 DRM_UDELAY(1);
2630         }
2631
2632         if (i < rdev->usec_timeout) {
2633                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2634         } else {
2635                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2636                           ring->idx, tmp);
2637                 r = -EINVAL;
2638         }
2639         return r;
2640 }
2641
2642 /*
2643  * CP fences/semaphores
2644  */
2645
2646 void r600_fence_ring_emit(struct radeon_device *rdev,
2647                           struct radeon_fence *fence)
2648 {
2649         struct radeon_ring *ring = &rdev->ring[fence->ring];
2650
2651         if (rdev->wb.use_event) {
2652                 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2653                 /* flush read cache over gart */
2654                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2655                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2656                                         PACKET3_VC_ACTION_ENA |
2657                                         PACKET3_SH_ACTION_ENA);
2658                 radeon_ring_write(ring, 0xFFFFFFFF);
2659                 radeon_ring_write(ring, 0);
2660                 radeon_ring_write(ring, 10); /* poll interval */
2661                 /* EVENT_WRITE_EOP - flush caches, send int */
2662                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2663                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2664                 radeon_ring_write(ring, addr & 0xffffffff);
2665                 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2666                 radeon_ring_write(ring, fence->seq);
2667                 radeon_ring_write(ring, 0);
2668         } else {
2669                 /* flush read cache over gart */
2670                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2671                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2672                                         PACKET3_VC_ACTION_ENA |
2673                                         PACKET3_SH_ACTION_ENA);
2674                 radeon_ring_write(ring, 0xFFFFFFFF);
2675                 radeon_ring_write(ring, 0);
2676                 radeon_ring_write(ring, 10); /* poll interval */
2677                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2678                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2679                 /* wait for 3D idle clean */
2680                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2681                 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2682                 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2683                 /* Emit fence sequence & fire IRQ */
2684                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2685                 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2686                 radeon_ring_write(ring, fence->seq);
2687                 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2688                 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2689                 radeon_ring_write(ring, RB_INT_STAT);
2690         }
2691 }
2692
2693 void r600_semaphore_ring_emit(struct radeon_device *rdev,
2694                               struct radeon_ring *ring,
2695                               struct radeon_semaphore *semaphore,
2696                               bool emit_wait)
2697 {
2698         uint64_t addr = semaphore->gpu_addr;
2699         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2700
2701         if (rdev->family < CHIP_CAYMAN)
2702                 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
2703
2704         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2705         radeon_ring_write(ring, addr & 0xffffffff);
2706         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
2707 }
2708
2709 /*
2710  * DMA fences/semaphores
2711  */
2712
2713 /**
2714  * r600_dma_fence_ring_emit - emit a fence on the DMA ring
2715  *
2716  * @rdev: radeon_device pointer
2717  * @fence: radeon fence object
2718  *
2719  * Add a DMA fence packet to the ring to write
2720  * the fence seq number and DMA trap packet to generate
2721  * an interrupt if needed (r6xx-r7xx).
2722  */
2723 void r600_dma_fence_ring_emit(struct radeon_device *rdev,
2724                               struct radeon_fence *fence)
2725 {
2726         struct radeon_ring *ring = &rdev->ring[fence->ring];
2727         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2728
2729         /* write the fence */
2730         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
2731         radeon_ring_write(ring, addr & 0xfffffffc);
2732         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
2733         radeon_ring_write(ring, lower_32_bits(fence->seq));
2734         /* generate an interrupt */
2735         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
2736 }
2737
2738 /**
2739  * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
2740  *
2741  * @rdev: radeon_device pointer
2742  * @ring: radeon_ring structure holding ring information
2743  * @semaphore: radeon semaphore object
2744  * @emit_wait: wait or signal semaphore
2745  *
2746  * Add a DMA semaphore packet to the ring wait on or signal
2747  * other rings (r6xx-SI).
2748  */
2749 void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
2750                                   struct radeon_ring *ring,
2751                                   struct radeon_semaphore *semaphore,
2752                                   bool emit_wait)
2753 {
2754         u64 addr = semaphore->gpu_addr;
2755         u32 s = emit_wait ? 0 : 1;
2756
2757         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
2758         radeon_ring_write(ring, addr & 0xfffffffc);
2759         radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
2760 }
2761
2762 int r600_copy_blit(struct radeon_device *rdev,
2763                    uint64_t src_offset,
2764                    uint64_t dst_offset,
2765                    unsigned num_gpu_pages,
2766                    struct radeon_fence **fence)
2767 {
2768         struct radeon_semaphore *sem = NULL;
2769         struct radeon_sa_bo *vb = NULL;
2770         int r;
2771
2772         r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
2773         if (r) {
2774                 return r;
2775         }
2776         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
2777         r600_blit_done_copy(rdev, fence, vb, sem);
2778         return 0;
2779 }
2780
2781 /**
2782  * r600_copy_dma - copy pages using the DMA engine
2783  *
2784  * @rdev: radeon_device pointer
2785  * @src_offset: src GPU address
2786  * @dst_offset: dst GPU address
2787  * @num_gpu_pages: number of GPU pages to xfer
2788  * @fence: radeon fence object
2789  *
2790  * Copy GPU paging using the DMA engine (r6xx).
2791  * Used by the radeon ttm implementation to move pages if
2792  * registered as the asic copy callback.
2793  */
2794 int r600_copy_dma(struct radeon_device *rdev,
2795                   uint64_t src_offset, uint64_t dst_offset,
2796                   unsigned num_gpu_pages,
2797                   struct radeon_fence **fence)
2798 {
2799         struct radeon_semaphore *sem = NULL;
2800         int ring_index = rdev->asic->copy.dma_ring_index;
2801         struct radeon_ring *ring = &rdev->ring[ring_index];
2802         u32 size_in_dw, cur_size_in_dw;
2803         int i, num_loops;
2804         int r = 0;
2805
2806         r = radeon_semaphore_create(rdev, &sem);
2807         if (r) {
2808                 DRM_ERROR("radeon: moving bo (%d).\n", r);
2809                 return r;
2810         }
2811
2812         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
2813         num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
2814         r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
2815         if (r) {
2816                 DRM_ERROR("radeon: moving bo (%d).\n", r);
2817                 radeon_semaphore_free(rdev, &sem, NULL);
2818                 return r;
2819         }
2820
2821         if (radeon_fence_need_sync(*fence, ring->idx)) {
2822                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
2823                                             ring->idx);
2824                 radeon_fence_note_sync(*fence, ring->idx);
2825         } else {
2826                 radeon_semaphore_free(rdev, &sem, NULL);
2827         }
2828
2829         for (i = 0; i < num_loops; i++) {
2830                 cur_size_in_dw = size_in_dw;
2831                 if (cur_size_in_dw > 0xFFFE)
2832                         cur_size_in_dw = 0xFFFE;
2833                 size_in_dw -= cur_size_in_dw;
2834                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
2835                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
2836                 radeon_ring_write(ring, src_offset & 0xfffffffc);
2837                 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
2838                                          (upper_32_bits(src_offset) & 0xff)));
2839                 src_offset += cur_size_in_dw * 4;
2840                 dst_offset += cur_size_in_dw * 4;
2841         }
2842
2843         r = radeon_fence_emit(rdev, fence, ring->idx);
2844         if (r) {
2845                 radeon_ring_unlock_undo(rdev, ring);
2846                 return r;
2847         }
2848
2849         radeon_ring_unlock_commit(rdev, ring);
2850         radeon_semaphore_free(rdev, &sem, *fence);
2851
2852         return r;
2853 }
2854
2855 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
2856                          uint32_t tiling_flags, uint32_t pitch,
2857                          uint32_t offset, uint32_t obj_size)
2858 {
2859         /* FIXME: implement */
2860         return 0;
2861 }
2862
2863 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2864 {
2865         /* FIXME: implement */
2866 }
2867
2868 static int r600_startup(struct radeon_device *rdev)
2869 {
2870         struct radeon_ring *ring;
2871         int r;
2872
2873         /* enable pcie gen2 link */
2874         r600_pcie_gen2_enable(rdev);
2875
2876         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2877                 r = r600_init_microcode(rdev);
2878                 if (r) {
2879                         DRM_ERROR("Failed to load firmware!\n");
2880                         return r;
2881                 }
2882         }
2883
2884         r = r600_vram_scratch_init(rdev);
2885         if (r)
2886                 return r;
2887
2888         r600_mc_program(rdev);
2889         if (rdev->flags & RADEON_IS_AGP) {
2890                 r600_agp_enable(rdev);
2891         } else {
2892                 r = r600_pcie_gart_enable(rdev);
2893                 if (r)
2894                         return r;
2895         }
2896         r600_gpu_init(rdev);
2897         r = r600_blit_init(rdev);
2898         if (r) {
2899                 r600_blit_fini(rdev);
2900                 rdev->asic->copy.copy = NULL;
2901                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2902         }
2903
2904         /* allocate wb buffer */
2905         r = radeon_wb_init(rdev);
2906         if (r)
2907                 return r;
2908
2909         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2910         if (r) {
2911                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2912                 return r;
2913         }
2914
2915         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2916         if (r) {
2917                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2918                 return r;
2919         }
2920
2921         /* Enable IRQ */
2922         r = r600_irq_init(rdev);
2923         if (r) {
2924                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2925                 radeon_irq_kms_fini(rdev);
2926                 return r;
2927         }
2928         r600_irq_set(rdev);
2929
2930         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2931         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2932                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2933                              0, 0xfffff, RADEON_CP_PACKET2);
2934         if (r)
2935                 return r;
2936
2937         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2938         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2939                              DMA_RB_RPTR, DMA_RB_WPTR,
2940                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2941         if (r)
2942                 return r;
2943
2944         r = r600_cp_load_microcode(rdev);
2945         if (r)
2946                 return r;
2947         r = r600_cp_resume(rdev);
2948         if (r)
2949                 return r;
2950
2951         r = r600_dma_resume(rdev);
2952         if (r)
2953                 return r;
2954
2955         r = radeon_ib_pool_init(rdev);
2956         if (r) {
2957                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2958                 return r;
2959         }
2960
2961         r = r600_audio_init(rdev);
2962         if (r) {
2963                 DRM_ERROR("radeon: audio init failed\n");
2964                 return r;
2965         }
2966
2967         return 0;
2968 }
2969
2970 void r600_vga_set_state(struct radeon_device *rdev, bool state)
2971 {
2972         uint32_t temp;
2973
2974         temp = RREG32(CONFIG_CNTL);
2975         if (state == false) {
2976                 temp &= ~(1<<0);
2977                 temp |= (1<<1);
2978         } else {
2979                 temp &= ~(1<<1);
2980         }
2981         WREG32(CONFIG_CNTL, temp);
2982 }
2983
2984 int r600_resume(struct radeon_device *rdev)
2985 {
2986         int r;
2987
2988         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
2989          * posting will perform necessary task to bring back GPU into good
2990          * shape.
2991          */
2992         /* post card */
2993         atom_asic_init(rdev->mode_info.atom_context);
2994
2995         rdev->accel_working = true;
2996         r = r600_startup(rdev);
2997         if (r) {
2998                 DRM_ERROR("r600 startup failed on resume\n");
2999                 rdev->accel_working = false;
3000                 return r;
3001         }
3002
3003         return r;
3004 }
3005
3006 int r600_suspend(struct radeon_device *rdev)
3007 {
3008         r600_audio_fini(rdev);
3009         r600_cp_stop(rdev);
3010         r600_dma_stop(rdev);
3011         r600_irq_suspend(rdev);
3012         radeon_wb_disable(rdev);
3013         r600_pcie_gart_disable(rdev);
3014
3015         return 0;
3016 }
3017
3018 /* Plan is to move initialization in that function and use
3019  * helper function so that radeon_device_init pretty much
3020  * do nothing more than calling asic specific function. This
3021  * should also allow to remove a bunch of callback function
3022  * like vram_info.
3023  */
3024 int r600_init(struct radeon_device *rdev)
3025 {
3026         int r;
3027
3028         if (r600_debugfs_mc_info_init(rdev)) {
3029                 DRM_ERROR("Failed to register debugfs file for mc !\n");
3030         }
3031         /* Read BIOS */
3032         if (!radeon_get_bios(rdev)) {
3033                 if (ASIC_IS_AVIVO(rdev))
3034                         return -EINVAL;
3035         }
3036         /* Must be an ATOMBIOS */
3037         if (!rdev->is_atom_bios) {
3038                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3039                 return -EINVAL;
3040         }
3041         r = radeon_atombios_init(rdev);
3042         if (r)
3043                 return r;
3044         /* Post card if necessary */
3045         if (!radeon_card_posted(rdev)) {
3046                 if (!rdev->bios) {
3047                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3048                         return -EINVAL;
3049                 }
3050                 DRM_INFO("GPU not posted. posting now...\n");
3051                 atom_asic_init(rdev->mode_info.atom_context);
3052         }
3053         /* Initialize scratch registers */
3054         r600_scratch_init(rdev);
3055         /* Initialize surface registers */
3056         radeon_surface_init(rdev);
3057         /* Initialize clocks */
3058         radeon_get_clock_info(rdev->ddev);
3059         /* Fence driver */
3060         r = radeon_fence_driver_init(rdev);
3061         if (r)
3062                 return r;
3063         if (rdev->flags & RADEON_IS_AGP) {
3064                 r = radeon_agp_init(rdev);
3065                 if (r)
3066                         radeon_agp_disable(rdev);
3067         }
3068         r = r600_mc_init(rdev);
3069         if (r)
3070                 return r;
3071         /* Memory manager */
3072         r = radeon_bo_init(rdev);
3073         if (r)
3074                 return r;
3075
3076         r = radeon_irq_kms_init(rdev);
3077         if (r)
3078                 return r;
3079
3080         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3081         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3082
3083         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3084         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3085
3086         rdev->ih.ring_obj = NULL;
3087         r600_ih_ring_init(rdev, 64 * 1024);
3088
3089         r = r600_pcie_gart_init(rdev);
3090         if (r)
3091                 return r;
3092
3093         rdev->accel_working = true;
3094         r = r600_startup(rdev);
3095         if (r) {
3096                 dev_err(rdev->dev, "disabling GPU acceleration\n");
3097                 r600_cp_fini(rdev);
3098                 r600_dma_fini(rdev);
3099                 r600_irq_fini(rdev);
3100                 radeon_wb_fini(rdev);
3101                 radeon_ib_pool_fini(rdev);
3102                 radeon_irq_kms_fini(rdev);
3103                 r600_pcie_gart_fini(rdev);
3104                 rdev->accel_working = false;
3105         }
3106
3107         return 0;
3108 }
3109
3110 void r600_fini(struct radeon_device *rdev)
3111 {
3112         r600_audio_fini(rdev);
3113         r600_blit_fini(rdev);
3114         r600_cp_fini(rdev);
3115         r600_dma_fini(rdev);
3116         r600_irq_fini(rdev);
3117         radeon_wb_fini(rdev);
3118         radeon_ib_pool_fini(rdev);
3119         radeon_irq_kms_fini(rdev);
3120         r600_pcie_gart_fini(rdev);
3121         r600_vram_scratch_fini(rdev);
3122         radeon_agp_fini(rdev);
3123         radeon_gem_fini(rdev);
3124         radeon_fence_driver_fini(rdev);
3125         radeon_bo_fini(rdev);
3126         radeon_atombios_fini(rdev);
3127         kfree(rdev->bios);
3128         rdev->bios = NULL;
3129 }
3130
3131
3132 /*
3133  * CS stuff
3134  */
3135 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3136 {
3137         struct radeon_ring *ring = &rdev->ring[ib->ring];
3138         u32 next_rptr;
3139
3140         if (ring->rptr_save_reg) {
3141                 next_rptr = ring->wptr + 3 + 4;
3142                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3143                 radeon_ring_write(ring, ((ring->rptr_save_reg -
3144                                          PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3145                 radeon_ring_write(ring, next_rptr);
3146         } else if (rdev->wb.enabled) {
3147                 next_rptr = ring->wptr + 5 + 4;
3148                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3149                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3150                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3151                 radeon_ring_write(ring, next_rptr);
3152                 radeon_ring_write(ring, 0);
3153         }
3154
3155         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3156         radeon_ring_write(ring,
3157 #ifdef __BIG_ENDIAN
3158                           (2 << 0) |
3159 #endif
3160                           (ib->gpu_addr & 0xFFFFFFFC));
3161         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3162         radeon_ring_write(ring, ib->length_dw);
3163 }
3164
3165 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3166 {
3167         struct radeon_ib ib;
3168         uint32_t scratch;
3169         uint32_t tmp = 0;
3170         unsigned i;
3171         int r;
3172
3173         r = radeon_scratch_get(rdev, &scratch);
3174         if (r) {
3175                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3176                 return r;
3177         }
3178         WREG32(scratch, 0xCAFEDEAD);
3179         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3180         if (r) {
3181                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3182                 goto free_scratch;
3183         }
3184         ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3185         ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3186         ib.ptr[2] = 0xDEADBEEF;
3187         ib.length_dw = 3;
3188         r = radeon_ib_schedule(rdev, &ib, NULL);
3189         if (r) {
3190                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3191                 goto free_ib;
3192         }
3193         r = radeon_fence_wait(ib.fence, false);
3194         if (r) {
3195                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3196                 goto free_ib;
3197         }
3198         for (i = 0; i < rdev->usec_timeout; i++) {
3199                 tmp = RREG32(scratch);
3200                 if (tmp == 0xDEADBEEF)
3201                         break;
3202                 DRM_UDELAY(1);
3203         }
3204         if (i < rdev->usec_timeout) {
3205                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3206         } else {
3207                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3208                           scratch, tmp);
3209                 r = -EINVAL;
3210         }
3211 free_ib:
3212         radeon_ib_free(rdev, &ib);
3213 free_scratch:
3214         radeon_scratch_free(rdev, scratch);
3215         return r;
3216 }
3217
3218 /**
3219  * r600_dma_ib_test - test an IB on the DMA engine
3220  *
3221  * @rdev: radeon_device pointer
3222  * @ring: radeon_ring structure holding ring information
3223  *
3224  * Test a simple IB in the DMA ring (r6xx-SI).
3225  * Returns 0 on success, error on failure.
3226  */
3227 int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3228 {
3229         struct radeon_ib ib;
3230         unsigned i;
3231         int r;
3232         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3233         u32 tmp = 0;
3234
3235         if (!ptr) {
3236                 DRM_ERROR("invalid vram scratch pointer\n");
3237                 return -EINVAL;
3238         }
3239
3240         tmp = 0xCAFEDEAD;
3241         writel(tmp, ptr);
3242
3243         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3244         if (r) {
3245                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3246                 return r;
3247         }
3248
3249         ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3250         ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3251         ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3252         ib.ptr[3] = 0xDEADBEEF;
3253         ib.length_dw = 4;
3254
3255         r = radeon_ib_schedule(rdev, &ib, NULL);
3256         if (r) {
3257                 radeon_ib_free(rdev, &ib);
3258                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3259                 return r;
3260         }
3261         r = radeon_fence_wait(ib.fence, false);
3262         if (r) {
3263                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3264                 return r;
3265         }
3266         for (i = 0; i < rdev->usec_timeout; i++) {
3267                 tmp = readl(ptr);
3268                 if (tmp == 0xDEADBEEF)
3269                         break;
3270                 DRM_UDELAY(1);
3271         }
3272         if (i < rdev->usec_timeout) {
3273                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3274         } else {
3275                 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3276                 r = -EINVAL;
3277         }
3278         radeon_ib_free(rdev, &ib);
3279         return r;
3280 }
3281
3282 /**
3283  * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3284  *
3285  * @rdev: radeon_device pointer
3286  * @ib: IB object to schedule
3287  *
3288  * Schedule an IB in the DMA ring (r6xx-r7xx).
3289  */
3290 void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3291 {
3292         struct radeon_ring *ring = &rdev->ring[ib->ring];
3293
3294         if (rdev->wb.enabled) {
3295                 u32 next_rptr = ring->wptr + 4;
3296                 while ((next_rptr & 7) != 5)
3297                         next_rptr++;
3298                 next_rptr += 3;
3299                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3300                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3301                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3302                 radeon_ring_write(ring, next_rptr);
3303         }
3304
3305         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3306          * Pad as necessary with NOPs.
3307          */
3308         while ((ring->wptr & 7) != 5)
3309                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3310         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3311         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3312         radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3313
3314 }
3315
3316 /*
3317  * Interrupts
3318  *
3319  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
3320  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
3321  * writing to the ring and the GPU consuming, the GPU writes to the ring
3322  * and host consumes.  As the host irq handler processes interrupts, it
3323  * increments the rptr.  When the rptr catches up with the wptr, all the
3324  * current interrupts have been processed.
3325  */
3326
3327 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3328 {
3329         u32 rb_bufsz;
3330
3331         /* Align ring size */
3332         rb_bufsz = drm_order(ring_size / 4);
3333         ring_size = (1 << rb_bufsz) * 4;
3334         rdev->ih.ring_size = ring_size;
3335         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3336         rdev->ih.rptr = 0;
3337 }
3338
3339 int r600_ih_ring_alloc(struct radeon_device *rdev)
3340 {
3341         int r;
3342
3343         /* Allocate ring buffer */
3344         if (rdev->ih.ring_obj == NULL) {
3345                 r = radeon_bo_create(rdev, rdev->ih.ring_size,
3346                                      PAGE_SIZE, true,
3347                                      RADEON_GEM_DOMAIN_GTT,
3348                                      NULL, &rdev->ih.ring_obj);
3349                 if (r) {
3350                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3351                         return r;
3352                 }
3353                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3354                 if (unlikely(r != 0))
3355                         return r;
3356                 r = radeon_bo_pin(rdev->ih.ring_obj,
3357                                   RADEON_GEM_DOMAIN_GTT,
3358                                   &rdev->ih.gpu_addr);
3359                 if (r) {
3360                         radeon_bo_unreserve(rdev->ih.ring_obj);
3361                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3362                         return r;
3363                 }
3364                 r = radeon_bo_kmap(rdev->ih.ring_obj,
3365                                    (void **)&rdev->ih.ring);
3366                 radeon_bo_unreserve(rdev->ih.ring_obj);
3367                 if (r) {
3368                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3369                         return r;
3370                 }
3371         }
3372         return 0;
3373 }
3374
3375 void r600_ih_ring_fini(struct radeon_device *rdev)
3376 {
3377         int r;
3378         if (rdev->ih.ring_obj) {
3379                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3380                 if (likely(r == 0)) {
3381                         radeon_bo_kunmap(rdev->ih.ring_obj);
3382                         radeon_bo_unpin(rdev->ih.ring_obj);
3383                         radeon_bo_unreserve(rdev->ih.ring_obj);
3384                 }
3385                 radeon_bo_unref(&rdev->ih.ring_obj);
3386                 rdev->ih.ring = NULL;
3387                 rdev->ih.ring_obj = NULL;
3388         }
3389 }
3390
3391 void r600_rlc_stop(struct radeon_device *rdev)
3392 {
3393
3394         if ((rdev->family >= CHIP_RV770) &&
3395             (rdev->family <= CHIP_RV740)) {
3396                 /* r7xx asics need to soft reset RLC before halting */
3397                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3398                 RREG32(SRBM_SOFT_RESET);
3399                 mdelay(15);
3400                 WREG32(SRBM_SOFT_RESET, 0);
3401                 RREG32(SRBM_SOFT_RESET);
3402         }
3403
3404         WREG32(RLC_CNTL, 0);
3405 }
3406
3407 static void r600_rlc_start(struct radeon_device *rdev)
3408 {
3409         WREG32(RLC_CNTL, RLC_ENABLE);
3410 }
3411
3412 static int r600_rlc_init(struct radeon_device *rdev)
3413 {
3414         u32 i;
3415         const __be32 *fw_data;
3416
3417         if (!rdev->rlc_fw)
3418                 return -EINVAL;
3419
3420         r600_rlc_stop(rdev);
3421
3422         WREG32(RLC_HB_CNTL, 0);
3423
3424         if (rdev->family == CHIP_ARUBA) {
3425                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3426                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3427         }
3428         if (rdev->family <= CHIP_CAYMAN) {
3429                 WREG32(RLC_HB_BASE, 0);
3430                 WREG32(RLC_HB_RPTR, 0);
3431                 WREG32(RLC_HB_WPTR, 0);
3432         }
3433         if (rdev->family <= CHIP_CAICOS) {
3434                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3435                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3436         }
3437         WREG32(RLC_MC_CNTL, 0);
3438         WREG32(RLC_UCODE_CNTL, 0);
3439
3440         fw_data = (const __be32 *)rdev->rlc_fw->data;
3441         if (rdev->family >= CHIP_ARUBA) {
3442                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
3443                         WREG32(RLC_UCODE_ADDR, i);
3444                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3445                 }
3446         } else if (rdev->family >= CHIP_CAYMAN) {
3447                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
3448                         WREG32(RLC_UCODE_ADDR, i);
3449                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3450                 }
3451         } else if (rdev->family >= CHIP_CEDAR) {
3452                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
3453                         WREG32(RLC_UCODE_ADDR, i);
3454                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3455                 }
3456         } else if (rdev->family >= CHIP_RV770) {
3457                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3458                         WREG32(RLC_UCODE_ADDR, i);
3459                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3460                 }
3461         } else {
3462                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
3463                         WREG32(RLC_UCODE_ADDR, i);
3464                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3465                 }
3466         }
3467         WREG32(RLC_UCODE_ADDR, 0);
3468
3469         r600_rlc_start(rdev);
3470
3471         return 0;
3472 }
3473
3474 static void r600_enable_interrupts(struct radeon_device *rdev)
3475 {
3476         u32 ih_cntl = RREG32(IH_CNTL);
3477         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3478
3479         ih_cntl |= ENABLE_INTR;
3480         ih_rb_cntl |= IH_RB_ENABLE;
3481         WREG32(IH_CNTL, ih_cntl);
3482         WREG32(IH_RB_CNTL, ih_rb_cntl);
3483         rdev->ih.enabled = true;
3484 }
3485
3486 void r600_disable_interrupts(struct radeon_device *rdev)
3487 {
3488         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3489         u32 ih_cntl = RREG32(IH_CNTL);
3490
3491         ih_rb_cntl &= ~IH_RB_ENABLE;
3492         ih_cntl &= ~ENABLE_INTR;
3493         WREG32(IH_RB_CNTL, ih_rb_cntl);
3494         WREG32(IH_CNTL, ih_cntl);
3495         /* set rptr, wptr to 0 */
3496         WREG32(IH_RB_RPTR, 0);
3497         WREG32(IH_RB_WPTR, 0);
3498         rdev->ih.enabled = false;
3499         rdev->ih.rptr = 0;
3500 }
3501
3502 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3503 {
3504         u32 tmp;
3505
3506         WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3507         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3508         WREG32(DMA_CNTL, tmp);
3509         WREG32(GRBM_INT_CNTL, 0);
3510         WREG32(DxMODE_INT_MASK, 0);
3511         WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3512         WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3513         if (ASIC_IS_DCE3(rdev)) {
3514                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3515                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3516                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3517                 WREG32(DC_HPD1_INT_CONTROL, tmp);
3518                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3519                 WREG32(DC_HPD2_INT_CONTROL, tmp);
3520                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3521                 WREG32(DC_HPD3_INT_CONTROL, tmp);
3522                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3523                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3524                 if (ASIC_IS_DCE32(rdev)) {
3525                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3526                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3527                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3528                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3529                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3530                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3531                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3532                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3533                 } else {
3534                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3535                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3536                         tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3537                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3538                 }
3539         } else {
3540                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3541                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3542                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3543                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3544                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3545                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3546                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3547                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3548                 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3549                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3550                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3551                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3552         }
3553 }
3554
3555 int r600_irq_init(struct radeon_device *rdev)
3556 {
3557         int ret = 0;
3558         int rb_bufsz;
3559         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3560
3561         /* allocate ring */
3562         ret = r600_ih_ring_alloc(rdev);
3563         if (ret)
3564                 return ret;
3565
3566         /* disable irqs */
3567         r600_disable_interrupts(rdev);
3568
3569         /* init rlc */
3570         ret = r600_rlc_init(rdev);
3571         if (ret) {
3572                 r600_ih_ring_fini(rdev);
3573                 return ret;
3574         }
3575
3576         /* setup interrupt control */
3577         /* set dummy read address to ring address */
3578         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3579         interrupt_cntl = RREG32(INTERRUPT_CNTL);
3580         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3581          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3582          */
3583         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3584         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3585         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3586         WREG32(INTERRUPT_CNTL, interrupt_cntl);
3587
3588         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3589         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3590
3591         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3592                       IH_WPTR_OVERFLOW_CLEAR |
3593                       (rb_bufsz << 1));
3594
3595         if (rdev->wb.enabled)
3596                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3597
3598         /* set the writeback address whether it's enabled or not */
3599         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3600         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3601
3602         WREG32(IH_RB_CNTL, ih_rb_cntl);
3603
3604         /* set rptr, wptr to 0 */
3605         WREG32(IH_RB_RPTR, 0);
3606         WREG32(IH_RB_WPTR, 0);
3607
3608         /* Default settings for IH_CNTL (disabled at first) */
3609         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3610         /* RPTR_REARM only works if msi's are enabled */
3611         if (rdev->msi_enabled)
3612                 ih_cntl |= RPTR_REARM;
3613         WREG32(IH_CNTL, ih_cntl);
3614
3615         /* force the active interrupt state to all disabled */
3616         if (rdev->family >= CHIP_CEDAR)
3617                 evergreen_disable_interrupt_state(rdev);
3618         else
3619                 r600_disable_interrupt_state(rdev);
3620
3621         /* at this point everything should be setup correctly to enable master */
3622         pci_set_master(rdev->pdev);
3623
3624         /* enable irqs */
3625         r600_enable_interrupts(rdev);
3626
3627         return ret;
3628 }
3629
3630 void r600_irq_suspend(struct radeon_device *rdev)
3631 {
3632         r600_irq_disable(rdev);
3633         r600_rlc_stop(rdev);
3634 }
3635
3636 void r600_irq_fini(struct radeon_device *rdev)
3637 {
3638         r600_irq_suspend(rdev);
3639         r600_ih_ring_fini(rdev);
3640 }
3641
3642 int r600_irq_set(struct radeon_device *rdev)
3643 {
3644         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3645         u32 mode_int = 0;
3646         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
3647         u32 grbm_int_cntl = 0;
3648         u32 hdmi0, hdmi1;
3649         u32 d1grph = 0, d2grph = 0;
3650         u32 dma_cntl;
3651
3652         if (!rdev->irq.installed) {
3653                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3654                 return -EINVAL;
3655         }
3656         /* don't enable anything if the ih is disabled */
3657         if (!rdev->ih.enabled) {
3658                 r600_disable_interrupts(rdev);
3659                 /* force the active interrupt state to all disabled */
3660                 r600_disable_interrupt_state(rdev);
3661                 return 0;
3662         }
3663
3664         if (ASIC_IS_DCE3(rdev)) {
3665                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3666                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3667                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3668                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3669                 if (ASIC_IS_DCE32(rdev)) {
3670                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3671                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3672                         hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3673                         hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3674                 } else {
3675                         hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3676                         hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3677                 }
3678         } else {
3679                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3680                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3681                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3682                 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3683                 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3684         }
3685         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3686
3687         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3688                 DRM_DEBUG("r600_irq_set: sw int\n");
3689                 cp_int_cntl |= RB_INT_ENABLE;
3690                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3691         }
3692
3693         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3694                 DRM_DEBUG("r600_irq_set: sw int dma\n");
3695                 dma_cntl |= TRAP_ENABLE;
3696         }
3697
3698         if (rdev->irq.crtc_vblank_int[0] ||
3699             atomic_read(&rdev->irq.pflip[0])) {
3700                 DRM_DEBUG("r600_irq_set: vblank 0\n");
3701                 mode_int |= D1MODE_VBLANK_INT_MASK;
3702         }
3703         if (rdev->irq.crtc_vblank_int[1] ||
3704             atomic_read(&rdev->irq.pflip[1])) {
3705                 DRM_DEBUG("r600_irq_set: vblank 1\n");
3706                 mode_int |= D2MODE_VBLANK_INT_MASK;
3707         }
3708         if (rdev->irq.hpd[0]) {
3709                 DRM_DEBUG("r600_irq_set: hpd 1\n");
3710                 hpd1 |= DC_HPDx_INT_EN;
3711         }
3712         if (rdev->irq.hpd[1]) {
3713                 DRM_DEBUG("r600_irq_set: hpd 2\n");
3714                 hpd2 |= DC_HPDx_INT_EN;
3715         }
3716         if (rdev->irq.hpd[2]) {
3717                 DRM_DEBUG("r600_irq_set: hpd 3\n");
3718                 hpd3 |= DC_HPDx_INT_EN;
3719         }
3720         if (rdev->irq.hpd[3]) {
3721                 DRM_DEBUG("r600_irq_set: hpd 4\n");
3722                 hpd4 |= DC_HPDx_INT_EN;
3723         }
3724         if (rdev->irq.hpd[4]) {
3725                 DRM_DEBUG("r600_irq_set: hpd 5\n");
3726                 hpd5 |= DC_HPDx_INT_EN;
3727         }
3728         if (rdev->irq.hpd[5]) {
3729                 DRM_DEBUG("r600_irq_set: hpd 6\n");
3730                 hpd6 |= DC_HPDx_INT_EN;
3731         }
3732         if (rdev->irq.afmt[0]) {
3733                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3734                 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3735         }
3736         if (rdev->irq.afmt[1]) {
3737                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3738                 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3739         }
3740
3741         WREG32(CP_INT_CNTL, cp_int_cntl);
3742         WREG32(DMA_CNTL, dma_cntl);
3743         WREG32(DxMODE_INT_MASK, mode_int);
3744         WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
3745         WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
3746         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3747         if (ASIC_IS_DCE3(rdev)) {
3748                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3749                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3750                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3751                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3752                 if (ASIC_IS_DCE32(rdev)) {
3753                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
3754                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
3755                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
3756                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
3757                 } else {
3758                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3759                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3760                 }
3761         } else {
3762                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
3763                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
3764                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
3765                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3766                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3767         }
3768
3769         return 0;
3770 }
3771
3772 static void r600_irq_ack(struct radeon_device *rdev)
3773 {
3774         u32 tmp;
3775
3776         if (ASIC_IS_DCE3(rdev)) {
3777                 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
3778                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
3779                 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
3780                 if (ASIC_IS_DCE32(rdev)) {
3781                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
3782                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
3783                 } else {
3784                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3785                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
3786                 }
3787         } else {
3788                 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3789                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3790                 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
3791                 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3792                 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
3793         }
3794         rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
3795         rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
3796
3797         if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3798                 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3799         if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3800                 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3801         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
3802                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3803         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
3804                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3805         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
3806                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3807         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
3808                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3809         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3810                 if (ASIC_IS_DCE3(rdev)) {
3811                         tmp = RREG32(DC_HPD1_INT_CONTROL);
3812                         tmp |= DC_HPDx_INT_ACK;
3813                         WREG32(DC_HPD1_INT_CONTROL, tmp);
3814                 } else {
3815                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
3816                         tmp |= DC_HPDx_INT_ACK;
3817                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3818                 }
3819         }
3820         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3821                 if (ASIC_IS_DCE3(rdev)) {
3822                         tmp = RREG32(DC_HPD2_INT_CONTROL);
3823                         tmp |= DC_HPDx_INT_ACK;
3824                         WREG32(DC_HPD2_INT_CONTROL, tmp);
3825                 } else {
3826                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
3827                         tmp |= DC_HPDx_INT_ACK;
3828                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3829                 }
3830         }
3831         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3832                 if (ASIC_IS_DCE3(rdev)) {
3833                         tmp = RREG32(DC_HPD3_INT_CONTROL);
3834                         tmp |= DC_HPDx_INT_ACK;
3835                         WREG32(DC_HPD3_INT_CONTROL, tmp);
3836                 } else {
3837                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
3838                         tmp |= DC_HPDx_INT_ACK;
3839                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3840                 }
3841         }
3842         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3843                 tmp = RREG32(DC_HPD4_INT_CONTROL);
3844                 tmp |= DC_HPDx_INT_ACK;
3845                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3846         }
3847         if (ASIC_IS_DCE32(rdev)) {
3848                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3849                         tmp = RREG32(DC_HPD5_INT_CONTROL);
3850                         tmp |= DC_HPDx_INT_ACK;
3851                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3852                 }
3853                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3854                         tmp = RREG32(DC_HPD5_INT_CONTROL);
3855                         tmp |= DC_HPDx_INT_ACK;
3856                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3857                 }
3858                 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
3859                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
3860                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3861                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3862                 }
3863                 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
3864                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
3865                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3866                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3867                 }
3868         } else {
3869                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
3870                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
3871                         tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3872                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3873                 }
3874                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
3875                         if (ASIC_IS_DCE3(rdev)) {
3876                                 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
3877                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3878                                 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3879                         } else {
3880                                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
3881                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3882                                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3883                         }
3884                 }
3885         }
3886 }
3887
3888 void r600_irq_disable(struct radeon_device *rdev)
3889 {
3890         r600_disable_interrupts(rdev);
3891         /* Wait and acknowledge irq */
3892         mdelay(1);
3893         r600_irq_ack(rdev);
3894         r600_disable_interrupt_state(rdev);
3895 }
3896
3897 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
3898 {
3899         u32 wptr, tmp;
3900
3901         if (rdev->wb.enabled)
3902                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3903         else
3904                 wptr = RREG32(IH_RB_WPTR);
3905
3906         if (wptr & RB_OVERFLOW) {
3907                 /* When a ring buffer overflow happen start parsing interrupt
3908                  * from the last not overwritten vector (wptr + 16). Hopefully
3909                  * this should allow us to catchup.
3910                  */
3911                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3912                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3913                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3914                 tmp = RREG32(IH_RB_CNTL);
3915                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3916                 WREG32(IH_RB_CNTL, tmp);
3917         }
3918         return (wptr & rdev->ih.ptr_mask);
3919 }
3920
3921 /*        r600 IV Ring
3922  * Each IV ring entry is 128 bits:
3923  * [7:0]    - interrupt source id
3924  * [31:8]   - reserved
3925  * [59:32]  - interrupt source data
3926  * [127:60]  - reserved
3927  *
3928  * The basic interrupt vector entries
3929  * are decoded as follows:
3930  * src_id  src_data  description
3931  *      1         0  D1 Vblank
3932  *      1         1  D1 Vline
3933  *      5         0  D2 Vblank
3934  *      5         1  D2 Vline
3935  *     19         0  FP Hot plug detection A
3936  *     19         1  FP Hot plug detection B
3937  *     19         2  DAC A auto-detection
3938  *     19         3  DAC B auto-detection
3939  *     21         4  HDMI block A
3940  *     21         5  HDMI block B
3941  *    176         -  CP_INT RB
3942  *    177         -  CP_INT IB1
3943  *    178         -  CP_INT IB2
3944  *    181         -  EOP Interrupt
3945  *    233         -  GUI Idle
3946  *
3947  * Note, these are based on r600 and may need to be
3948  * adjusted or added to on newer asics
3949  */
3950
3951 int r600_irq_process(struct radeon_device *rdev)
3952 {
3953         u32 wptr;
3954         u32 rptr;
3955         u32 src_id, src_data;
3956         u32 ring_index;
3957         bool queue_hotplug = false;
3958         bool queue_hdmi = false;
3959
3960         if (!rdev->ih.enabled || rdev->shutdown)
3961                 return IRQ_NONE;
3962
3963         /* No MSIs, need a dummy read to flush PCI DMAs */
3964         if (!rdev->msi_enabled)
3965                 RREG32(IH_RB_WPTR);
3966
3967         wptr = r600_get_ih_wptr(rdev);
3968
3969 restart_ih:
3970         /* is somebody else already processing irqs? */
3971         if (atomic_xchg(&rdev->ih.lock, 1))
3972                 return IRQ_NONE;
3973
3974         rptr = rdev->ih.rptr;
3975         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3976
3977         /* Order reading of wptr vs. reading of IH ring data */
3978         rmb();
3979
3980         /* display interrupts */
3981         r600_irq_ack(rdev);
3982
3983         while (rptr != wptr) {
3984                 /* wptr/rptr are in bytes! */
3985                 ring_index = rptr / 4;
3986                 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3987                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3988
3989                 switch (src_id) {
3990                 case 1: /* D1 vblank/vline */
3991                         switch (src_data) {
3992                         case 0: /* D1 vblank */
3993                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
3994                                         if (rdev->irq.crtc_vblank_int[0]) {
3995                                                 drm_handle_vblank(rdev->ddev, 0);
3996                                                 rdev->pm.vblank_sync = true;
3997                                                 wake_up(&rdev->irq.vblank_queue);
3998                                         }
3999                                         if (atomic_read(&rdev->irq.pflip[0]))
4000                                                 radeon_crtc_handle_flip(rdev, 0);
4001                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4002                                         DRM_DEBUG("IH: D1 vblank\n");
4003                                 }
4004                                 break;
4005                         case 1: /* D1 vline */
4006                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
4007                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4008                                         DRM_DEBUG("IH: D1 vline\n");
4009                                 }
4010                                 break;
4011                         default:
4012                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4013                                 break;
4014                         }
4015                         break;
4016                 case 5: /* D2 vblank/vline */
4017                         switch (src_data) {
4018                         case 0: /* D2 vblank */
4019                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
4020                                         if (rdev->irq.crtc_vblank_int[1]) {
4021                                                 drm_handle_vblank(rdev->ddev, 1);
4022                                                 rdev->pm.vblank_sync = true;
4023                                                 wake_up(&rdev->irq.vblank_queue);
4024                                         }
4025                                         if (atomic_read(&rdev->irq.pflip[1]))
4026                                                 radeon_crtc_handle_flip(rdev, 1);
4027                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
4028                                         DRM_DEBUG("IH: D2 vblank\n");
4029                                 }
4030                                 break;
4031                         case 1: /* D1 vline */
4032                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
4033                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
4034                                         DRM_DEBUG("IH: D2 vline\n");
4035                                 }
4036                                 break;
4037                         default:
4038                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4039                                 break;
4040                         }
4041                         break;
4042                 case 19: /* HPD/DAC hotplug */
4043                         switch (src_data) {
4044                         case 0:
4045                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4046                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
4047                                         queue_hotplug = true;
4048                                         DRM_DEBUG("IH: HPD1\n");
4049                                 }
4050                                 break;
4051                         case 1:
4052                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4053                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
4054                                         queue_hotplug = true;
4055                                         DRM_DEBUG("IH: HPD2\n");
4056                                 }
4057                                 break;
4058                         case 4:
4059                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4060                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
4061                                         queue_hotplug = true;
4062                                         DRM_DEBUG("IH: HPD3\n");
4063                                 }
4064                                 break;
4065                         case 5:
4066                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4067                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
4068                                         queue_hotplug = true;
4069                                         DRM_DEBUG("IH: HPD4\n");
4070                                 }
4071                                 break;
4072                         case 10:
4073                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4074                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
4075                                         queue_hotplug = true;
4076                                         DRM_DEBUG("IH: HPD5\n");
4077                                 }
4078                                 break;
4079                         case 12:
4080                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4081                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
4082                                         queue_hotplug = true;
4083                                         DRM_DEBUG("IH: HPD6\n");
4084                                 }
4085                                 break;
4086                         default:
4087                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4088                                 break;
4089                         }
4090                         break;
4091                 case 21: /* hdmi */
4092                         switch (src_data) {
4093                         case 4:
4094                                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4095                                         rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4096                                         queue_hdmi = true;
4097                                         DRM_DEBUG("IH: HDMI0\n");
4098                                 }
4099                                 break;
4100                         case 5:
4101                                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4102                                         rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4103                                         queue_hdmi = true;
4104                                         DRM_DEBUG("IH: HDMI1\n");
4105                                 }
4106                                 break;
4107                         default:
4108                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4109                                 break;
4110                         }
4111                         break;
4112                 case 176: /* CP_INT in ring buffer */
4113                 case 177: /* CP_INT in IB1 */
4114                 case 178: /* CP_INT in IB2 */
4115                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4116                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4117                         break;
4118                 case 181: /* CP EOP event */
4119                         DRM_DEBUG("IH: CP EOP\n");
4120                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4121                         break;
4122                 case 224: /* DMA trap event */
4123                         DRM_DEBUG("IH: DMA trap\n");
4124                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4125                         break;
4126                 case 233: /* GUI IDLE */
4127                         DRM_DEBUG("IH: GUI idle\n");
4128                         break;
4129                 default:
4130                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4131                         break;
4132                 }
4133
4134                 /* wptr/rptr are in bytes! */
4135                 rptr += 16;
4136                 rptr &= rdev->ih.ptr_mask;
4137         }
4138         if (queue_hotplug)
4139                 schedule_work(&rdev->hotplug_work);
4140         if (queue_hdmi)
4141                 schedule_work(&rdev->audio_work);
4142         rdev->ih.rptr = rptr;
4143         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4144         atomic_set(&rdev->ih.lock, 0);
4145
4146         /* make sure wptr hasn't changed while processing */
4147         wptr = r600_get_ih_wptr(rdev);
4148         if (wptr != rptr)
4149                 goto restart_ih;
4150
4151         return IRQ_HANDLED;
4152 }
4153
4154 /*
4155  * Debugfs info
4156  */
4157 #if defined(CONFIG_DEBUG_FS)
4158
4159 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4160 {
4161         struct drm_info_node *node = (struct drm_info_node *) m->private;
4162         struct drm_device *dev = node->minor->dev;
4163         struct radeon_device *rdev = dev->dev_private;
4164
4165         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4166         DREG32_SYS(m, rdev, VM_L2_STATUS);
4167         return 0;
4168 }
4169
4170 static struct drm_info_list r600_mc_info_list[] = {
4171         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
4172 };
4173 #endif
4174
4175 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4176 {
4177 #if defined(CONFIG_DEBUG_FS)
4178         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4179 #else
4180         return 0;
4181 #endif
4182 }
4183
4184 /**
4185  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4186  * rdev: radeon device structure
4187  * bo: buffer object struct which userspace is waiting for idle
4188  *
4189  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4190  * through ring buffer, this leads to corruption in rendering, see
4191  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4192  * directly perform HDP flush by writing register through MMIO.
4193  */
4194 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4195 {
4196         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
4197          * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4198          * This seems to cause problems on some AGP cards. Just use the old
4199          * method for them.
4200          */
4201         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4202             rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4203                 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
4204                 u32 tmp;
4205
4206                 WREG32(HDP_DEBUG1, 0);
4207                 tmp = readl((void __iomem *)ptr);
4208         } else
4209                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4210 }
4211
4212 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4213 {
4214         u32 link_width_cntl, mask, target_reg;
4215
4216         if (rdev->flags & RADEON_IS_IGP)
4217                 return;
4218
4219         if (!(rdev->flags & RADEON_IS_PCIE))
4220                 return;
4221
4222         /* x2 cards have a special sequence */
4223         if (ASIC_IS_X2(rdev))
4224                 return;
4225
4226         /* FIXME wait for idle */
4227
4228         switch (lanes) {
4229         case 0:
4230                 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4231                 break;
4232         case 1:
4233                 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4234                 break;
4235         case 2:
4236                 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4237                 break;
4238         case 4:
4239                 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4240                 break;
4241         case 8:
4242                 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4243                 break;
4244         case 12:
4245                 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4246                 break;
4247         case 16:
4248         default:
4249                 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4250                 break;
4251         }
4252
4253         link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4254
4255         if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
4256             (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
4257                 return;
4258
4259         if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
4260                 return;
4261
4262         link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
4263                              RADEON_PCIE_LC_RECONFIG_NOW |
4264                              R600_PCIE_LC_RENEGOTIATE_EN |
4265                              R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4266         link_width_cntl |= mask;
4267
4268         WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4269
4270         /* some northbridges can renegotiate the link rather than requiring
4271          * a complete re-config.
4272          * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
4273          */
4274         if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
4275                 link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
4276         else
4277                 link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
4278
4279         WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
4280                                                        RADEON_PCIE_LC_RECONFIG_NOW));
4281
4282         if (rdev->family >= CHIP_RV770)
4283                 target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
4284         else
4285                 target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
4286
4287         /* wait for lane set to complete */
4288         link_width_cntl = RREG32(target_reg);
4289         while (link_width_cntl == 0xffffffff)
4290                 link_width_cntl = RREG32(target_reg);
4291
4292 }
4293
4294 int r600_get_pcie_lanes(struct radeon_device *rdev)
4295 {
4296         u32 link_width_cntl;
4297
4298         if (rdev->flags & RADEON_IS_IGP)
4299                 return 0;
4300
4301         if (!(rdev->flags & RADEON_IS_PCIE))
4302                 return 0;
4303
4304         /* x2 cards have a special sequence */
4305         if (ASIC_IS_X2(rdev))
4306                 return 0;
4307
4308         /* FIXME wait for idle */
4309
4310         link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4311
4312         switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4313         case RADEON_PCIE_LC_LINK_WIDTH_X0:
4314                 return 0;
4315         case RADEON_PCIE_LC_LINK_WIDTH_X1:
4316                 return 1;
4317         case RADEON_PCIE_LC_LINK_WIDTH_X2:
4318                 return 2;
4319         case RADEON_PCIE_LC_LINK_WIDTH_X4:
4320                 return 4;
4321         case RADEON_PCIE_LC_LINK_WIDTH_X8:
4322                 return 8;
4323         case RADEON_PCIE_LC_LINK_WIDTH_X16:
4324         default:
4325                 return 16;
4326         }
4327 }
4328
4329 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4330 {
4331         u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4332         u16 link_cntl2;
4333         u32 mask;
4334         int ret;
4335
4336         if (radeon_pcie_gen2 == 0)
4337                 return;
4338
4339         if (rdev->flags & RADEON_IS_IGP)
4340                 return;
4341
4342         if (!(rdev->flags & RADEON_IS_PCIE))
4343                 return;
4344
4345         /* x2 cards have a special sequence */
4346         if (ASIC_IS_X2(rdev))
4347                 return;
4348
4349         /* only RV6xx+ chips are supported */
4350         if (rdev->family <= CHIP_R600)
4351                 return;
4352
4353         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
4354         if (ret != 0)
4355                 return;
4356
4357         if (!(mask & DRM_PCIE_SPEED_50))
4358                 return;
4359
4360         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4361         if (speed_cntl & LC_CURRENT_DATA_RATE) {
4362                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4363                 return;
4364         }
4365
4366         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4367
4368         /* 55 nm r6xx asics */
4369         if ((rdev->family == CHIP_RV670) ||
4370             (rdev->family == CHIP_RV620) ||
4371             (rdev->family == CHIP_RV635)) {
4372                 /* advertise upconfig capability */
4373                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4374                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4375                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4376                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4377                 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4378                         lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4379                         link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4380                                              LC_RECONFIG_ARC_MISSING_ESCAPE);
4381                         link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4382                         WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4383                 } else {
4384                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4385                         WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4386                 }
4387         }
4388
4389         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4390         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4391             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4392
4393                 /* 55 nm r6xx asics */
4394                 if ((rdev->family == CHIP_RV670) ||
4395                     (rdev->family == CHIP_RV620) ||
4396                     (rdev->family == CHIP_RV635)) {
4397                         WREG32(MM_CFGREGS_CNTL, 0x8);
4398                         link_cntl2 = RREG32(0x4088);
4399                         WREG32(MM_CFGREGS_CNTL, 0);
4400                         /* not supported yet */
4401                         if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4402                                 return;
4403                 }
4404
4405                 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4406                 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4407                 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4408                 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4409                 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4410                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4411
4412                 tmp = RREG32(0x541c);
4413                 WREG32(0x541c, tmp | 0x8);
4414                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4415                 link_cntl2 = RREG16(0x4088);
4416                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4417                 link_cntl2 |= 0x2;
4418                 WREG16(0x4088, link_cntl2);
4419                 WREG32(MM_CFGREGS_CNTL, 0);
4420
4421                 if ((rdev->family == CHIP_RV670) ||
4422                     (rdev->family == CHIP_RV620) ||
4423                     (rdev->family == CHIP_RV635)) {
4424                         training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
4425                         training_cntl &= ~LC_POINT_7_PLUS_EN;
4426                         WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
4427                 } else {
4428                         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4429                         speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4430                         WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4431                 }
4432
4433                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4434                 speed_cntl |= LC_GEN2_EN_STRAP;
4435                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4436
4437         } else {
4438                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4439                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4440                 if (1)
4441                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4442                 else
4443                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4444                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4445         }
4446 }
4447
4448 /**
4449  * r600_get_gpu_clock - return GPU clock counter snapshot
4450  *
4451  * @rdev: radeon_device pointer
4452  *
4453  * Fetches a GPU clock counter snapshot (R6xx-cayman).
4454  * Returns the 64 bit clock counter snapshot.
4455  */
4456 uint64_t r600_get_gpu_clock(struct radeon_device *rdev)
4457 {
4458         uint64_t clock;
4459
4460         mutex_lock(&rdev->gpu_clock_mutex);
4461         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4462         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4463                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4464         mutex_unlock(&rdev->gpu_clock_mutex);
4465         return clock;
4466 }