sys/dev/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  *
  24  * $FreeBSD: head/sys/dev/drm2/radeon/evergreen.c 254885 2013-08-25 19:37:15Z dumbbell $
  25  */
  26
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <uapi_drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36
  37 #define EVERGREEN_PFP_UCODE_SIZE 1120
  38 #define EVERGREEN_PM4_UCODE_SIZE 1376
  39
  40 static const u32 crtc_offsets[6] =
  41 {
  42         EVERGREEN_CRTC0_REGISTER_OFFSET,
  43         EVERGREEN_CRTC1_REGISTER_OFFSET,
  44         EVERGREEN_CRTC2_REGISTER_OFFSET,
  45         EVERGREEN_CRTC3_REGISTER_OFFSET,
  46         EVERGREEN_CRTC4_REGISTER_OFFSET,
  47         EVERGREEN_CRTC5_REGISTER_OFFSET
  48 };
  49
  50 static void evergreen_gpu_init(struct radeon_device *rdev);
  51 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
  52 void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
  53 bool evergreen_is_display_hung(struct radeon_device *rdev);
  54
  55 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
  56                              unsigned *bankh, unsigned *mtaspect,
  57                              unsigned *tile_split)
  58 {
  59         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
  60         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
  61         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
  62         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
  63         switch (*bankw) {
  64         default:
  65         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
  66         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
  67         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
  68         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
  69         }
  70         switch (*bankh) {
  71         default:
  72         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
  73         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
  74         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
  75         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
  76         }
  77         switch (*mtaspect) {
  78         default:
  79         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
  80         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
  81         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
  82         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
  83         }
  84 }
  85
  86 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
  87 {
  88         u16 ctl, v;
  89         int err, cap;
  90
  91         err = pci_find_extcap(rdev->dev, PCIY_EXPRESS, &cap);
  92         if (err)
  93                 return;
  94
  95         cap += PCIER_DEVCTRL;
  96
  97         ctl = pci_read_config(rdev->dev, cap, 2);
  98
  99         v = (ctl & PCIEM_DEVCTL_MAX_READRQ_MASK) >> 12;
 100
 101         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
 102          * to avoid hangs or perfomance issues
 103          */
 104         if ((v == 0) || (v == 6) || (v == 7)) {
 105                 ctl &= ~PCIEM_DEVCTL_MAX_READRQ_MASK;
 106                 ctl |= (2 << 12);
 107                 pci_write_config(rdev->dev, cap, ctl, 2);
 108         }
 109 }
 110
 111 /**
 112  * dce4_wait_for_vblank - vblank wait asic callback.
 113  *
 114  * @rdev: radeon_device pointer
 115  * @crtc: crtc to wait for vblank on
 116  *
 117  * Wait for vblank on the requested crtc (evergreen+).
 118  */
 119 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
 120 {
 121         int i;
 122
 123         if (crtc >= rdev->num_crtc)
 124                 return;
 125
 126         if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
 127                 for (i = 0; i < rdev->usec_timeout; i++) {
 128                         if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
 129                                 break;
 130                         DRM_UDELAY(1);
 131                 }
 132                 for (i = 0; i < rdev->usec_timeout; i++) {
 133                         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
 134                                 break;
 135                         DRM_UDELAY(1);
 136                 }
 137         }
 138 }
 139
 140 /**
 141  * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
 142  *
 143  * @rdev: radeon_device pointer
 144  * @crtc: crtc to prepare for pageflip on
 145  *
 146  * Pre-pageflip callback (evergreen+).
 147  * Enables the pageflip irq (vblank irq).
 148  */
 149 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
 150 {
 151         /* enable the pflip int */
 152         radeon_irq_kms_pflip_irq_get(rdev, crtc);
 153 }
 154
 155 /**
 156  * evergreen_post_page_flip - pos-pageflip callback.
 157  *
 158  * @rdev: radeon_device pointer
 159  * @crtc: crtc to cleanup pageflip on
 160  *
 161  * Post-pageflip callback (evergreen+).
 162  * Disables the pageflip irq (vblank irq).
 163  */
 164 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
 165 {
 166         /* disable the pflip int */
 167         radeon_irq_kms_pflip_irq_put(rdev, crtc);
 168 }
 169
 170 /**
 171  * evergreen_page_flip - pageflip callback.
 172  *
 173  * @rdev: radeon_device pointer
 174  * @crtc_id: crtc to cleanup pageflip on
 175  * @crtc_base: new address of the crtc (GPU MC address)
 176  *
 177  * Does the actual pageflip (evergreen+).
 178  * During vblank we take the crtc lock and wait for the update_pending
 179  * bit to go high, when it does, we release the lock, and allow the
 180  * double buffered update to take place.
 181  * Returns the current update pending status.
 182  */
 183 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 184 {
 185         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 186         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
 187         int i;
 188
 189         /* Lock the graphics update lock */
 190         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
 191         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 192
 193         /* update the scanout addresses */
 194         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
 195                upper_32_bits(crtc_base));
 196         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 197                (u32)crtc_base);
 198
 199         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
 200                upper_32_bits(crtc_base));
 201         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 202                (u32)crtc_base);
 203
 204         /* Wait for update_pending to go high. */
 205         for (i = 0; i < rdev->usec_timeout; i++) {
 206                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
 207                         break;
 208                 DRM_UDELAY(1);
 209         }
 210         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 211
 212         /* Unlock the lock, so double-buffering can take place inside vblank */
 213         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
 214         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 215
 216         /* Return current update_pending status: */
 217         return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
 218 }
 219
 220 /* get temperature in millidegrees */
 221 int evergreen_get_temp(struct radeon_device *rdev)
 222 {
 223         u32 temp, toffset;
 224         int actual_temp = 0;
 225
 226         if (rdev->family == CHIP_JUNIPER) {
 227                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
 228                         TOFFSET_SHIFT;
 229                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
 230                         TS0_ADC_DOUT_SHIFT;
 231
 232                 if (toffset & 0x100)
 233                         actual_temp = temp / 2 - (0x200 - toffset);
 234                 else
 235                         actual_temp = temp / 2 + toffset;
 236
 237                 actual_temp = actual_temp * 1000;
 238
 239         } else {
 240                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 241                         ASIC_T_SHIFT;
 242
 243                 if (temp & 0x400)
 244                         actual_temp = -256;
 245                 else if (temp & 0x200)
 246                         actual_temp = 255;
 247                 else if (temp & 0x100) {
 248                         actual_temp = temp & 0x1ff;
 249                         actual_temp |= ~0x1ff;
 250                 } else
 251                         actual_temp = temp & 0xff;
 252
 253                 actual_temp = (actual_temp * 1000) / 2;
 254         }
 255
 256         return actual_temp;
 257 }
 258
 259 int sumo_get_temp(struct radeon_device *rdev)
 260 {
 261         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
 262         int actual_temp = temp - 49;
 263
 264         return actual_temp * 1000;
 265 }
 266
 267 /**
 268  * sumo_pm_init_profile - Initialize power profiles callback.
 269  *
 270  * @rdev: radeon_device pointer
 271  *
 272  * Initialize the power states used in profile mode
 273  * (sumo, trinity, SI).
 274  * Used for profile mode only.
 275  */
 276 void sumo_pm_init_profile(struct radeon_device *rdev)
 277 {
 278         int idx;
 279
 280         /* default */
 281         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 282         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 283         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 284         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 285
 286         /* low,mid sh/mh */
 287         if (rdev->flags & RADEON_IS_MOBILITY)
 288                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
 289         else
 290                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 291
 292         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
 293         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
 294         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 295         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 296
 297         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
 298         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
 299         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 300         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 301
 302         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
 303         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
 304         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 305         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 306
 307         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
 308         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
 309         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 310         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 311
 312         /* high sh/mh */
 313         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 314         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
 315         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
 316         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 317         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
 318                 rdev->pm.power_state[idx].num_clock_modes - 1;
 319
 320         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
 321         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
 322         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 323         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
 324                 rdev->pm.power_state[idx].num_clock_modes - 1;
 325 }
 326
 327 /**
 328  * btc_pm_init_profile - Initialize power profiles callback.
 329  *
 330  * @rdev: radeon_device pointer
 331  *
 332  * Initialize the power states used in profile mode
 333  * (BTC, cayman).
 334  * Used for profile mode only.
 335  */
 336 void btc_pm_init_profile(struct radeon_device *rdev)
 337 {
 338         int idx;
 339
 340         /* default */
 341         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 342         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 343         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 344         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 345         /* starting with BTC, there is one state that is used for both
 346          * MH and SH.  Difference is that we always use the high clock index for
 347          * mclk.
 348          */
 349         if (rdev->flags & RADEON_IS_MOBILITY)
 350                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
 351         else
 352                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 353         /* low sh */
 354         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
 355         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
 356         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 357         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 358         /* mid sh */
 359         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
 360         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
 361         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 362         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 363         /* high sh */
 364         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
 365         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
 366         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 367         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 368         /* low mh */
 369         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
 370         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
 371         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 372         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 373         /* mid mh */
 374         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
 375         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
 376         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 377         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 378         /* high mh */
 379         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
 380         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
 381         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 382         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 383 }
 384
 385 /**
 386  * evergreen_pm_misc - set additional pm hw parameters callback.
 387  *
 388  * @rdev: radeon_device pointer
 389  *
 390  * Set non-clock parameters associated with a power state
 391  * (voltage, etc.) (evergreen+).
 392  */
 393 void evergreen_pm_misc(struct radeon_device *rdev)
 394 {
 395         int req_ps_idx = rdev->pm.requested_power_state_index;
 396         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 397         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 398         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 399
 400         if (voltage->type == VOLTAGE_SW) {
 401                 /* 0xff01 is a flag rather then an actual voltage */
 402                 if (voltage->voltage == 0xff01)
 403                         return;
 404                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
 405                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 406                         rdev->pm.current_vddc = voltage->voltage;
 407                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
 408                 }
 409
 410                 /* starting with BTC, there is one state that is used for both
 411                  * MH and SH.  Difference is that we always use the high clock index for
 412                  * mclk and vddci.
 413                  */
 414                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
 415                     (rdev->family >= CHIP_BARTS) &&
 416                     rdev->pm.active_crtc_count &&
 417                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
 418                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
 419                         voltage = &rdev->pm.power_state[req_ps_idx].
 420                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
 421
 422                 /* 0xff01 is a flag rather then an actual voltage */
 423                 if (voltage->vddci == 0xff01)
 424                         return;
 425                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
 426                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
 427                         rdev->pm.current_vddci = voltage->vddci;
 428                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
 429                 }
 430         }
 431 }
 432
 433 /**
 434  * evergreen_pm_prepare - pre-power state change callback.
 435  *
 436  * @rdev: radeon_device pointer
 437  *
 438  * Prepare for a power state change (evergreen+).
 439  */
 440 void evergreen_pm_prepare(struct radeon_device *rdev)
 441 {
 442         struct drm_device *ddev = rdev->ddev;
 443         struct drm_crtc *crtc;
 444         struct radeon_crtc *radeon_crtc;
 445         u32 tmp;
 446
 447         /* disable any active CRTCs */
 448         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
 449                 radeon_crtc = to_radeon_crtc(crtc);
 450                 if (radeon_crtc->enabled) {
 451                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
 452                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
 453                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
 454                 }
 455         }
 456 }
 457
 458 /**
 459  * evergreen_pm_finish - post-power state change callback.
 460  *
 461  * @rdev: radeon_device pointer
 462  *
 463  * Clean up after a power state change (evergreen+).
 464  */
 465 void evergreen_pm_finish(struct radeon_device *rdev)
 466 {
 467         struct drm_device *ddev = rdev->ddev;
 468         struct drm_crtc *crtc;
 469         struct radeon_crtc *radeon_crtc;
 470         u32 tmp;
 471
 472         /* enable any active CRTCs */
 473         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
 474                 radeon_crtc = to_radeon_crtc(crtc);
 475                 if (radeon_crtc->enabled) {
 476                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
 477                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
 478                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
 479                 }
 480         }
 481 }
 482
 483 /**
 484  * evergreen_hpd_sense - hpd sense callback.
 485  *
 486  * @rdev: radeon_device pointer
 487  * @hpd: hpd (hotplug detect) pin
 488  *
 489  * Checks if a digital monitor is connected (evergreen+).
 490  * Returns true if connected, false if not connected.
 491  */
 492 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 493 {
 494         bool connected = false;
 495
 496         switch (hpd) {
 497         case RADEON_HPD_1:
 498                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 499                         connected = true;
 500                 break;
 501         case RADEON_HPD_2:
 502                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 503                         connected = true;
 504                 break;
 505         case RADEON_HPD_3:
 506                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 507                         connected = true;
 508                 break;
 509         case RADEON_HPD_4:
 510                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 511                         connected = true;
 512                 break;
 513         case RADEON_HPD_5:
 514                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 515                         connected = true;
 516                 break;
 517         case RADEON_HPD_6:
 518                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 519                         connected = true;
 520                         break;
 521         default:
 522                 break;
 523         }
 524
 525         return connected;
 526 }
 527
 528 /**
 529  * evergreen_hpd_set_polarity - hpd set polarity callback.
 530  *
 531  * @rdev: radeon_device pointer
 532  * @hpd: hpd (hotplug detect) pin
 533  *
 534  * Set the polarity of the hpd pin (evergreen+).
 535  */
 536 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
 537                                 enum radeon_hpd_id hpd)
 538 {
 539         u32 tmp;
 540         bool connected = evergreen_hpd_sense(rdev, hpd);
 541
 542         switch (hpd) {
 543         case RADEON_HPD_1:
 544                 tmp = RREG32(DC_HPD1_INT_CONTROL);
 545                 if (connected)
 546                         tmp &= ~DC_HPDx_INT_POLARITY;
 547                 else
 548                         tmp |= DC_HPDx_INT_POLARITY;
 549                 WREG32(DC_HPD1_INT_CONTROL, tmp);
 550                 break;
 551         case RADEON_HPD_2:
 552                 tmp = RREG32(DC_HPD2_INT_CONTROL);
 553                 if (connected)
 554                         tmp &= ~DC_HPDx_INT_POLARITY;
 555                 else
 556                         tmp |= DC_HPDx_INT_POLARITY;
 557                 WREG32(DC_HPD2_INT_CONTROL, tmp);
 558                 break;
 559         case RADEON_HPD_3:
 560                 tmp = RREG32(DC_HPD3_INT_CONTROL);
 561                 if (connected)
 562                         tmp &= ~DC_HPDx_INT_POLARITY;
 563                 else
 564                         tmp |= DC_HPDx_INT_POLARITY;
 565                 WREG32(DC_HPD3_INT_CONTROL, tmp);
 566                 break;
 567         case RADEON_HPD_4:
 568                 tmp = RREG32(DC_HPD4_INT_CONTROL);
 569                 if (connected)
 570                         tmp &= ~DC_HPDx_INT_POLARITY;
 571                 else
 572                         tmp |= DC_HPDx_INT_POLARITY;
 573                 WREG32(DC_HPD4_INT_CONTROL, tmp);
 574                 break;
 575         case RADEON_HPD_5:
 576                 tmp = RREG32(DC_HPD5_INT_CONTROL);
 577                 if (connected)
 578                         tmp &= ~DC_HPDx_INT_POLARITY;
 579                 else
 580                         tmp |= DC_HPDx_INT_POLARITY;
 581                 WREG32(DC_HPD5_INT_CONTROL, tmp);
 582                         break;
 583         case RADEON_HPD_6:
 584                 tmp = RREG32(DC_HPD6_INT_CONTROL);
 585                 if (connected)
 586                         tmp &= ~DC_HPDx_INT_POLARITY;
 587                 else
 588                         tmp |= DC_HPDx_INT_POLARITY;
 589                 WREG32(DC_HPD6_INT_CONTROL, tmp);
 590                 break;
 591         default:
 592                 break;
 593         }
 594 }
 595
 596 /**
 597  * evergreen_hpd_init - hpd setup callback.
 598  *
 599  * @rdev: radeon_device pointer
 600  *
 601  * Setup the hpd pins used by the card (evergreen+).
 602  * Enable the pin, set the polarity, and enable the hpd interrupts.
 603  */
 604 void evergreen_hpd_init(struct radeon_device *rdev)
 605 {
 606         struct drm_device *dev = rdev->ddev;
 607         struct drm_connector *connector;
 608         unsigned enabled = 0;
 609         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
 610                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
 611
 612         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 613                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 614                 switch (radeon_connector->hpd.hpd) {
 615                 case RADEON_HPD_1:
 616                         WREG32(DC_HPD1_CONTROL, tmp);
 617                         break;
 618                 case RADEON_HPD_2:
 619                         WREG32(DC_HPD2_CONTROL, tmp);
 620                         break;
 621                 case RADEON_HPD_3:
 622                         WREG32(DC_HPD3_CONTROL, tmp);
 623                         break;
 624                 case RADEON_HPD_4:
 625                         WREG32(DC_HPD4_CONTROL, tmp);
 626                         break;
 627                 case RADEON_HPD_5:
 628                         WREG32(DC_HPD5_CONTROL, tmp);
 629                         break;
 630                 case RADEON_HPD_6:
 631                         WREG32(DC_HPD6_CONTROL, tmp);
 632                         break;
 633                 default:
 634                         break;
 635                 }
 636                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
 637                 enabled |= 1 << radeon_connector->hpd.hpd;
 638         }
 639         radeon_irq_kms_enable_hpd(rdev, enabled);
 640 }
 641
 642 /**
 643  * evergreen_hpd_fini - hpd tear down callback.
 644  *
 645  * @rdev: radeon_device pointer
 646  *
 647  * Tear down the hpd pins used by the card (evergreen+).
 648  * Disable the hpd interrupts.
 649  */
 650 void evergreen_hpd_fini(struct radeon_device *rdev)
 651 {
 652         struct drm_device *dev = rdev->ddev;
 653         struct drm_connector *connector;
 654         unsigned disabled = 0;
 655
 656         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 657                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 658                 switch (radeon_connector->hpd.hpd) {
 659                 case RADEON_HPD_1:
 660                         WREG32(DC_HPD1_CONTROL, 0);
 661                         break;
 662                 case RADEON_HPD_2:
 663                         WREG32(DC_HPD2_CONTROL, 0);
 664                         break;
 665                 case RADEON_HPD_3:
 666                         WREG32(DC_HPD3_CONTROL, 0);
 667                         break;
 668                 case RADEON_HPD_4:
 669                         WREG32(DC_HPD4_CONTROL, 0);
 670                         break;
 671                 case RADEON_HPD_5:
 672                         WREG32(DC_HPD5_CONTROL, 0);
 673                         break;
 674                 case RADEON_HPD_6:
 675                         WREG32(DC_HPD6_CONTROL, 0);
 676                         break;
 677                 default:
 678                         break;
 679                 }
 680                 disabled |= 1 << radeon_connector->hpd.hpd;
 681         }
 682         radeon_irq_kms_disable_hpd(rdev, disabled);
 683 }
 684
 685 /* watermark setup */
 686
 687 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
 688                                         struct radeon_crtc *radeon_crtc,
 689                                         struct drm_display_mode *mode,
 690                                         struct drm_display_mode *other_mode)
 691 {
 692         u32 tmp;
 693         /*
 694          * Line Buffer Setup
 695          * There are 3 line buffers, each one shared by 2 display controllers.
 696          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
 697          * the display controllers.  The paritioning is done via one of four
 698          * preset allocations specified in bits 2:0:
 699          * first display controller
 700          *  0 - first half of lb (3840 * 2)
 701          *  1 - first 3/4 of lb (5760 * 2)
 702          *  2 - whole lb (7680 * 2), other crtc must be disabled
 703          *  3 - first 1/4 of lb (1920 * 2)
 704          * second display controller
 705          *  4 - second half of lb (3840 * 2)
 706          *  5 - second 3/4 of lb (5760 * 2)
 707          *  6 - whole lb (7680 * 2), other crtc must be disabled
 708          *  7 - last 1/4 of lb (1920 * 2)
 709          */
 710         /* this can get tricky if we have two large displays on a paired group
 711          * of crtcs.  Ideally for multiple large displays we'd assign them to
 712          * non-linked crtcs for maximum line buffer allocation.
 713          */
 714         if (radeon_crtc->base.enabled && mode) {
 715                 if (other_mode)
 716                         tmp = 0; /* 1/2 */
 717                 else
 718                         tmp = 2; /* whole */
 719         } else
 720                 tmp = 0;
 721
 722         /* second controller of the pair uses second half of the lb */
 723         if (radeon_crtc->crtc_id % 2)
 724                 tmp += 4;
 725         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
 726
 727         if (radeon_crtc->base.enabled && mode) {
 728                 switch (tmp) {
 729                 case 0:
 730                 case 4:
 731                 default:
 732                         if (ASIC_IS_DCE5(rdev))
 733                                 return 4096 * 2;
 734                         else
 735                                 return 3840 * 2;
 736                 case 1:
 737                 case 5:
 738                         if (ASIC_IS_DCE5(rdev))
 739                                 return 6144 * 2;
 740                         else
 741                                 return 5760 * 2;
 742                 case 2:
 743                 case 6:
 744                         if (ASIC_IS_DCE5(rdev))
 745                                 return 8192 * 2;
 746                         else
 747                                 return 7680 * 2;
 748                 case 3:
 749                 case 7:
 750                         if (ASIC_IS_DCE5(rdev))
 751                                 return 2048 * 2;
 752                         else
 753                                 return 1920 * 2;
 754                 }
 755         }
 756
 757         /* controller not enabled, so no lb used */
 758         return 0;
 759 }
 760
 761 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
 762 {
 763         u32 tmp = RREG32(MC_SHARED_CHMAP);
 764
 765         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
 766         case 0:
 767         default:
 768                 return 1;
 769         case 1:
 770                 return 2;
 771         case 2:
 772                 return 4;
 773         case 3:
 774                 return 8;
 775         }
 776 }
 777
 778 struct evergreen_wm_params {
 779         u32 dram_channels; /* number of dram channels */
 780         u32 yclk;          /* bandwidth per dram data pin in kHz */
 781         u32 sclk;          /* engine clock in kHz */
 782         u32 disp_clk;      /* display clock in kHz */
 783         u32 src_width;     /* viewport width */
 784         u32 active_time;   /* active display time in ns */
 785         u32 blank_time;    /* blank time in ns */
 786         bool interlaced;    /* mode is interlaced */
 787         fixed20_12 vsc;    /* vertical scale ratio */
 788         u32 num_heads;     /* number of active crtcs */
 789         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
 790         u32 lb_size;       /* line buffer allocated to pipe */
 791         u32 vtaps;         /* vertical scaler taps */
 792 };
 793
 794 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
 795 {
 796         /* Calculate DRAM Bandwidth and the part allocated to display. */
 797         fixed20_12 dram_efficiency; /* 0.7 */
 798         fixed20_12 yclk, dram_channels, bandwidth;
 799         fixed20_12 a;
 800
 801         a.full = dfixed_const(1000);
 802         yclk.full = dfixed_const(wm->yclk);
 803         yclk.full = dfixed_div(yclk, a);
 804         dram_channels.full = dfixed_const(wm->dram_channels * 4);
 805         a.full = dfixed_const(10);
 806         dram_efficiency.full = dfixed_const(7);
 807         dram_efficiency.full = dfixed_div(dram_efficiency, a);
 808         bandwidth.full = dfixed_mul(dram_channels, yclk);
 809         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
 810
 811         return dfixed_trunc(bandwidth);
 812 }
 813
 814 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
 815 {
 816         /* Calculate DRAM Bandwidth and the part allocated to display. */
 817         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
 818         fixed20_12 yclk, dram_channels, bandwidth;
 819         fixed20_12 a;
 820
 821         a.full = dfixed_const(1000);
 822         yclk.full = dfixed_const(wm->yclk);
 823         yclk.full = dfixed_div(yclk, a);
 824         dram_channels.full = dfixed_const(wm->dram_channels * 4);
 825         a.full = dfixed_const(10);
 826         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
 827         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
 828         bandwidth.full = dfixed_mul(dram_channels, yclk);
 829         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
 830
 831         return dfixed_trunc(bandwidth);
 832 }
 833
 834 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
 835 {
 836         /* Calculate the display Data return Bandwidth */
 837         fixed20_12 return_efficiency; /* 0.8 */
 838         fixed20_12 sclk, bandwidth;
 839         fixed20_12 a;
 840
 841         a.full = dfixed_const(1000);
 842         sclk.full = dfixed_const(wm->sclk);
 843         sclk.full = dfixed_div(sclk, a);
 844         a.full = dfixed_const(10);
 845         return_efficiency.full = dfixed_const(8);
 846         return_efficiency.full = dfixed_div(return_efficiency, a);
 847         a.full = dfixed_const(32);
 848         bandwidth.full = dfixed_mul(a, sclk);
 849         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
 850
 851         return dfixed_trunc(bandwidth);
 852 }
 853
 854 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
 855 {
 856         /* Calculate the DMIF Request Bandwidth */
 857         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
 858         fixed20_12 disp_clk, bandwidth;
 859         fixed20_12 a;
 860
 861         a.full = dfixed_const(1000);
 862         disp_clk.full = dfixed_const(wm->disp_clk);
 863         disp_clk.full = dfixed_div(disp_clk, a);
 864         a.full = dfixed_const(10);
 865         disp_clk_request_efficiency.full = dfixed_const(8);
 866         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
 867         a.full = dfixed_const(32);
 868         bandwidth.full = dfixed_mul(a, disp_clk);
 869         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
 870
 871         return dfixed_trunc(bandwidth);
 872 }
 873
 874 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
 875 {
 876         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
 877         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
 878         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
 879         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
 880
 881         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
 882 }
 883
 884 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
 885 {
 886         /* Calculate the display mode Average Bandwidth
 887          * DisplayMode should contain the source and destination dimensions,
 888          * timing, etc.
 889          */
 890         fixed20_12 bpp;
 891         fixed20_12 line_time;
 892         fixed20_12 src_width;
 893         fixed20_12 bandwidth;
 894         fixed20_12 a;
 895
 896         a.full = dfixed_const(1000);
 897         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
 898         line_time.full = dfixed_div(line_time, a);
 899         bpp.full = dfixed_const(wm->bytes_per_pixel);
 900         src_width.full = dfixed_const(wm->src_width);
 901         bandwidth.full = dfixed_mul(src_width, bpp);
 902         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
 903         bandwidth.full = dfixed_div(bandwidth, line_time);
 904
 905         return dfixed_trunc(bandwidth);
 906 }
 907
 908 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
 909 {
 910         /* First calcualte the latency in ns */
 911         u32 mc_latency = 2000; /* 2000 ns. */
 912         u32 available_bandwidth = evergreen_available_bandwidth(wm);
 913         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
 914         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
 915         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
 916         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
 917                 (wm->num_heads * cursor_line_pair_return_time);
 918         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
 919         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
 920         fixed20_12 a, b, c;
 921
 922         if (wm->num_heads == 0)
 923                 return 0;
 924
 925         a.full = dfixed_const(2);
 926         b.full = dfixed_const(1);
 927         if ((wm->vsc.full > a.full) ||
 928             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
 929             (wm->vtaps >= 5) ||
 930             ((wm->vsc.full >= a.full) && wm->interlaced))
 931                 max_src_lines_per_dst_line = 4;
 932         else
 933                 max_src_lines_per_dst_line = 2;
 934
 935         a.full = dfixed_const(available_bandwidth);
 936         b.full = dfixed_const(wm->num_heads);
 937         a.full = dfixed_div(a, b);
 938
 939         b.full = dfixed_const(1000);
 940         c.full = dfixed_const(wm->disp_clk);
 941         b.full = dfixed_div(c, b);
 942         c.full = dfixed_const(wm->bytes_per_pixel);
 943         b.full = dfixed_mul(b, c);
 944
 945         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
 946
 947         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
 948         b.full = dfixed_const(1000);
 949         c.full = dfixed_const(lb_fill_bw);
 950         b.full = dfixed_div(c, b);
 951         a.full = dfixed_div(a, b);
 952         line_fill_time = dfixed_trunc(a);
 953
 954         if (line_fill_time < wm->active_time)
 955                 return latency;
 956         else
 957                 return latency + (line_fill_time - wm->active_time);
 958
 959 }
 960
 961 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
 962 {
 963         if (evergreen_average_bandwidth(wm) <=
 964             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
 965                 return true;
 966         else
 967                 return false;
 968 };
 969
 970 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
 971 {
 972         if (evergreen_average_bandwidth(wm) <=
 973             (evergreen_available_bandwidth(wm) / wm->num_heads))
 974                 return true;
 975         else
 976                 return false;
 977 };
 978
 979 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
 980 {
 981         u32 lb_partitions = wm->lb_size / wm->src_width;
 982         u32 line_time = wm->active_time + wm->blank_time;
 983         u32 latency_tolerant_lines;
 984         u32 latency_hiding;
 985         fixed20_12 a;
 986
 987         a.full = dfixed_const(1);
 988         if (wm->vsc.full > a.full)
 989                 latency_tolerant_lines = 1;
 990         else {
 991                 if (lb_partitions <= (wm->vtaps + 1))
 992                         latency_tolerant_lines = 1;
 993                 else
 994                         latency_tolerant_lines = 2;
 995         }
 996
 997         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
 998
 999         if (evergreen_latency_watermark(wm) <= latency_hiding)
1000                 return true;
1001         else
1002                 return false;
1003 }
1004
1005 static void evergreen_program_watermarks(struct radeon_device *rdev,
1006                                          struct radeon_crtc *radeon_crtc,
1007                                          u32 lb_size, u32 num_heads)
1008 {
1009         struct drm_display_mode *mode = &radeon_crtc->base.mode;
1010         struct evergreen_wm_params wm;
1011         u32 pixel_period;
1012         u32 line_time = 0;
1013         u32 latency_watermark_a = 0, latency_watermark_b = 0;
1014         u32 priority_a_mark = 0, priority_b_mark = 0;
1015         u32 priority_a_cnt = PRIORITY_OFF;
1016         u32 priority_b_cnt = PRIORITY_OFF;
1017         u32 pipe_offset = radeon_crtc->crtc_id * 16;
1018         u32 tmp, arb_control3;
1019         fixed20_12 a, b, c;
1020
1021         if (radeon_crtc->base.enabled && num_heads && mode) {
1022                 pixel_period = 1000000 / (u32)mode->clock;
1023                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1024                 priority_a_cnt = 0;
1025                 priority_b_cnt = 0;
1026
1027                 wm.yclk = rdev->pm.current_mclk * 10;
1028                 wm.sclk = rdev->pm.current_sclk * 10;
1029                 wm.disp_clk = mode->clock;
1030                 wm.src_width = mode->crtc_hdisplay;
1031                 wm.active_time = mode->crtc_hdisplay * pixel_period;
1032                 wm.blank_time = line_time - wm.active_time;
1033                 wm.interlaced = false;
1034                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1035                         wm.interlaced = true;
1036                 wm.vsc = radeon_crtc->vsc;
1037                 wm.vtaps = 1;
1038                 if (radeon_crtc->rmx_type != RMX_OFF)
1039                         wm.vtaps = 2;
1040                 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1041                 wm.lb_size = lb_size;
1042                 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1043                 wm.num_heads = num_heads;
1044
1045                 /* set for high clocks */
1046                 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
1047                 /* set for low clocks */
1048                 /* wm.yclk = low clk; wm.sclk = low clk */
1049                 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
1050
1051                 /* possibly force display priority to high */
1052                 /* should really do this at mode validation time... */
1053                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1054                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
1055                     !evergreen_check_latency_hiding(&wm) ||
1056                     (rdev->disp_priority == 2)) {
1057                         DRM_DEBUG_KMS("force priority to high\n");
1058                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
1059                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
1060                 }
1061
1062                 a.full = dfixed_const(1000);
1063                 b.full = dfixed_const(mode->clock);
1064                 b.full = dfixed_div(b, a);
1065                 c.full = dfixed_const(latency_watermark_a);
1066                 c.full = dfixed_mul(c, b);
1067                 c.full = dfixed_mul(c, radeon_crtc->hsc);
1068                 c.full = dfixed_div(c, a);
1069                 a.full = dfixed_const(16);
1070                 c.full = dfixed_div(c, a);
1071                 priority_a_mark = dfixed_trunc(c);
1072                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1073
1074                 a.full = dfixed_const(1000);
1075                 b.full = dfixed_const(mode->clock);
1076                 b.full = dfixed_div(b, a);
1077                 c.full = dfixed_const(latency_watermark_b);
1078                 c.full = dfixed_mul(c, b);
1079                 c.full = dfixed_mul(c, radeon_crtc->hsc);
1080                 c.full = dfixed_div(c, a);
1081                 a.full = dfixed_const(16);
1082                 c.full = dfixed_div(c, a);
1083                 priority_b_mark = dfixed_trunc(c);
1084                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1085         }
1086
1087         /* select wm A */
1088         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
1089         tmp = arb_control3;
1090         tmp &= ~LATENCY_WATERMARK_MASK(3);
1091         tmp |= LATENCY_WATERMARK_MASK(1);
1092         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
1093         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
1094                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1095                 LATENCY_HIGH_WATERMARK(line_time)));
1096         /* select wm B */
1097         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
1098         tmp &= ~LATENCY_WATERMARK_MASK(3);
1099         tmp |= LATENCY_WATERMARK_MASK(2);
1100         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
1101         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
1102                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1103                 LATENCY_HIGH_WATERMARK(line_time)));
1104         /* restore original selection */
1105         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
1106
1107         /* write the priority marks */
1108         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1109         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1110
1111 }
1112
1113 /**
1114  * evergreen_bandwidth_update - update display watermarks callback.
1115  *
1116  * @rdev: radeon_device pointer
1117  *
1118  * Update the display watermarks based on the requested mode(s)
1119  * (evergreen+).
1120  */
1121 void evergreen_bandwidth_update(struct radeon_device *rdev)
1122 {
1123         struct drm_display_mode *mode0 = NULL;
1124         struct drm_display_mode *mode1 = NULL;
1125         u32 num_heads = 0, lb_size;
1126         int i;
1127
1128         radeon_update_display_priority(rdev);
1129
1130         for (i = 0; i < rdev->num_crtc; i++) {
1131                 if (rdev->mode_info.crtcs[i]->base.enabled)
1132                         num_heads++;
1133         }
1134         for (i = 0; i < rdev->num_crtc; i += 2) {
1135                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1136                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1137                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1138                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1139                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1140                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1141         }
1142 }
1143
1144 /**
1145  * evergreen_mc_wait_for_idle - wait for MC idle callback.
1146  *
1147  * @rdev: radeon_device pointer
1148  *
1149  * Wait for the MC (memory controller) to be idle.
1150  * (evergreen+).
1151  * Returns 0 if the MC is idle, -1 if not.
1152  */
1153 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1154 {
1155         unsigned i;
1156         u32 tmp;
1157
1158         for (i = 0; i < rdev->usec_timeout; i++) {
1159                 /* read MC_STATUS */
1160                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
1161                 if (!tmp)
1162                         return 0;
1163                 DRM_UDELAY(1);
1164         }
1165         return -1;
1166 }
1167
1168 /*
1169  * GART
1170  */
1171 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
1172 {
1173         unsigned i;
1174         u32 tmp;
1175
1176         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1177
1178         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
1179         for (i = 0; i < rdev->usec_timeout; i++) {
1180                 /* read MC_STATUS */
1181                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
1182                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
1183                 if (tmp == 2) {
1184                         DRM_ERROR("[drm] r600 flush TLB failed\n");
1185                         return;
1186                 }
1187                 if (tmp) {
1188                         return;
1189                 }
1190                 DRM_UDELAY(1);
1191         }
1192 }
1193
1194 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
1195 {
1196         u32 tmp;
1197         int r;
1198
1199         if (rdev->gart.robj == NULL) {
1200                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1201                 return -EINVAL;
1202         }
1203         r = radeon_gart_table_vram_pin(rdev);
1204         if (r)
1205                 return r;
1206         radeon_gart_restore(rdev);
1207         /* Setup L2 cache */
1208         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1209                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1210                                 EFFECTIVE_L2_QUEUE_SIZE(7));
1211         WREG32(VM_L2_CNTL2, 0);
1212         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1213         /* Setup TLB control */
1214         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1215                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1216                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1217                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1218         if (rdev->flags & RADEON_IS_IGP) {
1219                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
1220                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
1221                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
1222         } else {
1223                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1224                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1225                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1226                 if ((rdev->family == CHIP_JUNIPER) ||
1227                     (rdev->family == CHIP_CYPRESS) ||
1228                     (rdev->family == CHIP_HEMLOCK) ||
1229                     (rdev->family == CHIP_BARTS))
1230                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
1231         }
1232         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1233         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1234         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1235         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1236         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1237         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1238         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1239         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1240                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1241         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1242                         (u32)(rdev->dummy_page.addr >> 12));
1243         WREG32(VM_CONTEXT1_CNTL, 0);
1244
1245         evergreen_pcie_gart_tlb_flush(rdev);
1246         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1247                  (unsigned)(rdev->mc.gtt_size >> 20),
1248                  (unsigned long long)rdev->gart.table_addr);
1249         rdev->gart.ready = true;
1250         return 0;
1251 }
1252
1253 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
1254 {
1255         u32 tmp;
1256
1257         /* Disable all tables */
1258         WREG32(VM_CONTEXT0_CNTL, 0);
1259         WREG32(VM_CONTEXT1_CNTL, 0);
1260
1261         /* Setup L2 cache */
1262         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
1263                                 EFFECTIVE_L2_QUEUE_SIZE(7));
1264         WREG32(VM_L2_CNTL2, 0);
1265         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1266         /* Setup TLB control */
1267         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1268         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1269         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1270         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1271         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1272         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1273         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1274         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1275         radeon_gart_table_vram_unpin(rdev);
1276 }
1277
1278 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
1279 {
1280         evergreen_pcie_gart_disable(rdev);
1281         radeon_gart_table_vram_free(rdev);
1282         radeon_gart_fini(rdev);
1283 }
1284
1285
1286 static void evergreen_agp_enable(struct radeon_device *rdev)
1287 {
1288         u32 tmp;
1289
1290         /* Setup L2 cache */
1291         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1292                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1293                                 EFFECTIVE_L2_QUEUE_SIZE(7));
1294         WREG32(VM_L2_CNTL2, 0);
1295         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
1296         /* Setup TLB control */
1297         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1298                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1299                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
1300                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1301         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
1302         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
1303         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1304         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
1305         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
1306         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
1307         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1308         WREG32(VM_CONTEXT0_CNTL, 0);
1309         WREG32(VM_CONTEXT1_CNTL, 0);
1310 }
1311
1312 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1313 {
1314         u32 crtc_enabled, tmp, frame_count, blackout;
1315         int i, j;
1316
1317         save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
1318         save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
1319
1320         /* disable VGA render */
1321         WREG32(VGA_RENDER_CONTROL, 0);
1322         /* blank the display controllers */
1323         for (i = 0; i < rdev->num_crtc; i++) {
1324                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
1325                 if (crtc_enabled) {
1326                         save->crtc_enabled[i] = true;
1327                         if (ASIC_IS_DCE6(rdev)) {
1328                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
1329                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
1330                                         radeon_wait_for_vblank(rdev, i);
1331                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
1332                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1333                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
1334                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1335                                 }
1336                         } else {
1337                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
1338                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
1339                                         radeon_wait_for_vblank(rdev, i);
1340                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1341                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1342                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
1343                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1344                                 }
1345                         }
1346                         /* wait for the next frame */
1347                         frame_count = radeon_get_vblank_counter(rdev, i);
1348                         for (j = 0; j < rdev->usec_timeout; j++) {
1349                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
1350                                         break;
1351                                 DRM_UDELAY(1);
1352                         }
1353                 } else {
1354                         save->crtc_enabled[i] = false;
1355                 }
1356         }
1357
1358         radeon_mc_wait_for_idle(rdev);
1359
1360         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1361         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
1362                 /* Block CPU access */
1363                 WREG32(BIF_FB_EN, 0);
1364                 /* blackout the MC */
1365                 blackout &= ~BLACKOUT_MODE_MASK;
1366                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1367         }
1368         /* wait for the MC to settle */
1369         DRM_UDELAY(100);
1370 }
1371
1372 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1373 {
1374         u32 tmp, frame_count;
1375         int i, j;
1376
1377         /* update crtc base addresses */
1378         for (i = 0; i < rdev->num_crtc; i++) {
1379                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
1380                        upper_32_bits(rdev->mc.vram_start));
1381                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
1382                        upper_32_bits(rdev->mc.vram_start));
1383                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
1384                        (u32)rdev->mc.vram_start);
1385                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
1386                        (u32)rdev->mc.vram_start);
1387         }
1388         WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1389         WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1390
1391         /* unblackout the MC */
1392         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
1393         tmp &= ~BLACKOUT_MODE_MASK;
1394         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
1395         /* allow CPU access */
1396         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
1397
1398         for (i = 0; i < rdev->num_crtc; i++) {
1399                 if (save->crtc_enabled[i]) {
1400                         if (ASIC_IS_DCE6(rdev)) {
1401                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
1402                                 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
1403                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1404                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
1405                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1406                         } else {
1407                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
1408                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1409                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
1410                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
1411                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
1412                         }
1413                         /* wait for the next frame */
1414                         frame_count = radeon_get_vblank_counter(rdev, i);
1415                         for (j = 0; j < rdev->usec_timeout; j++) {
1416                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
1417                                         break;
1418                                 DRM_UDELAY(1);
1419                         }
1420                 }
1421         }
1422         /* Unlock vga access */
1423         WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1424         DRM_MDELAY(1);
1425         WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1426 }
1427
1428 void evergreen_mc_program(struct radeon_device *rdev)
1429 {
1430         struct evergreen_mc_save save;
1431         u32 tmp;
1432         int i, j;
1433
1434         /* Initialize HDP */
1435         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1436                 WREG32((0x2c14 + j), 0x00000000);
1437                 WREG32((0x2c18 + j), 0x00000000);
1438                 WREG32((0x2c1c + j), 0x00000000);
1439                 WREG32((0x2c20 + j), 0x00000000);
1440                 WREG32((0x2c24 + j), 0x00000000);
1441         }
1442         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1443
1444         evergreen_mc_stop(rdev, &save);
1445         if (evergreen_mc_wait_for_idle(rdev)) {
1446                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1447         }
1448         /* Lockout access through VGA aperture*/
1449         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1450         /* Update configuration */
1451         if (rdev->flags & RADEON_IS_AGP) {
1452                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1453                         /* VRAM before AGP */
1454                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1455                                 rdev->mc.vram_start >> 12);
1456                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1457                                 rdev->mc.gtt_end >> 12);
1458                 } else {
1459                         /* VRAM after AGP */
1460                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1461                                 rdev->mc.gtt_start >> 12);
1462                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1463                                 rdev->mc.vram_end >> 12);
1464                 }
1465         } else {
1466                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1467                         rdev->mc.vram_start >> 12);
1468                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1469                         rdev->mc.vram_end >> 12);
1470         }
1471         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1472         /* llano/ontario only */
1473         if ((rdev->family == CHIP_PALM) ||
1474             (rdev->family == CHIP_SUMO) ||
1475             (rdev->family == CHIP_SUMO2)) {
1476                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1477                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1478                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1479                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1480         }
1481         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1482         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1483         WREG32(MC_VM_FB_LOCATION, tmp);
1484         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1485         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1486         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1487         if (rdev->flags & RADEON_IS_AGP) {
1488                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1489                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1490                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1491         } else {
1492                 WREG32(MC_VM_AGP_BASE, 0);
1493                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1494                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1495         }
1496         if (evergreen_mc_wait_for_idle(rdev)) {
1497                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1498         }
1499         evergreen_mc_resume(rdev, &save);
1500         /* we need to own VRAM, so turn off the VGA renderer here
1501          * to stop it overwriting our objects */
1502         rv515_vga_render_disable(rdev);
1503 }
1504
1505 /*
1506  * CP.
1507  */
1508 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1509 {
1510         struct radeon_ring *ring = &rdev->ring[ib->ring];
1511         u32 next_rptr;
1512
1513         /* set to DX10/11 mode */
1514         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1515         radeon_ring_write(ring, 1);
1516
1517         if (ring->rptr_save_reg) {
1518                 next_rptr = ring->wptr + 3 + 4;
1519                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1520                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1521                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1522                 radeon_ring_write(ring, next_rptr);
1523         } else if (rdev->wb.enabled) {
1524                 next_rptr = ring->wptr + 5 + 4;
1525                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
1526                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1527                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
1528                 radeon_ring_write(ring, next_rptr);
1529                 radeon_ring_write(ring, 0);
1530         }
1531
1532         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1533         radeon_ring_write(ring,
1534 #ifdef __BIG_ENDIAN
1535                           (2 << 0) |
1536 #endif
1537                           (ib->gpu_addr & 0xFFFFFFFC));
1538         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1539         radeon_ring_write(ring, ib->length_dw);
1540 }
1541
1542
1543 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1544 {
1545         const __be32 *fw_data;
1546         int i;
1547
1548         if (!rdev->me_fw || !rdev->pfp_fw)
1549                 return -EINVAL;
1550
1551         r700_cp_stop(rdev);
1552         WREG32(CP_RB_CNTL,
1553 #ifdef __BIG_ENDIAN
1554                BUF_SWAP_32BIT |
1555 #endif
1556                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1557
1558         fw_data = (const __be32 *)rdev->pfp_fw->data;
1559         WREG32(CP_PFP_UCODE_ADDR, 0);
1560         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1561                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1562         WREG32(CP_PFP_UCODE_ADDR, 0);
1563
1564         fw_data = (const __be32 *)rdev->me_fw->data;
1565         WREG32(CP_ME_RAM_WADDR, 0);
1566         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1567                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1568
1569         WREG32(CP_PFP_UCODE_ADDR, 0);
1570         WREG32(CP_ME_RAM_WADDR, 0);
1571         WREG32(CP_ME_RAM_RADDR, 0);
1572         return 0;
1573 }
1574
1575 static int evergreen_cp_start(struct radeon_device *rdev)
1576 {
1577         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1578         int r, i;
1579         uint32_t cp_me;
1580
1581         r = radeon_ring_lock(rdev, ring, 7);
1582         if (r) {
1583                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1584                 return r;
1585         }
1586         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1587         radeon_ring_write(ring, 0x1);
1588         radeon_ring_write(ring, 0x0);
1589         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
1590         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1591         radeon_ring_write(ring, 0);
1592         radeon_ring_write(ring, 0);
1593         radeon_ring_unlock_commit(rdev, ring);
1594
1595         cp_me = 0xff;
1596         WREG32(CP_ME_CNTL, cp_me);
1597
1598         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1599         if (r) {
1600                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1601                 return r;
1602         }
1603
1604         /* setup clear context state */
1605         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1606         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1607
1608         for (i = 0; i < evergreen_default_size; i++)
1609                 radeon_ring_write(ring, evergreen_default_state[i]);
1610
1611         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1612         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1613
1614         /* set clear context state */
1615         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1616         radeon_ring_write(ring, 0);
1617
1618         /* SQ_VTX_BASE_VTX_LOC */
1619         radeon_ring_write(ring, 0xc0026f00);
1620         radeon_ring_write(ring, 0x00000000);
1621         radeon_ring_write(ring, 0x00000000);
1622         radeon_ring_write(ring, 0x00000000);
1623
1624         /* Clear consts */
1625         radeon_ring_write(ring, 0xc0036f00);
1626         radeon_ring_write(ring, 0x00000bc4);
1627         radeon_ring_write(ring, 0xffffffff);
1628         radeon_ring_write(ring, 0xffffffff);
1629         radeon_ring_write(ring, 0xffffffff);
1630
1631         radeon_ring_write(ring, 0xc0026900);
1632         radeon_ring_write(ring, 0x00000316);
1633         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1634         radeon_ring_write(ring, 0x00000010); /*  */
1635
1636         radeon_ring_unlock_commit(rdev, ring);
1637
1638         return 0;
1639 }
1640
1641 static int evergreen_cp_resume(struct radeon_device *rdev)
1642 {
1643         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1644         u32 tmp;
1645         u32 rb_bufsz;
1646         int r;
1647
1648         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1649         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1650                                  SOFT_RESET_PA |
1651                                  SOFT_RESET_SH |
1652                                  SOFT_RESET_VGT |
1653                                  SOFT_RESET_SPI |
1654                                  SOFT_RESET_SX));
1655         RREG32(GRBM_SOFT_RESET);
1656         DRM_MDELAY(15);
1657         WREG32(GRBM_SOFT_RESET, 0);
1658         RREG32(GRBM_SOFT_RESET);
1659
1660         /* Set ring buffer size */
1661         rb_bufsz = drm_order(ring->ring_size / 8);
1662         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1663 #ifdef __BIG_ENDIAN
1664         tmp |= BUF_SWAP_32BIT;
1665 #endif
1666         WREG32(CP_RB_CNTL, tmp);
1667         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1668         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1669
1670         /* Set the write pointer delay */
1671         WREG32(CP_RB_WPTR_DELAY, 0);
1672
1673         /* Initialize the ring buffer's read and write pointers */
1674         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1675         WREG32(CP_RB_RPTR_WR, 0);
1676         ring->wptr = 0;
1677         WREG32(CP_RB_WPTR, ring->wptr);
1678
1679         /* set the wb address whether it's enabled or not */
1680         WREG32(CP_RB_RPTR_ADDR,
1681                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1682         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1683         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1684
1685         if (rdev->wb.enabled)
1686                 WREG32(SCRATCH_UMSK, 0xff);
1687         else {
1688                 tmp |= RB_NO_UPDATE;
1689                 WREG32(SCRATCH_UMSK, 0);
1690         }
1691
1692         DRM_MDELAY(1);
1693         WREG32(CP_RB_CNTL, tmp);
1694
1695         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1696         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1697
1698         ring->rptr = RREG32(CP_RB_RPTR);
1699
1700         evergreen_cp_start(rdev);
1701         ring->ready = true;
1702         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1703         if (r) {
1704                 ring->ready = false;
1705                 return r;
1706         }
1707         return 0;
1708 }
1709
1710 /*
1711  * Core functions
1712  */
1713 static void evergreen_gpu_init(struct radeon_device *rdev)
1714 {
1715         u32 gb_addr_config;
1716         u32 mc_shared_chmap, mc_arb_ramcfg;
1717         u32 sx_debug_1;
1718         u32 smx_dc_ctl0;
1719         u32 sq_config;
1720         u32 sq_lds_resource_mgmt;
1721         u32 sq_gpr_resource_mgmt_1;
1722         u32 sq_gpr_resource_mgmt_2;
1723         u32 sq_gpr_resource_mgmt_3;
1724         u32 sq_thread_resource_mgmt;
1725         u32 sq_thread_resource_mgmt_2;
1726         u32 sq_stack_resource_mgmt_1;
1727         u32 sq_stack_resource_mgmt_2;
1728         u32 sq_stack_resource_mgmt_3;
1729         u32 vgt_cache_invalidation;
1730         u32 hdp_host_path_cntl, tmp;
1731         u32 disabled_rb_mask;
1732         int i, j, num_shader_engines, ps_thread_count;
1733
1734         switch (rdev->family) {
1735         case CHIP_CYPRESS:
1736         case CHIP_HEMLOCK:
1737                 rdev->config.evergreen.num_ses = 2;
1738                 rdev->config.evergreen.max_pipes = 4;
1739                 rdev->config.evergreen.max_tile_pipes = 8;
1740                 rdev->config.evergreen.max_simds = 10;
1741                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1742                 rdev->config.evergreen.max_gprs = 256;
1743                 rdev->config.evergreen.max_threads = 248;
1744                 rdev->config.evergreen.max_gs_threads = 32;
1745                 rdev->config.evergreen.max_stack_entries = 512;
1746                 rdev->config.evergreen.sx_num_of_sets = 4;
1747                 rdev->config.evergreen.sx_max_export_size = 256;
1748                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1749                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1750                 rdev->config.evergreen.max_hw_contexts = 8;
1751                 rdev->config.evergreen.sq_num_cf_insts = 2;
1752
1753                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1754                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1755                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1756                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
1757                 break;
1758         case CHIP_JUNIPER:
1759                 rdev->config.evergreen.num_ses = 1;
1760                 rdev->config.evergreen.max_pipes = 4;
1761                 rdev->config.evergreen.max_tile_pipes = 4;
1762                 rdev->config.evergreen.max_simds = 10;
1763                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1764                 rdev->config.evergreen.max_gprs = 256;
1765                 rdev->config.evergreen.max_threads = 248;
1766                 rdev->config.evergreen.max_gs_threads = 32;
1767                 rdev->config.evergreen.max_stack_entries = 512;
1768                 rdev->config.evergreen.sx_num_of_sets = 4;
1769                 rdev->config.evergreen.sx_max_export_size = 256;
1770                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1771                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1772                 rdev->config.evergreen.max_hw_contexts = 8;
1773                 rdev->config.evergreen.sq_num_cf_insts = 2;
1774
1775                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1776                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1777                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1778                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
1779                 break;
1780         case CHIP_REDWOOD:
1781                 rdev->config.evergreen.num_ses = 1;
1782                 rdev->config.evergreen.max_pipes = 4;
1783                 rdev->config.evergreen.max_tile_pipes = 4;
1784                 rdev->config.evergreen.max_simds = 5;
1785                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1786                 rdev->config.evergreen.max_gprs = 256;
1787                 rdev->config.evergreen.max_threads = 248;
1788                 rdev->config.evergreen.max_gs_threads = 32;
1789                 rdev->config.evergreen.max_stack_entries = 256;
1790                 rdev->config.evergreen.sx_num_of_sets = 4;
1791                 rdev->config.evergreen.sx_max_export_size = 256;
1792                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1793                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1794                 rdev->config.evergreen.max_hw_contexts = 8;
1795                 rdev->config.evergreen.sq_num_cf_insts = 2;
1796
1797                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1798                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1799                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1800                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1801                 break;
1802         case CHIP_CEDAR:
1803         default:
1804                 rdev->config.evergreen.num_ses = 1;
1805                 rdev->config.evergreen.max_pipes = 2;
1806                 rdev->config.evergreen.max_tile_pipes = 2;
1807                 rdev->config.evergreen.max_simds = 2;
1808                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1809                 rdev->config.evergreen.max_gprs = 256;
1810                 rdev->config.evergreen.max_threads = 192;
1811                 rdev->config.evergreen.max_gs_threads = 16;
1812                 rdev->config.evergreen.max_stack_entries = 256;
1813                 rdev->config.evergreen.sx_num_of_sets = 4;
1814                 rdev->config.evergreen.sx_max_export_size = 128;
1815                 rdev->config.evergreen.sx_max_export_pos_size = 32;
1816                 rdev->config.evergreen.sx_max_export_smx_size = 96;
1817                 rdev->config.evergreen.max_hw_contexts = 4;
1818                 rdev->config.evergreen.sq_num_cf_insts = 1;
1819
1820                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1821                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1822                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1823                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1824                 break;
1825         case CHIP_PALM:
1826                 rdev->config.evergreen.num_ses = 1;
1827                 rdev->config.evergreen.max_pipes = 2;
1828                 rdev->config.evergreen.max_tile_pipes = 2;
1829                 rdev->config.evergreen.max_simds = 2;
1830                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1831                 rdev->config.evergreen.max_gprs = 256;
1832                 rdev->config.evergreen.max_threads = 192;
1833                 rdev->config.evergreen.max_gs_threads = 16;
1834                 rdev->config.evergreen.max_stack_entries = 256;
1835                 rdev->config.evergreen.sx_num_of_sets = 4;
1836                 rdev->config.evergreen.sx_max_export_size = 128;
1837                 rdev->config.evergreen.sx_max_export_pos_size = 32;
1838                 rdev->config.evergreen.sx_max_export_smx_size = 96;
1839                 rdev->config.evergreen.max_hw_contexts = 4;
1840                 rdev->config.evergreen.sq_num_cf_insts = 1;
1841
1842                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1843                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1844                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1845                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1846                 break;
1847         case CHIP_SUMO:
1848                 rdev->config.evergreen.num_ses = 1;
1849                 rdev->config.evergreen.max_pipes = 4;
1850                 rdev->config.evergreen.max_tile_pipes = 4;
1851                 if (rdev->ddev->pci_device == 0x9648)
1852                         rdev->config.evergreen.max_simds = 3;
1853                 else if ((rdev->ddev->pci_device == 0x9647) ||
1854                          (rdev->ddev->pci_device == 0x964a))
1855                         rdev->config.evergreen.max_simds = 4;
1856                 else
1857                         rdev->config.evergreen.max_simds = 5;
1858                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1859                 rdev->config.evergreen.max_gprs = 256;
1860                 rdev->config.evergreen.max_threads = 248;
1861                 rdev->config.evergreen.max_gs_threads = 32;
1862                 rdev->config.evergreen.max_stack_entries = 256;
1863                 rdev->config.evergreen.sx_num_of_sets = 4;
1864                 rdev->config.evergreen.sx_max_export_size = 256;
1865                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1866                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1867                 rdev->config.evergreen.max_hw_contexts = 8;
1868                 rdev->config.evergreen.sq_num_cf_insts = 2;
1869
1870                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1871                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1872                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1873                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
1874                 break;
1875         case CHIP_SUMO2:
1876                 rdev->config.evergreen.num_ses = 1;
1877                 rdev->config.evergreen.max_pipes = 4;
1878                 rdev->config.evergreen.max_tile_pipes = 4;
1879                 rdev->config.evergreen.max_simds = 2;
1880                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1881                 rdev->config.evergreen.max_gprs = 256;
1882                 rdev->config.evergreen.max_threads = 248;
1883                 rdev->config.evergreen.max_gs_threads = 32;
1884                 rdev->config.evergreen.max_stack_entries = 512;
1885                 rdev->config.evergreen.sx_num_of_sets = 4;
1886                 rdev->config.evergreen.sx_max_export_size = 256;
1887                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1888                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1889                 rdev->config.evergreen.max_hw_contexts = 8;
1890                 rdev->config.evergreen.sq_num_cf_insts = 2;
1891
1892                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1893                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1894                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1895                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
1896                 break;
1897         case CHIP_BARTS:
1898                 rdev->config.evergreen.num_ses = 2;
1899                 rdev->config.evergreen.max_pipes = 4;
1900                 rdev->config.evergreen.max_tile_pipes = 8;
1901                 rdev->config.evergreen.max_simds = 7;
1902                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1903                 rdev->config.evergreen.max_gprs = 256;
1904                 rdev->config.evergreen.max_threads = 248;
1905                 rdev->config.evergreen.max_gs_threads = 32;
1906                 rdev->config.evergreen.max_stack_entries = 512;
1907                 rdev->config.evergreen.sx_num_of_sets = 4;
1908                 rdev->config.evergreen.sx_max_export_size = 256;
1909                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1910                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1911                 rdev->config.evergreen.max_hw_contexts = 8;
1912                 rdev->config.evergreen.sq_num_cf_insts = 2;
1913
1914                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1915                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1916                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1917                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
1918                 break;
1919         case CHIP_TURKS:
1920                 rdev->config.evergreen.num_ses = 1;
1921                 rdev->config.evergreen.max_pipes = 4;
1922                 rdev->config.evergreen.max_tile_pipes = 4;
1923                 rdev->config.evergreen.max_simds = 6;
1924                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1925                 rdev->config.evergreen.max_gprs = 256;
1926                 rdev->config.evergreen.max_threads = 248;
1927                 rdev->config.evergreen.max_gs_threads = 32;
1928                 rdev->config.evergreen.max_stack_entries = 256;
1929                 rdev->config.evergreen.sx_num_of_sets = 4;
1930                 rdev->config.evergreen.sx_max_export_size = 256;
1931                 rdev->config.evergreen.sx_max_export_pos_size = 64;
1932                 rdev->config.evergreen.sx_max_export_smx_size = 192;
1933                 rdev->config.evergreen.max_hw_contexts = 8;
1934                 rdev->config.evergreen.sq_num_cf_insts = 2;
1935
1936                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1937                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1938                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1939                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
1940                 break;
1941         case CHIP_CAICOS:
1942                 rdev->config.evergreen.num_ses = 1;
1943                 rdev->config.evergreen.max_pipes = 2;
1944                 rdev->config.evergreen.max_tile_pipes = 2;
1945                 rdev->config.evergreen.max_simds = 2;
1946                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1947                 rdev->config.evergreen.max_gprs = 256;
1948                 rdev->config.evergreen.max_threads = 192;
1949                 rdev->config.evergreen.max_gs_threads = 16;
1950                 rdev->config.evergreen.max_stack_entries = 256;
1951                 rdev->config.evergreen.sx_num_of_sets = 4;
1952                 rdev->config.evergreen.sx_max_export_size = 128;
1953                 rdev->config.evergreen.sx_max_export_pos_size = 32;
1954                 rdev->config.evergreen.sx_max_export_smx_size = 96;
1955                 rdev->config.evergreen.max_hw_contexts = 4;
1956                 rdev->config.evergreen.sq_num_cf_insts = 1;
1957
1958                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1959                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1960                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1961                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
1962                 break;
1963         }
1964
1965         /* Initialize HDP */
1966         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1967                 WREG32((0x2c14 + j), 0x00000000);
1968                 WREG32((0x2c18 + j), 0x00000000);
1969                 WREG32((0x2c1c + j), 0x00000000);
1970                 WREG32((0x2c20 + j), 0x00000000);
1971                 WREG32((0x2c24 + j), 0x00000000);
1972         }
1973
1974         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1975
1976         evergreen_fix_pci_max_read_req_size(rdev);
1977
1978         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1979         if ((rdev->family == CHIP_PALM) ||
1980             (rdev->family == CHIP_SUMO) ||
1981             (rdev->family == CHIP_SUMO2))
1982                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
1983         else
1984                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1985
1986         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1987          * not have bank info, so create a custom tiling dword.
1988          * bits 3:0   num_pipes
1989          * bits 7:4   num_banks
1990          * bits 11:8  group_size
1991          * bits 15:12 row_size
1992          */
1993         rdev->config.evergreen.tile_config = 0;
1994         switch (rdev->config.evergreen.max_tile_pipes) {
1995         case 1:
1996         default:
1997                 rdev->config.evergreen.tile_config |= (0 << 0);
1998                 break;
1999         case 2:
2000                 rdev->config.evergreen.tile_config |= (1 << 0);
2001                 break;
2002         case 4:
2003                 rdev->config.evergreen.tile_config |= (2 << 0);
2004                 break;
2005         case 8:
2006                 rdev->config.evergreen.tile_config |= (3 << 0);
2007                 break;
2008         }
2009         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
2010         if (rdev->flags & RADEON_IS_IGP)
2011                 rdev->config.evergreen.tile_config |= 1 << 4;
2012         else {
2013                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2014                 case 0: /* four banks */
2015                         rdev->config.evergreen.tile_config |= 0 << 4;
2016                         break;
2017                 case 1: /* eight banks */
2018                         rdev->config.evergreen.tile_config |= 1 << 4;
2019                         break;
2020                 case 2: /* sixteen banks */
2021                 default:
2022                         rdev->config.evergreen.tile_config |= 2 << 4;
2023                         break;
2024                 }
2025         }
2026         rdev->config.evergreen.tile_config |= 0 << 8;
2027         rdev->config.evergreen.tile_config |=
2028                 ((gb_addr_config & 0x30000000) >> 28) << 12;
2029
2030         num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
2031
2032         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
2033                 u32 efuse_straps_4;
2034                 u32 efuse_straps_3;
2035
2036                 WREG32(RCU_IND_INDEX, 0x204);
2037                 efuse_straps_4 = RREG32(RCU_IND_DATA);
2038                 WREG32(RCU_IND_INDEX, 0x203);
2039                 efuse_straps_3 = RREG32(RCU_IND_DATA);
2040                 tmp = (((efuse_straps_4 & 0xf) << 4) |
2041                       ((efuse_straps_3 & 0xf0000000) >> 28));
2042         } else {
2043                 tmp = 0;
2044                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
2045                         u32 rb_disable_bitmap;
2046
2047                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
2048                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
2049                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
2050                         tmp <<= 4;
2051                         tmp |= rb_disable_bitmap;
2052                 }
2053         }
2054         /* enabled rb are just the one not disabled :) */
2055         disabled_rb_mask = tmp;
2056
2057         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
2058         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
2059
2060         WREG32(GB_ADDR_CONFIG, gb_addr_config);
2061         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2062         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2063         WREG32(DMA_TILING_CONFIG, gb_addr_config);
2064
2065         if ((rdev->config.evergreen.max_backends == 1) &&
2066             (rdev->flags & RADEON_IS_IGP)) {
2067                 if ((disabled_rb_mask & 3) == 1) {
2068                         /* RB0 disabled, RB1 enabled */
2069                         tmp = 0x11111111;
2070                 } else {
2071                         /* RB1 disabled, RB0 enabled */
2072                         tmp = 0x00000000;
2073                 }
2074         } else {
2075                 tmp = gb_addr_config & NUM_PIPES_MASK;
2076                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
2077                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
2078         }
2079         WREG32(GB_BACKEND_MAP, tmp);
2080
2081         WREG32(CGTS_SYS_TCC_DISABLE, 0);
2082         WREG32(CGTS_TCC_DISABLE, 0);
2083         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2084         WREG32(CGTS_USER_TCC_DISABLE, 0);
2085
2086         /* set HW defaults for 3D engine */
2087         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2088                                      ROQ_IB2_START(0x2b)));
2089
2090         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2091
2092         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2093                              SYNC_GRADIENT |
2094                              SYNC_WALKER |
2095                              SYNC_ALIGNER));
2096
2097         sx_debug_1 = RREG32(SX_DEBUG_1);
2098         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2099         WREG32(SX_DEBUG_1, sx_debug_1);
2100
2101
2102         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2103         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2104         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2105         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2106
2107         if (rdev->family <= CHIP_SUMO2)
2108                 WREG32(SMX_SAR_CTL0, 0x00010000);
2109
2110         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2111                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2112                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2113
2114         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2115                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2116                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2117
2118         WREG32(VGT_NUM_INSTANCES, 1);
2119         WREG32(SPI_CONFIG_CNTL, 0);
2120         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2121         WREG32(CP_PERFMON_CNTL, 0);
2122
2123         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2124                                   FETCH_FIFO_HIWATER(0x4) |
2125                                   DONE_FIFO_HIWATER(0xe0) |
2126                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
2127
2128         sq_config = RREG32(SQ_CONFIG);
2129         sq_config &= ~(PS_PRIO(3) |
2130                        VS_PRIO(3) |
2131                        GS_PRIO(3) |
2132                        ES_PRIO(3));
2133         sq_config |= (VC_ENABLE |
2134                       EXPORT_SRC_C |
2135                       PS_PRIO(0) |
2136                       VS_PRIO(1) |
2137                       GS_PRIO(2) |
2138                       ES_PRIO(3));
2139
2140         switch (rdev->family) {
2141         case CHIP_CEDAR:
2142         case CHIP_PALM:
2143         case CHIP_SUMO:
2144         case CHIP_SUMO2:
2145         case CHIP_CAICOS:
2146                 /* no vertex cache */
2147                 sq_config &= ~VC_ENABLE;
2148                 break;
2149         default:
2150                 break;
2151         }
2152
2153         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2154
2155         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2156         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2157         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2158         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2159         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2160         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2161         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2162
2163         switch (rdev->family) {
2164         case CHIP_CEDAR:
2165         case CHIP_PALM:
2166         case CHIP_SUMO:
2167         case CHIP_SUMO2:
2168                 ps_thread_count = 96;
2169                 break;
2170         default:
2171                 ps_thread_count = 128;
2172                 break;
2173         }
2174
2175         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2176         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2177         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2178         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2179         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2180         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2181
2182         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2183         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2184         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2185         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2186         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2187         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2188
2189         WREG32(SQ_CONFIG, sq_config);
2190         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2191         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2192         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2193         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2194         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2195         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2196         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2197         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2198         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2199         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2200
2201         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2202                                           FORCE_EOV_MAX_REZ_CNT(255)));
2203
2204         switch (rdev->family) {
2205         case CHIP_CEDAR:
2206         case CHIP_PALM:
2207         case CHIP_SUMO:
2208         case CHIP_SUMO2:
2209         case CHIP_CAICOS:
2210                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2211                 break;
2212         default:
2213                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2214                 break;
2215         }
2216         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2217         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2218
2219         WREG32(VGT_GS_VERTEX_REUSE, 16);
2220         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2221         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2222
2223         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2224         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2225
2226         WREG32(CB_PERF_CTR0_SEL_0, 0);
2227         WREG32(CB_PERF_CTR0_SEL_1, 0);
2228         WREG32(CB_PERF_CTR1_SEL_0, 0);
2229         WREG32(CB_PERF_CTR1_SEL_1, 0);
2230         WREG32(CB_PERF_CTR2_SEL_0, 0);
2231         WREG32(CB_PERF_CTR2_SEL_1, 0);
2232         WREG32(CB_PERF_CTR3_SEL_0, 0);
2233         WREG32(CB_PERF_CTR3_SEL_1, 0);
2234
2235         /* clear render buffer base addresses */
2236         WREG32(CB_COLOR0_BASE, 0);
2237         WREG32(CB_COLOR1_BASE, 0);
2238         WREG32(CB_COLOR2_BASE, 0);
2239         WREG32(CB_COLOR3_BASE, 0);
2240         WREG32(CB_COLOR4_BASE, 0);
2241         WREG32(CB_COLOR5_BASE, 0);
2242         WREG32(CB_COLOR6_BASE, 0);
2243         WREG32(CB_COLOR7_BASE, 0);
2244         WREG32(CB_COLOR8_BASE, 0);
2245         WREG32(CB_COLOR9_BASE, 0);
2246         WREG32(CB_COLOR10_BASE, 0);
2247         WREG32(CB_COLOR11_BASE, 0);
2248
2249         /* set the shader const cache sizes to 0 */
2250         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2251                 WREG32(i, 0);
2252         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2253                 WREG32(i, 0);
2254
2255         tmp = RREG32(HDP_MISC_CNTL);
2256         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2257         WREG32(HDP_MISC_CNTL, tmp);
2258
2259         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2260         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2261
2262         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2263
2264         DRM_UDELAY(50);
2265
2266 }
2267
2268 int evergreen_mc_init(struct radeon_device *rdev)
2269 {
2270         u32 tmp;
2271         int chansize, numchan;
2272
2273         /* Get VRAM informations */
2274         rdev->mc.vram_is_ddr = true;
2275         if ((rdev->family == CHIP_PALM) ||
2276             (rdev->family == CHIP_SUMO) ||
2277             (rdev->family == CHIP_SUMO2))
2278                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
2279         else
2280                 tmp = RREG32(MC_ARB_RAMCFG);
2281         if (tmp & CHANSIZE_OVERRIDE) {
2282                 chansize = 16;
2283         } else if (tmp & CHANSIZE_MASK) {
2284                 chansize = 64;
2285         } else {
2286                 chansize = 32;
2287         }
2288         tmp = RREG32(MC_SHARED_CHMAP);
2289         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2290         case 0:
2291         default:
2292                 numchan = 1;
2293                 break;
2294         case 1:
2295                 numchan = 2;
2296                 break;
2297         case 2:
2298                 numchan = 4;
2299                 break;
2300         case 3:
2301                 numchan = 8;
2302                 break;
2303         }
2304         rdev->mc.vram_width = numchan * chansize;
2305         /* Could aper size report 0 ? */
2306         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
2307         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
2308         /* Setup GPU memory space */
2309         if ((rdev->family == CHIP_PALM) ||
2310             (rdev->family == CHIP_SUMO) ||
2311             (rdev->family == CHIP_SUMO2)) {
2312                 /* size in bytes on fusion */
2313                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2314                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2315         } else {
2316                 /* size in MB on evergreen/cayman/tn */
2317                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2318                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2319         }
2320         rdev->mc.visible_vram_size = rdev->mc.aper_size;
2321         r700_vram_gtt_location(rdev, &rdev->mc);
2322         radeon_update_bandwidth_info(rdev);
2323
2324         return 0;
2325 }
2326
2327 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
2328 {
2329         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
2330                 RREG32(GRBM_STATUS));
2331         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
2332                 RREG32(GRBM_STATUS_SE0));
2333         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
2334                 RREG32(GRBM_STATUS_SE1));
2335         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
2336                 RREG32(SRBM_STATUS));
2337         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
2338                 RREG32(SRBM_STATUS2));
2339         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
2340                 RREG32(CP_STALLED_STAT1));
2341         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
2342                 RREG32(CP_STALLED_STAT2));
2343         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
2344                 RREG32(CP_BUSY_STAT));
2345         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
2346                 RREG32(CP_STAT));
2347         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
2348                 RREG32(DMA_STATUS_REG));
2349         if (rdev->family >= CHIP_CAYMAN) {
2350                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
2351                          RREG32(DMA_STATUS_REG + 0x800));
2352         }
2353 }
2354
2355 bool evergreen_is_display_hung(struct radeon_device *rdev)
2356 {
2357         u32 crtc_hung = 0;
2358         u32 crtc_status[6];
2359         u32 i, j, tmp;
2360
2361         for (i = 0; i < rdev->num_crtc; i++) {
2362                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
2363                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
2364                         crtc_hung |= (1 << i);
2365                 }
2366         }
2367
2368         for (j = 0; j < 10; j++) {
2369                 for (i = 0; i < rdev->num_crtc; i++) {
2370                         if (crtc_hung & (1 << i)) {
2371                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
2372                                 if (tmp != crtc_status[i])
2373                                         crtc_hung &= ~(1 << i);
2374                         }
2375                 }
2376                 if (crtc_hung == 0)
2377                         return false;
2378                 DRM_UDELAY(100);
2379         }
2380
2381         return true;
2382 }
2383
2384 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
2385 {
2386         u32 reset_mask = 0;
2387         u32 tmp;
2388
2389         /* GRBM_STATUS */
2390         tmp = RREG32(GRBM_STATUS);
2391         if (tmp & (PA_BUSY | SC_BUSY |
2392                    SH_BUSY | SX_BUSY |
2393                    TA_BUSY | VGT_BUSY |
2394                    DB_BUSY | CB_BUSY |
2395                    SPI_BUSY | VGT_BUSY_NO_DMA))
2396                 reset_mask |= RADEON_RESET_GFX;
2397
2398         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
2399                    CP_BUSY | CP_COHERENCY_BUSY))
2400                 reset_mask |= RADEON_RESET_CP;
2401
2402         if (tmp & GRBM_EE_BUSY)
2403                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
2404
2405         /* DMA_STATUS_REG */
2406         tmp = RREG32(DMA_STATUS_REG);
2407         if (!(tmp & DMA_IDLE))
2408                 reset_mask |= RADEON_RESET_DMA;
2409
2410         /* SRBM_STATUS2 */
2411         tmp = RREG32(SRBM_STATUS2);
2412         if (tmp & DMA_BUSY)
2413                 reset_mask |= RADEON_RESET_DMA;
2414
2415         /* SRBM_STATUS */
2416         tmp = RREG32(SRBM_STATUS);
2417         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
2418                 reset_mask |= RADEON_RESET_RLC;
2419
2420         if (tmp & IH_BUSY)
2421                 reset_mask |= RADEON_RESET_IH;
2422
2423         if (tmp & SEM_BUSY)
2424                 reset_mask |= RADEON_RESET_SEM;
2425
2426         if (tmp & GRBM_RQ_PENDING)
2427                 reset_mask |= RADEON_RESET_GRBM;
2428
2429         if (tmp & VMC_BUSY)
2430                 reset_mask |= RADEON_RESET_VMC;
2431
2432         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
2433                    MCC_BUSY | MCD_BUSY))
2434                 reset_mask |= RADEON_RESET_MC;
2435
2436         if (evergreen_is_display_hung(rdev))
2437                 reset_mask |= RADEON_RESET_DISPLAY;
2438
2439         /* VM_L2_STATUS */
2440         tmp = RREG32(VM_L2_STATUS);
2441         if (tmp & L2_BUSY)
2442                 reset_mask |= RADEON_RESET_VMC;
2443
2444         /* Skip MC reset as it's mostly likely not hung, just busy */
2445         if (reset_mask & RADEON_RESET_MC) {
2446                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
2447                 reset_mask &= ~RADEON_RESET_MC;
2448         }
2449
2450         return reset_mask;
2451 }
2452
2453 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
2454 {
2455         struct evergreen_mc_save save;
2456         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
2457         u32 tmp;
2458
2459         if (reset_mask == 0)
2460                 return;
2461
2462         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
2463
2464         evergreen_print_gpu_status_regs(rdev);
2465
2466         /* Disable CP parsing/prefetching */
2467         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2468
2469         if (reset_mask & RADEON_RESET_DMA) {
2470                 /* Disable DMA */
2471                 tmp = RREG32(DMA_RB_CNTL);
2472                 tmp &= ~DMA_RB_ENABLE;
2473                 WREG32(DMA_RB_CNTL, tmp);
2474         }
2475
2476         DRM_UDELAY(50);
2477
2478         evergreen_mc_stop(rdev, &save);
2479         if (evergreen_mc_wait_for_idle(rdev)) {
2480                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2481         }
2482
2483         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
2484                 grbm_soft_reset |= SOFT_RESET_DB |
2485                         SOFT_RESET_CB |
2486                         SOFT_RESET_PA |
2487                         SOFT_RESET_SC |
2488                         SOFT_RESET_SPI |
2489                         SOFT_RESET_SX |
2490                         SOFT_RESET_SH |
2491                         SOFT_RESET_TC |
2492                         SOFT_RESET_TA |
2493                         SOFT_RESET_VC |
2494                         SOFT_RESET_VGT;
2495         }
2496
2497         if (reset_mask & RADEON_RESET_CP) {
2498                 grbm_soft_reset |= SOFT_RESET_CP |
2499                         SOFT_RESET_VGT;
2500
2501                 srbm_soft_reset |= SOFT_RESET_GRBM;
2502         }
2503
2504         if (reset_mask & RADEON_RESET_DMA)
2505                 srbm_soft_reset |= SOFT_RESET_DMA;
2506
2507         if (reset_mask & RADEON_RESET_DISPLAY)
2508                 srbm_soft_reset |= SOFT_RESET_DC;
2509
2510         if (reset_mask & RADEON_RESET_RLC)
2511                 srbm_soft_reset |= SOFT_RESET_RLC;
2512
2513         if (reset_mask & RADEON_RESET_SEM)
2514                 srbm_soft_reset |= SOFT_RESET_SEM;
2515
2516         if (reset_mask & RADEON_RESET_IH)
2517                 srbm_soft_reset |= SOFT_RESET_IH;
2518
2519         if (reset_mask & RADEON_RESET_GRBM)
2520                 srbm_soft_reset |= SOFT_RESET_GRBM;
2521
2522         if (reset_mask & RADEON_RESET_VMC)
2523                 srbm_soft_reset |= SOFT_RESET_VMC;
2524
2525         if (!(rdev->flags & RADEON_IS_IGP)) {
2526                 if (reset_mask & RADEON_RESET_MC)
2527                         srbm_soft_reset |= SOFT_RESET_MC;
2528         }
2529
2530         if (grbm_soft_reset) {
2531                 tmp = RREG32(GRBM_SOFT_RESET);
2532                 tmp |= grbm_soft_reset;
2533                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2534                 WREG32(GRBM_SOFT_RESET, tmp);
2535                 tmp = RREG32(GRBM_SOFT_RESET);
2536
2537                 DRM_UDELAY(50);
2538
2539                 tmp &= ~grbm_soft_reset;
2540                 WREG32(GRBM_SOFT_RESET, tmp);
2541                 tmp = RREG32(GRBM_SOFT_RESET);
2542         }
2543
2544         if (srbm_soft_reset) {
2545                 tmp = RREG32(SRBM_SOFT_RESET);
2546                 tmp |= srbm_soft_reset;
2547                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
2548                 WREG32(SRBM_SOFT_RESET, tmp);
2549                 tmp = RREG32(SRBM_SOFT_RESET);
2550
2551                 DRM_UDELAY(50);
2552
2553                 tmp &= ~srbm_soft_reset;
2554                 WREG32(SRBM_SOFT_RESET, tmp);
2555                 tmp = RREG32(SRBM_SOFT_RESET);
2556         }
2557
2558         /* Wait a little for things to settle down */
2559         DRM_UDELAY(50);
2560
2561         evergreen_mc_resume(rdev, &save);
2562         DRM_UDELAY(50);
2563
2564         evergreen_print_gpu_status_regs(rdev);
2565 }
2566
2567 int evergreen_asic_reset(struct radeon_device *rdev)
2568 {
2569         u32 reset_mask;
2570
2571         reset_mask = evergreen_gpu_check_soft_reset(rdev);
2572
2573         if (reset_mask)
2574                 r600_set_bios_scratch_engine_hung(rdev, true);
2575
2576         evergreen_gpu_soft_reset(rdev, reset_mask);
2577
2578         reset_mask = evergreen_gpu_check_soft_reset(rdev);
2579
2580         if (!reset_mask)
2581                 r600_set_bios_scratch_engine_hung(rdev, false);
2582
2583         return 0;
2584 }
2585
2586 /**
2587  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
2588  *
2589  * @rdev: radeon_device pointer
2590  * @ring: radeon_ring structure holding ring information
2591  *
2592  * Check if the GFX engine is locked up.
2593  * Returns true if the engine appears to be locked up, false if not.
2594  */
2595 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2596 {
2597         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2598
2599         if (!(reset_mask & (RADEON_RESET_GFX |
2600                             RADEON_RESET_COMPUTE |
2601                             RADEON_RESET_CP))) {
2602                 radeon_ring_lockup_update(ring);
2603                 return false;
2604         }
2605         /* force CP activities */
2606         radeon_ring_force_activity(rdev, ring);
2607         return radeon_ring_test_lockup(rdev, ring);
2608 }
2609
2610 /**
2611  * evergreen_dma_is_lockup - Check if the DMA engine is locked up
2612  *
2613  * @rdev: radeon_device pointer
2614  * @ring: radeon_ring structure holding ring information
2615  *
2616  * Check if the async DMA engine is locked up.
2617  * Returns true if the engine appears to be locked up, false if not.
2618  */
2619 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
2620 {
2621         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
2622
2623         if (!(reset_mask & RADEON_RESET_DMA)) {
2624                 radeon_ring_lockup_update(ring);
2625                 return false;
2626         }
2627         /* force ring activities */
2628         radeon_ring_force_activity(rdev, ring);
2629         return radeon_ring_test_lockup(rdev, ring);
2630 }
2631
2632 /* Interrupts */
2633
2634 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2635 {
2636         if (crtc >= rdev->num_crtc)
2637                 return 0;
2638         else
2639                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
2640 }
2641
2642 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2643 {
2644         u32 tmp;
2645
2646         if (rdev->family >= CHIP_CAYMAN) {
2647                 cayman_cp_int_cntl_setup(rdev, 0,
2648                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2649                 cayman_cp_int_cntl_setup(rdev, 1, 0);
2650                 cayman_cp_int_cntl_setup(rdev, 2, 0);
2651                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
2652                 WREG32(CAYMAN_DMA1_CNTL, tmp);
2653         } else
2654                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2655         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2656         WREG32(DMA_CNTL, tmp);
2657         WREG32(GRBM_INT_CNTL, 0);
2658         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2659         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2660         if (rdev->num_crtc >= 4) {
2661                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2662                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2663         }
2664         if (rdev->num_crtc >= 6) {
2665                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2666                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2667         }
2668
2669         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2670         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2671         if (rdev->num_crtc >= 4) {
2672                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2673                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2674         }
2675         if (rdev->num_crtc >= 6) {
2676                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2677                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2678         }
2679
2680         /* only one DAC on DCE6 */
2681         if (!ASIC_IS_DCE6(rdev))
2682                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2683         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2684
2685         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2686         WREG32(DC_HPD1_INT_CONTROL, tmp);
2687         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2688         WREG32(DC_HPD2_INT_CONTROL, tmp);
2689         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2690         WREG32(DC_HPD3_INT_CONTROL, tmp);
2691         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2692         WREG32(DC_HPD4_INT_CONTROL, tmp);
2693         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2694         WREG32(DC_HPD5_INT_CONTROL, tmp);
2695         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2696         WREG32(DC_HPD6_INT_CONTROL, tmp);
2697
2698 }
2699
2700 int evergreen_irq_set(struct radeon_device *rdev)
2701 {
2702         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2703         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2704         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2705         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2706         u32 grbm_int_cntl = 0;
2707         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2708         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
2709         u32 dma_cntl, dma_cntl1 = 0;
2710
2711         if (!rdev->irq.installed) {
2712                 dev_warn(rdev->dev, "Can't enable IRQ/MSI because no handler is installed\n");
2713                 return -EINVAL;
2714         }
2715         /* don't enable anything if the ih is disabled */
2716         if (!rdev->ih.enabled) {
2717                 r600_disable_interrupts(rdev);
2718                 /* force the active interrupt state to all disabled */
2719                 evergreen_disable_interrupt_state(rdev);
2720                 return 0;
2721         }
2722
2723         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2724         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2725         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2726         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2727         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2728         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2729
2730         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2731         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2732         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2733         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2734         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2735         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
2736
2737         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2738
2739         if (rdev->family >= CHIP_CAYMAN) {
2740                 /* enable CP interrupts on all rings */
2741                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2742                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2743                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2744                 }
2745                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
2746                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
2747                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
2748                 }
2749                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
2750                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
2751                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
2752                 }
2753         } else {
2754                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2755                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
2756                         cp_int_cntl |= RB_INT_ENABLE;
2757                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2758                 }
2759         }
2760
2761         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
2762                 DRM_DEBUG("r600_irq_set: sw int dma\n");
2763                 dma_cntl |= TRAP_ENABLE;
2764         }
2765
2766         if (rdev->family >= CHIP_CAYMAN) {
2767                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
2768                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
2769                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
2770                         dma_cntl1 |= TRAP_ENABLE;
2771                 }
2772         }
2773
2774         if (rdev->irq.crtc_vblank_int[0] ||
2775             atomic_read(&rdev->irq.pflip[0])) {
2776                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2777                 crtc1 |= VBLANK_INT_MASK;
2778         }
2779         if (rdev->irq.crtc_vblank_int[1] ||
2780             atomic_read(&rdev->irq.pflip[1])) {
2781                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2782                 crtc2 |= VBLANK_INT_MASK;
2783         }
2784         if (rdev->irq.crtc_vblank_int[2] ||
2785             atomic_read(&rdev->irq.pflip[2])) {
2786                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2787                 crtc3 |= VBLANK_INT_MASK;
2788         }
2789         if (rdev->irq.crtc_vblank_int[3] ||
2790             atomic_read(&rdev->irq.pflip[3])) {
2791                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2792                 crtc4 |= VBLANK_INT_MASK;
2793         }
2794         if (rdev->irq.crtc_vblank_int[4] ||
2795             atomic_read(&rdev->irq.pflip[4])) {
2796                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2797                 crtc5 |= VBLANK_INT_MASK;
2798         }
2799         if (rdev->irq.crtc_vblank_int[5] ||
2800             atomic_read(&rdev->irq.pflip[5])) {
2801                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2802                 crtc6 |= VBLANK_INT_MASK;
2803         }
2804         if (rdev->irq.hpd[0]) {
2805                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2806                 hpd1 |= DC_HPDx_INT_EN;
2807         }
2808         if (rdev->irq.hpd[1]) {
2809                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2810                 hpd2 |= DC_HPDx_INT_EN;
2811         }
2812         if (rdev->irq.hpd[2]) {
2813                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2814                 hpd3 |= DC_HPDx_INT_EN;
2815         }
2816         if (rdev->irq.hpd[3]) {
2817                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2818                 hpd4 |= DC_HPDx_INT_EN;
2819         }
2820         if (rdev->irq.hpd[4]) {
2821                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2822                 hpd5 |= DC_HPDx_INT_EN;
2823         }
2824         if (rdev->irq.hpd[5]) {
2825                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2826                 hpd6 |= DC_HPDx_INT_EN;
2827         }
2828         if (rdev->irq.afmt[0]) {
2829                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
2830                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
2831         }
2832         if (rdev->irq.afmt[1]) {
2833                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
2834                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
2835         }
2836         if (rdev->irq.afmt[2]) {
2837                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
2838                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
2839         }
2840         if (rdev->irq.afmt[3]) {
2841                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
2842                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
2843         }
2844         if (rdev->irq.afmt[4]) {
2845                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
2846                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
2847         }
2848         if (rdev->irq.afmt[5]) {
2849                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
2850                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
2851         }
2852
2853         if (rdev->family >= CHIP_CAYMAN) {
2854                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
2855                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
2856                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
2857         } else
2858                 WREG32(CP_INT_CNTL, cp_int_cntl);
2859
2860         WREG32(DMA_CNTL, dma_cntl);
2861
2862         if (rdev->family >= CHIP_CAYMAN)
2863                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
2864
2865         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2866
2867         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2868         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2869         if (rdev->num_crtc >= 4) {
2870                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2871                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2872         }
2873         if (rdev->num_crtc >= 6) {
2874                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2875                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2876         }
2877
2878         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2879         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2880         if (rdev->num_crtc >= 4) {
2881                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2882                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2883         }
2884         if (rdev->num_crtc >= 6) {
2885                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2886                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2887         }
2888
2889         WREG32(DC_HPD1_INT_CONTROL, hpd1);
2890         WREG32(DC_HPD2_INT_CONTROL, hpd2);
2891         WREG32(DC_HPD3_INT_CONTROL, hpd3);
2892         WREG32(DC_HPD4_INT_CONTROL, hpd4);
2893         WREG32(DC_HPD5_INT_CONTROL, hpd5);
2894         WREG32(DC_HPD6_INT_CONTROL, hpd6);
2895
2896         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
2897         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
2898         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
2899         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
2900         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
2901         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
2902
2903         return 0;
2904 }
2905
2906 static void evergreen_irq_ack(struct radeon_device *rdev)
2907 {
2908         u32 tmp;
2909
2910         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2911         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2912         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2913         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2914         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2915         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2916         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2917         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2918         if (rdev->num_crtc >= 4) {
2919                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2920                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2921         }
2922         if (rdev->num_crtc >= 6) {
2923                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2924                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2925         }
2926
2927         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2928         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2929         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2930         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2931         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2932         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2933
2934         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2935                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2936         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2937                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2938         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2939                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2940         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2941                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2942         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2943                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2944         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2945                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2946
2947         if (rdev->num_crtc >= 4) {
2948                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2949                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2950                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2951                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2952                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2953                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2954                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2955                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2956                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2957                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2958                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2959                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2960         }
2961
2962         if (rdev->num_crtc >= 6) {
2963                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2964                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2965                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2966                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2967                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2968                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2969                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2970                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2971                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2972                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2973                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2974                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2975         }
2976
2977         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2978                 tmp = RREG32(DC_HPD1_INT_CONTROL);
2979                 tmp |= DC_HPDx_INT_ACK;
2980                 WREG32(DC_HPD1_INT_CONTROL, tmp);
2981         }
2982         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2983                 tmp = RREG32(DC_HPD2_INT_CONTROL);
2984                 tmp |= DC_HPDx_INT_ACK;
2985                 WREG32(DC_HPD2_INT_CONTROL, tmp);
2986         }
2987         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2988                 tmp = RREG32(DC_HPD3_INT_CONTROL);
2989                 tmp |= DC_HPDx_INT_ACK;
2990                 WREG32(DC_HPD3_INT_CONTROL, tmp);
2991         }
2992         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2993                 tmp = RREG32(DC_HPD4_INT_CONTROL);
2994                 tmp |= DC_HPDx_INT_ACK;
2995                 WREG32(DC_HPD4_INT_CONTROL, tmp);
2996         }
2997         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2998                 tmp = RREG32(DC_HPD5_INT_CONTROL);
2999                 tmp |= DC_HPDx_INT_ACK;
3000                 WREG32(DC_HPD5_INT_CONTROL, tmp);
3001         }
3002         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3003                 tmp = RREG32(DC_HPD5_INT_CONTROL);
3004                 tmp |= DC_HPDx_INT_ACK;
3005                 WREG32(DC_HPD6_INT_CONTROL, tmp);
3006         }
3007         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
3008                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
3009                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3010                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
3011         }
3012         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
3013                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
3014                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3015                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
3016         }
3017         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
3018                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
3019                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3020                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
3021         }
3022         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
3023                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
3024                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3025                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
3026         }
3027         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
3028                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
3029                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3030                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
3031         }
3032         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
3033                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
3034                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3035                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
3036         }
3037 }
3038
3039 static void evergreen_irq_disable(struct radeon_device *rdev)
3040 {
3041         r600_disable_interrupts(rdev);
3042         /* Wait and acknowledge irq */
3043         DRM_MDELAY(1);
3044         evergreen_irq_ack(rdev);
3045         evergreen_disable_interrupt_state(rdev);
3046 }
3047
3048 void evergreen_irq_suspend(struct radeon_device *rdev)
3049 {
3050         evergreen_irq_disable(rdev);
3051         r600_rlc_stop(rdev);
3052 }
3053
3054 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
3055 {
3056         u32 wptr, tmp;
3057
3058         if (rdev->wb.enabled)
3059                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3060         else
3061                 wptr = RREG32(IH_RB_WPTR);
3062
3063         if (wptr & RB_OVERFLOW) {
3064                 /* When a ring buffer overflow happen start parsing interrupt
3065                  * from the last not overwritten vector (wptr + 16). Hopefully
3066                  * this should allow us to catchup.
3067                  */
3068                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3069                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3070                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3071                 tmp = RREG32(IH_RB_CNTL);
3072                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3073                 WREG32(IH_RB_CNTL, tmp);
3074         }
3075         return (wptr & rdev->ih.ptr_mask);
3076 }
3077
3078 irqreturn_t evergreen_irq_process(struct radeon_device *rdev)
3079 {
3080         u32 wptr;
3081         u32 rptr;
3082         u32 src_id, src_data;
3083         u32 ring_index;
3084         bool queue_hotplug = false;
3085         bool queue_hdmi = false;
3086
3087         if (!rdev->ih.enabled || rdev->shutdown)
3088                 return IRQ_NONE;
3089
3090         wptr = evergreen_get_ih_wptr(rdev);
3091
3092 restart_ih:
3093         /* is somebody else already processing irqs? */
3094         if (atomic_xchg(&rdev->ih.lock, 1))
3095                 return IRQ_NONE;
3096
3097         rptr = rdev->ih.rptr;
3098         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3099
3100         /* Order reading of wptr vs. reading of IH ring data */
3101         cpu_lfence();
3102
3103         /* display interrupts */
3104         evergreen_irq_ack(rdev);
3105
3106         while (rptr != wptr) {
3107                 /* wptr/rptr are in bytes! */
3108                 ring_index = rptr / 4;
3109                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3110                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3111
3112                 switch (src_id) {
3113                 case 1: /* D1 vblank/vline */
3114                         switch (src_data) {
3115                         case 0: /* D1 vblank */
3116                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
3117                                         if (rdev->irq.crtc_vblank_int[0]) {
3118                                                 drm_handle_vblank(rdev->ddev, 0);
3119                                                 rdev->pm.vblank_sync = true;
3120                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3121                                         }
3122                                         if (atomic_read(&rdev->irq.pflip[0]))
3123                                                 radeon_crtc_handle_flip(rdev, 0);
3124                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3125                                         DRM_DEBUG("IH: D1 vblank\n");
3126                                 }
3127                                 break;
3128                         case 1: /* D1 vline */
3129                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
3130                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3131                                         DRM_DEBUG("IH: D1 vline\n");
3132                                 }
3133                                 break;
3134                         default:
3135                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3136                                 break;
3137                         }
3138                         break;
3139                 case 2: /* D2 vblank/vline */
3140                         switch (src_data) {
3141                         case 0: /* D2 vblank */
3142                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
3143                                         if (rdev->irq.crtc_vblank_int[1]) {
3144                                                 drm_handle_vblank(rdev->ddev, 1);
3145                                                 rdev->pm.vblank_sync = true;
3146                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3147                                         }
3148                                         if (atomic_read(&rdev->irq.pflip[1]))
3149                                                 radeon_crtc_handle_flip(rdev, 1);
3150                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
3151                                         DRM_DEBUG("IH: D2 vblank\n");
3152                                 }
3153                                 break;
3154                         case 1: /* D2 vline */
3155                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
3156                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
3157                                         DRM_DEBUG("IH: D2 vline\n");
3158                                 }
3159                                 break;
3160                         default:
3161                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3162                                 break;
3163                         }
3164                         break;
3165                 case 3: /* D3 vblank/vline */
3166                         switch (src_data) {
3167                         case 0: /* D3 vblank */
3168                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
3169                                         if (rdev->irq.crtc_vblank_int[2]) {
3170                                                 drm_handle_vblank(rdev->ddev, 2);
3171                                                 rdev->pm.vblank_sync = true;
3172                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3173                                         }
3174                                         if (atomic_read(&rdev->irq.pflip[2]))
3175                                                 radeon_crtc_handle_flip(rdev, 2);
3176                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
3177                                         DRM_DEBUG("IH: D3 vblank\n");
3178                                 }
3179                                 break;
3180                         case 1: /* D3 vline */
3181                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
3182                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
3183                                         DRM_DEBUG("IH: D3 vline\n");
3184                                 }
3185                                 break;
3186                         default:
3187                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3188                                 break;
3189                         }
3190                         break;
3191                 case 4: /* D4 vblank/vline */
3192                         switch (src_data) {
3193                         case 0: /* D4 vblank */
3194                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
3195                                         if (rdev->irq.crtc_vblank_int[3]) {
3196                                                 drm_handle_vblank(rdev->ddev, 3);
3197                                                 rdev->pm.vblank_sync = true;
3198                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3199                                         }
3200                                         if (atomic_read(&rdev->irq.pflip[3]))
3201                                                 radeon_crtc_handle_flip(rdev, 3);
3202                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
3203                                         DRM_DEBUG("IH: D4 vblank\n");
3204                                 }
3205                                 break;
3206                         case 1: /* D4 vline */
3207                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
3208                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
3209                                         DRM_DEBUG("IH: D4 vline\n");
3210                                 }
3211                                 break;
3212                         default:
3213                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3214                                 break;
3215                         }
3216                         break;
3217                 case 5: /* D5 vblank/vline */
3218                         switch (src_data) {
3219                         case 0: /* D5 vblank */
3220                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
3221                                         if (rdev->irq.crtc_vblank_int[4]) {
3222                                                 drm_handle_vblank(rdev->ddev, 4);
3223                                                 rdev->pm.vblank_sync = true;
3224                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3225                                         }
3226                                         if (atomic_read(&rdev->irq.pflip[4]))
3227                                                 radeon_crtc_handle_flip(rdev, 4);
3228                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
3229                                         DRM_DEBUG("IH: D5 vblank\n");
3230                                 }
3231                                 break;
3232                         case 1: /* D5 vline */
3233                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
3234                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
3235                                         DRM_DEBUG("IH: D5 vline\n");
3236                                 }
3237                                 break;
3238                         default:
3239                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3240                                 break;
3241                         }
3242                         break;
3243                 case 6: /* D6 vblank/vline */
3244                         switch (src_data) {
3245                         case 0: /* D6 vblank */
3246                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
3247                                         if (rdev->irq.crtc_vblank_int[5]) {
3248                                                 drm_handle_vblank(rdev->ddev, 5);
3249                                                 rdev->pm.vblank_sync = true;
3250                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3251                                         }
3252                                         if (atomic_read(&rdev->irq.pflip[5]))
3253                                                 radeon_crtc_handle_flip(rdev, 5);
3254                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
3255                                         DRM_DEBUG("IH: D6 vblank\n");
3256                                 }
3257                                 break;
3258                         case 1: /* D6 vline */
3259                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
3260                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
3261                                         DRM_DEBUG("IH: D6 vline\n");
3262                                 }
3263                                 break;
3264                         default:
3265                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3266                                 break;
3267                         }
3268                         break;
3269                 case 42: /* HPD hotplug */
3270                         switch (src_data) {
3271                         case 0:
3272                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3273                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
3274                                         queue_hotplug = true;
3275                                         DRM_DEBUG("IH: HPD1\n");
3276                                 }
3277                                 break;
3278                         case 1:
3279                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3280                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
3281                                         queue_hotplug = true;
3282                                         DRM_DEBUG("IH: HPD2\n");
3283                                 }
3284                                 break;
3285                         case 2:
3286                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3287                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
3288                                         queue_hotplug = true;
3289                                         DRM_DEBUG("IH: HPD3\n");
3290                                 }
3291                                 break;
3292                         case 3:
3293                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3294                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
3295                                         queue_hotplug = true;
3296                                         DRM_DEBUG("IH: HPD4\n");
3297                                 }
3298                                 break;
3299                         case 4:
3300                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3301                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
3302                                         queue_hotplug = true;
3303                                         DRM_DEBUG("IH: HPD5\n");
3304                                 }
3305                                 break;
3306                         case 5:
3307                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3308                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
3309                                         queue_hotplug = true;
3310                                         DRM_DEBUG("IH: HPD6\n");
3311                                 }
3312                                 break;
3313                         default:
3314                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3315                                 break;
3316                         }
3317                         break;
3318                 case 44: /* hdmi */
3319                         switch (src_data) {
3320                         case 0:
3321                                 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
3322                                         rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
3323                                         queue_hdmi = true;
3324                                         DRM_DEBUG("IH: HDMI0\n");
3325                                 }
3326                                 break;
3327                         case 1:
3328                                 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
3329                                         rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
3330                                         queue_hdmi = true;
3331                                         DRM_DEBUG("IH: HDMI1\n");
3332                                 }
3333                                 break;
3334                         case 2:
3335                                 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
3336                                         rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
3337                                         queue_hdmi = true;
3338                                         DRM_DEBUG("IH: HDMI2\n");
3339                                 }
3340                                 break;
3341                         case 3:
3342                                 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
3343                                         rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
3344                                         queue_hdmi = true;
3345                                         DRM_DEBUG("IH: HDMI3\n");
3346                                 }
3347                                 break;
3348                         case 4:
3349                                 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
3350                                         rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
3351                                         queue_hdmi = true;
3352                                         DRM_DEBUG("IH: HDMI4\n");
3353                                 }
3354                                 break;
3355                         case 5:
3356                                 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
3357                                         rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
3358                                         queue_hdmi = true;
3359                                         DRM_DEBUG("IH: HDMI5\n");
3360                                 }
3361                                 break;
3362                         default:
3363                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
3364                                 break;
3365                         }
3366                         break;
3367                 case 146:
3368                 case 147:
3369                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
3370                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3371                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3372                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3373                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3374                         /* reset addr and status */
3375                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
3376                         break;
3377                 case 176: /* CP_INT in ring buffer */
3378                 case 177: /* CP_INT in IB1 */
3379                 case 178: /* CP_INT in IB2 */
3380                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
3381                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3382                         break;
3383                 case 181: /* CP EOP event */
3384                         DRM_DEBUG("IH: CP EOP\n");
3385                         if (rdev->family >= CHIP_CAYMAN) {
3386                                 switch (src_data) {
3387                                 case 0:
3388                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3389                                         break;
3390                                 case 1:
3391                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
3392                                         break;
3393                                 case 2:
3394                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
3395                                         break;
3396                                 }
3397                         } else
3398                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
3399                         break;
3400                 case 224: /* DMA trap event */
3401                         DRM_DEBUG("IH: DMA trap\n");
3402                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
3403                         break;
3404                 case 233: /* GUI IDLE */
3405                         DRM_DEBUG("IH: GUI idle\n");
3406                         break;
3407                 case 244: /* DMA trap event */
3408                         if (rdev->family >= CHIP_CAYMAN) {
3409                                 DRM_DEBUG("IH: DMA1 trap\n");
3410                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
3411                         }
3412                         break;
3413                 default:
3414                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3415                         break;
3416                 }
3417
3418                 /* wptr/rptr are in bytes! */
3419                 rptr += 16;
3420                 rptr &= rdev->ih.ptr_mask;
3421         }
3422         if (queue_hotplug)
3423                 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
3424         if (queue_hdmi)
3425                 taskqueue_enqueue(rdev->tq, &rdev->audio_work);
3426         rdev->ih.rptr = rptr;
3427         WREG32(IH_RB_RPTR, rdev->ih.rptr);
3428         atomic_set(&rdev->ih.lock, 0);
3429
3430         /* make sure wptr hasn't changed while processing */
3431         wptr = evergreen_get_ih_wptr(rdev);
3432         if (wptr != rptr)
3433                 goto restart_ih;
3434
3435         return IRQ_HANDLED;
3436 }
3437
3438 /**
3439  * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
3440  *
3441  * @rdev: radeon_device pointer
3442  * @fence: radeon fence object
3443  *
3444  * Add a DMA fence packet to the ring to write
3445  * the fence seq number and DMA trap packet to generate
3446  * an interrupt if needed (evergreen-SI).
3447  */
3448 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
3449                                    struct radeon_fence *fence)
3450 {
3451         struct radeon_ring *ring = &rdev->ring[fence->ring];
3452         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3453         /* write the fence */
3454         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
3455         radeon_ring_write(ring, addr & 0xfffffffc);
3456         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
3457         radeon_ring_write(ring, fence->seq);
3458         /* generate an interrupt */
3459         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
3460         /* flush HDP */
3461         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
3462         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
3463         radeon_ring_write(ring, 1);
3464 }
3465
3466 /**
3467  * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
3468  *
3469  * @rdev: radeon_device pointer
3470  * @ib: IB object to schedule
3471  *
3472  * Schedule an IB in the DMA ring (evergreen).
3473  */
3474 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
3475                                    struct radeon_ib *ib)
3476 {
3477         struct radeon_ring *ring = &rdev->ring[ib->ring];
3478
3479         if (rdev->wb.enabled) {
3480                 u32 next_rptr = ring->wptr + 4;
3481                 while ((next_rptr & 7) != 5)
3482                         next_rptr++;
3483                 next_rptr += 3;
3484                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
3485                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3486                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3487                 radeon_ring_write(ring, next_rptr);
3488         }
3489
3490         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3491          * Pad as necessary with NOPs.
3492          */
3493         while ((ring->wptr & 7) != 5)
3494                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
3495         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
3496         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3497         radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3498
3499 }
3500
3501 /**
3502  * evergreen_copy_dma - copy pages using the DMA engine
3503  *
3504  * @rdev: radeon_device pointer
3505  * @src_offset: src GPU address
3506  * @dst_offset: dst GPU address
3507  * @num_gpu_pages: number of GPU pages to xfer
3508  * @fence: radeon fence object
3509  *
3510  * Copy GPU paging using the DMA engine (evergreen-cayman).
3511  * Used by the radeon ttm implementation to move pages if
3512  * registered as the asic copy callback.
3513  */
3514 int evergreen_copy_dma(struct radeon_device *rdev,
3515                        uint64_t src_offset, uint64_t dst_offset,
3516                        unsigned num_gpu_pages,
3517                        struct radeon_fence **fence)
3518 {
3519         struct radeon_semaphore *sem = NULL;
3520         int ring_index = rdev->asic->copy.dma_ring_index;
3521         struct radeon_ring *ring = &rdev->ring[ring_index];
3522         u32 size_in_dw, cur_size_in_dw;
3523         int i, num_loops;
3524         int r = 0;
3525
3526         r = radeon_semaphore_create(rdev, &sem);
3527         if (r) {
3528                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3529                 return r;
3530         }
3531
3532         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
3533         num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
3534         r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
3535         if (r) {
3536                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3537                 radeon_semaphore_free(rdev, &sem, NULL);
3538                 return r;
3539         }
3540
3541         if (radeon_fence_need_sync(*fence, ring->idx)) {
3542                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3543                                             ring->idx);
3544                 radeon_fence_note_sync(*fence, ring->idx);
3545         } else {
3546                 radeon_semaphore_free(rdev, &sem, NULL);
3547         }
3548
3549         for (i = 0; i < num_loops; i++) {
3550                 cur_size_in_dw = size_in_dw;
3551                 if (cur_size_in_dw > 0xFFFFF)
3552                         cur_size_in_dw = 0xFFFFF;
3553                 size_in_dw -= cur_size_in_dw;
3554                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
3555                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
3556                 radeon_ring_write(ring, src_offset & 0xfffffffc);
3557                 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
3558                 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
3559                 src_offset += cur_size_in_dw * 4;
3560                 dst_offset += cur_size_in_dw * 4;
3561         }
3562
3563         r = radeon_fence_emit(rdev, fence, ring->idx);
3564         if (r) {
3565                 radeon_ring_unlock_undo(rdev, ring);
3566                 return r;
3567         }
3568
3569         radeon_ring_unlock_commit(rdev, ring);
3570         radeon_semaphore_free(rdev, &sem, *fence);
3571
3572         return r;
3573 }
3574
3575 static int evergreen_startup(struct radeon_device *rdev)
3576 {
3577         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3578         int r;
3579
3580         /* enable pcie gen2 link */
3581         evergreen_pcie_gen2_enable(rdev);
3582
3583         if (ASIC_IS_DCE5(rdev)) {
3584                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3585                         r = ni_init_microcode(rdev);
3586                         if (r) {
3587                                 DRM_ERROR("Failed to load firmware!\n");
3588                                 return r;
3589                         }
3590                 }
3591                 r = ni_mc_load_microcode(rdev);
3592                 if (r) {
3593                         DRM_ERROR("Failed to load MC firmware!\n");
3594                         return r;
3595                 }
3596         } else {
3597                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3598                         r = r600_init_microcode(rdev);
3599                         if (r) {
3600                                 DRM_ERROR("Failed to load firmware!\n");
3601                                 return r;
3602                         }
3603                 }
3604         }
3605
3606         r = r600_vram_scratch_init(rdev);
3607         if (r)
3608                 return r;
3609
3610         evergreen_mc_program(rdev);
3611         if (rdev->flags & RADEON_IS_AGP) {
3612                 evergreen_agp_enable(rdev);
3613         } else {
3614                 r = evergreen_pcie_gart_enable(rdev);
3615                 if (r)
3616                         return r;
3617         }
3618         evergreen_gpu_init(rdev);
3619
3620         r = evergreen_blit_init(rdev);
3621         if (r) {
3622                 r600_blit_fini(rdev);
3623                 rdev->asic->copy.copy = NULL;
3624                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3625         }
3626
3627         /* allocate wb buffer */
3628         r = radeon_wb_init(rdev);
3629         if (r)
3630                 return r;
3631
3632         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
3633         if (r) {
3634                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
3635                 return r;
3636         }
3637
3638         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
3639         if (r) {
3640                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
3641                 return r;
3642         }
3643
3644         /* Enable IRQ */
3645         r = r600_irq_init(rdev);
3646         if (r) {
3647                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
3648                 radeon_irq_kms_fini(rdev);
3649                 return r;
3650         }
3651         evergreen_irq_set(rdev);
3652
3653         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
3654                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
3655                              0, 0xfffff, RADEON_CP_PACKET2);
3656         if (r)
3657                 return r;
3658
3659         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3660         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
3661                              DMA_RB_RPTR, DMA_RB_WPTR,
3662                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
3663         if (r)
3664                 return r;
3665
3666         r = evergreen_cp_load_microcode(rdev);
3667         if (r)
3668                 return r;
3669         r = evergreen_cp_resume(rdev);
3670         if (r)
3671                 return r;
3672         r = r600_dma_resume(rdev);
3673         if (r)
3674                 return r;
3675
3676         r = radeon_ib_pool_init(rdev);
3677         if (r) {
3678                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
3679                 return r;
3680         }
3681
3682         r = r600_audio_init(rdev);
3683         if (r) {
3684                 DRM_ERROR("radeon: audio init failed\n");
3685                 return r;
3686         }
3687
3688         return 0;
3689 }
3690
3691 int evergreen_resume(struct radeon_device *rdev)
3692 {
3693         int r;
3694
3695         /* reset the asic, the gfx blocks are often in a bad state
3696          * after the driver is unloaded or after a resume
3697          */
3698         if (radeon_asic_reset(rdev))
3699                 dev_warn(rdev->dev, "GPU reset failed !\n");
3700         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
3701          * posting will perform necessary task to bring back GPU into good
3702          * shape.
3703          */
3704         /* post card */
3705         atom_asic_init(rdev->mode_info.atom_context);
3706
3707         rdev->accel_working = true;
3708         r = evergreen_startup(rdev);
3709         if (r) {
3710                 DRM_ERROR("evergreen startup failed on resume\n");
3711                 rdev->accel_working = false;
3712                 return r;
3713         }
3714
3715         return r;
3716
3717 }
3718
3719 int evergreen_suspend(struct radeon_device *rdev)
3720 {
3721         r600_audio_fini(rdev);
3722         r700_cp_stop(rdev);
3723         r600_dma_stop(rdev);
3724         evergreen_irq_suspend(rdev);
3725         radeon_wb_disable(rdev);
3726         evergreen_pcie_gart_disable(rdev);
3727
3728         return 0;
3729 }
3730
3731 /* Plan is to move initialization in that function and use
3732  * helper function so that radeon_device_init pretty much
3733  * do nothing more than calling asic specific function. This
3734  * should also allow to remove a bunch of callback function
3735  * like vram_info.
3736  */
3737 int evergreen_init(struct radeon_device *rdev)
3738 {
3739         int r;
3740
3741         /* Read BIOS */
3742         if (!radeon_get_bios(rdev)) {
3743                 if (ASIC_IS_AVIVO(rdev))
3744                         return -EINVAL;
3745         }
3746         /* Must be an ATOMBIOS */
3747         if (!rdev->is_atom_bios) {
3748                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
3749                 return -EINVAL;
3750         }
3751         r = radeon_atombios_init(rdev);
3752         if (r)
3753                 return r;
3754         /* reset the asic, the gfx blocks are often in a bad state
3755          * after the driver is unloaded or after a resume
3756          */
3757         if (radeon_asic_reset(rdev))
3758                 dev_warn(rdev->dev, "GPU reset failed !\n");
3759         /* Post card if necessary */
3760         if (!radeon_card_posted(rdev)) {
3761                 if (!rdev->bios) {
3762                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3763                         return -EINVAL;
3764                 }
3765                 DRM_INFO("GPU not posted. posting now...\n");
3766                 atom_asic_init(rdev->mode_info.atom_context);
3767         }
3768         /* Initialize scratch registers */
3769         r600_scratch_init(rdev);
3770         /* Initialize surface registers */
3771         radeon_surface_init(rdev);
3772         /* Initialize clocks */
3773         radeon_get_clock_info(rdev->ddev);
3774         /* Fence driver */
3775         r = radeon_fence_driver_init(rdev);
3776         if (r)
3777                 return r;
3778         /* initialize AGP */
3779         if (rdev->flags & RADEON_IS_AGP) {
3780                 r = radeon_agp_init(rdev);
3781                 if (r)
3782                         radeon_agp_disable(rdev);
3783         }
3784         /* initialize memory controller */
3785         r = evergreen_mc_init(rdev);
3786         if (r)
3787                 return r;
3788         /* Memory manager */
3789         r = radeon_bo_init(rdev);
3790         if (r)
3791                 return r;
3792
3793         r = radeon_irq_kms_init(rdev);
3794         if (r)
3795                 return r;
3796
3797         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3798         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3799
3800         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3801         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3802
3803         rdev->ih.ring_obj = NULL;
3804         r600_ih_ring_init(rdev, 64 * 1024);
3805
3806         r = r600_pcie_gart_init(rdev);
3807         if (r)
3808                 return r;
3809
3810         rdev->accel_working = true;
3811         r = evergreen_startup(rdev);
3812         if (r) {
3813                 dev_err(rdev->dev, "disabling GPU acceleration\n");
3814                 r700_cp_fini(rdev);
3815                 r600_dma_fini(rdev);
3816                 r600_irq_fini(rdev);
3817                 radeon_wb_fini(rdev);
3818                 radeon_ib_pool_fini(rdev);
3819                 radeon_irq_kms_fini(rdev);
3820                 evergreen_pcie_gart_fini(rdev);
3821                 rdev->accel_working = false;
3822         }
3823
3824         /* Don't start up if the MC ucode is missing on BTC parts.
3825          * The default clocks and voltages before the MC ucode
3826          * is loaded are not suffient for advanced operations.
3827          */
3828         if (ASIC_IS_DCE5(rdev)) {
3829                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
3830                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
3831                         return -EINVAL;
3832                 }
3833         }
3834
3835         return 0;
3836 }
3837
3838 void evergreen_fini(struct radeon_device *rdev)
3839 {
3840         r600_audio_fini(rdev);
3841         r600_blit_fini(rdev);
3842         r700_cp_fini(rdev);
3843         r600_dma_fini(rdev);
3844         r600_irq_fini(rdev);
3845         radeon_wb_fini(rdev);
3846         radeon_ib_pool_fini(rdev);
3847         radeon_irq_kms_fini(rdev);
3848         evergreen_pcie_gart_fini(rdev);
3849         r600_vram_scratch_fini(rdev);
3850         radeon_gem_fini(rdev);
3851         radeon_fence_driver_fini(rdev);
3852         radeon_agp_fini(rdev);
3853         radeon_bo_fini(rdev);
3854         radeon_atombios_fini(rdev);
3855         if (ASIC_IS_DCE5(rdev))
3856                 ni_fini_microcode(rdev);
3857         else
3858                 r600_fini_microcode(rdev);
3859         drm_free(rdev->bios, M_DRM);
3860         rdev->bios = NULL;
3861 }
3862
3863 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
3864 {
3865         u32 link_width_cntl, speed_cntl, mask;
3866         int ret;
3867
3868         if (radeon_pcie_gen2 == 0)
3869                 return;
3870
3871         if (rdev->flags & RADEON_IS_IGP)
3872                 return;
3873
3874         if (!(rdev->flags & RADEON_IS_PCIE))
3875                 return;
3876
3877         /* x2 cards have a special sequence */
3878         if (ASIC_IS_X2(rdev))
3879                 return;
3880
3881         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
3882         if (ret != 0)
3883                 return;
3884
3885         if (!(mask & DRM_PCIE_SPEED_50))
3886                 return;
3887
3888         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3889         if (speed_cntl & LC_CURRENT_DATA_RATE) {
3890                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
3891                 return;
3892         }
3893
3894         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
3895
3896         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3897             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3898
3899                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3900                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3901                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3902
3903                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3904                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3905                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3906
3907                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3908                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3909                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3910
3911                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3912                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3913                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3914
3915                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3916                 speed_cntl |= LC_GEN2_EN_STRAP;
3917                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3918
3919         } else {
3920                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3921                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3922                 if (1)
3923                         link_width_cntl |= LC_UPCONFIGURE_DIS;
3924                 else
3925                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3926                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3927         }
3928 }