sys/dev/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include "cikd.h"
  32 #include "atom.h"
  33 #include "cik_blit_shaders.h"
  34 #include "radeon_ucode.h"
  35 #include "clearstate_ci.h"
  36
  37 #define SH_MEM_CONFIG_GFX_DEFAULT \
  38         ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
  39
  40 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  45 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  46 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  47 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  48 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  49
  50 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
  51 MODULE_FIRMWARE("radeon/bonaire_me.bin");
  52 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
  53 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
  54 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
  55 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
  56 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
  57 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
  58 MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
  59
  60 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  61 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  62 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  63 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  64 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  65 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  66 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  67 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  68 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  69
  70 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
  71 MODULE_FIRMWARE("radeon/hawaii_me.bin");
  72 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
  73 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
  74 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
  75 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
  76 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
  77 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
  78 MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
  79
  80 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  81 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  82 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  83 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  84 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  85 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  86
  87 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
  88 MODULE_FIRMWARE("radeon/kaveri_me.bin");
  89 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
  90 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
  91 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
  92 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
  93 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
  94
  95 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  96 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  97 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  98 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  99 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
 100 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
 101
 102 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
 103 MODULE_FIRMWARE("radeon/kabini_me.bin");
 104 MODULE_FIRMWARE("radeon/kabini_ce.bin");
 105 MODULE_FIRMWARE("radeon/kabini_mec.bin");
 106 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
 107 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
 108
 109 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
 110 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
 111 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
 112 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
 113 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
 114 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
 115
 116 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
 117 MODULE_FIRMWARE("radeon/mullins_me.bin");
 118 MODULE_FIRMWARE("radeon/mullins_ce.bin");
 119 MODULE_FIRMWARE("radeon/mullins_mec.bin");
 120 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
 121 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
 122
 123 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 124 static void cik_rlc_stop(struct radeon_device *rdev);
 125 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 126 static void cik_program_aspm(struct radeon_device *rdev);
 127 static void cik_init_pg(struct radeon_device *rdev);
 128 static void cik_init_cg(struct radeon_device *rdev);
 129 static void cik_fini_pg(struct radeon_device *rdev);
 130 static void cik_fini_cg(struct radeon_device *rdev);
 131 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
 132                                           bool enable);
 133
 134 /**
 135  * cik_get_allowed_info_register - fetch the register for the info ioctl
 136  *
 137  * @rdev: radeon_device pointer
 138  * @reg: register offset in bytes
 139  * @val: register value
 140  *
 141  * Returns 0 for success or -EINVAL for an invalid register
 142  *
 143  */
 144 int cik_get_allowed_info_register(struct radeon_device *rdev,
 145                                   u32 reg, u32 *val)
 146 {
 147         switch (reg) {
 148         case GRBM_STATUS:
 149         case GRBM_STATUS2:
 150         case GRBM_STATUS_SE0:
 151         case GRBM_STATUS_SE1:
 152         case GRBM_STATUS_SE2:
 153         case GRBM_STATUS_SE3:
 154         case SRBM_STATUS:
 155         case SRBM_STATUS2:
 156         case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
 157         case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
 158         case UVD_STATUS:
 159         /* TODO VCE */
 160                 *val = RREG32(reg);
 161                 return 0;
 162         default:
 163                 return -EINVAL;
 164         }
 165 }
 166
 167 /*
 168  * Indirect registers accessor
 169  */
 170 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
 171 {
 172         unsigned long flags;
 173         u32 r;
 174
 175         spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 176         WREG32(CIK_DIDT_IND_INDEX, (reg));
 177         r = RREG32(CIK_DIDT_IND_DATA);
 178         spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 179         return r;
 180 }
 181
 182 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 183 {
 184         unsigned long flags;
 185
 186         spin_lock_irqsave(&rdev->didt_idx_lock, flags);
 187         WREG32(CIK_DIDT_IND_INDEX, (reg));
 188         WREG32(CIK_DIDT_IND_DATA, (v));
 189         spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
 190 }
 191
 192 /* get temperature in millidegrees */
 193 int ci_get_temp(struct radeon_device *rdev)
 194 {
 195         u32 temp;
 196         int actual_temp = 0;
 197
 198         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 199                 CTF_TEMP_SHIFT;
 200
 201         if (temp & 0x200)
 202                 actual_temp = 255;
 203         else
 204                 actual_temp = temp & 0x1ff;
 205
 206         actual_temp = actual_temp * 1000;
 207
 208         return actual_temp;
 209 }
 210
 211 /* get temperature in millidegrees */
 212 int kv_get_temp(struct radeon_device *rdev)
 213 {
 214         u32 temp;
 215         int actual_temp = 0;
 216
 217         temp = RREG32_SMC(0xC0300E0C);
 218
 219         if (temp)
 220                 actual_temp = (temp / 8) - 49;
 221         else
 222                 actual_temp = 0;
 223
 224         actual_temp = actual_temp * 1000;
 225
 226         return actual_temp;
 227 }
 228
 229 /*
 230  * Indirect registers accessor
 231  */
 232 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 233 {
 234         unsigned long flags;
 235         u32 r;
 236
 237         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 238         WREG32(PCIE_INDEX, reg);
 239         (void)RREG32(PCIE_INDEX);
 240         r = RREG32(PCIE_DATA);
 241         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 242         return r;
 243 }
 244
 245 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 246 {
 247         unsigned long flags;
 248
 249         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 250         WREG32(PCIE_INDEX, reg);
 251         (void)RREG32(PCIE_INDEX);
 252         WREG32(PCIE_DATA, v);
 253         (void)RREG32(PCIE_DATA);
 254         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 255 }
 256
 257 static const u32 spectre_rlc_save_restore_register_list[] =
 258 {
 259         (0x0e00 << 16) | (0xc12c >> 2),
 260         0x00000000,
 261         (0x0e00 << 16) | (0xc140 >> 2),
 262         0x00000000,
 263         (0x0e00 << 16) | (0xc150 >> 2),
 264         0x00000000,
 265         (0x0e00 << 16) | (0xc15c >> 2),
 266         0x00000000,
 267         (0x0e00 << 16) | (0xc168 >> 2),
 268         0x00000000,
 269         (0x0e00 << 16) | (0xc170 >> 2),
 270         0x00000000,
 271         (0x0e00 << 16) | (0xc178 >> 2),
 272         0x00000000,
 273         (0x0e00 << 16) | (0xc204 >> 2),
 274         0x00000000,
 275         (0x0e00 << 16) | (0xc2b4 >> 2),
 276         0x00000000,
 277         (0x0e00 << 16) | (0xc2b8 >> 2),
 278         0x00000000,
 279         (0x0e00 << 16) | (0xc2bc >> 2),
 280         0x00000000,
 281         (0x0e00 << 16) | (0xc2c0 >> 2),
 282         0x00000000,
 283         (0x0e00 << 16) | (0x8228 >> 2),
 284         0x00000000,
 285         (0x0e00 << 16) | (0x829c >> 2),
 286         0x00000000,
 287         (0x0e00 << 16) | (0x869c >> 2),
 288         0x00000000,
 289         (0x0600 << 16) | (0x98f4 >> 2),
 290         0x00000000,
 291         (0x0e00 << 16) | (0x98f8 >> 2),
 292         0x00000000,
 293         (0x0e00 << 16) | (0x9900 >> 2),
 294         0x00000000,
 295         (0x0e00 << 16) | (0xc260 >> 2),
 296         0x00000000,
 297         (0x0e00 << 16) | (0x90e8 >> 2),
 298         0x00000000,
 299         (0x0e00 << 16) | (0x3c000 >> 2),
 300         0x00000000,
 301         (0x0e00 << 16) | (0x3c00c >> 2),
 302         0x00000000,
 303         (0x0e00 << 16) | (0x8c1c >> 2),
 304         0x00000000,
 305         (0x0e00 << 16) | (0x9700 >> 2),
 306         0x00000000,
 307         (0x0e00 << 16) | (0xcd20 >> 2),
 308         0x00000000,
 309         (0x4e00 << 16) | (0xcd20 >> 2),
 310         0x00000000,
 311         (0x5e00 << 16) | (0xcd20 >> 2),
 312         0x00000000,
 313         (0x6e00 << 16) | (0xcd20 >> 2),
 314         0x00000000,
 315         (0x7e00 << 16) | (0xcd20 >> 2),
 316         0x00000000,
 317         (0x8e00 << 16) | (0xcd20 >> 2),
 318         0x00000000,
 319         (0x9e00 << 16) | (0xcd20 >> 2),
 320         0x00000000,
 321         (0xae00 << 16) | (0xcd20 >> 2),
 322         0x00000000,
 323         (0xbe00 << 16) | (0xcd20 >> 2),
 324         0x00000000,
 325         (0x0e00 << 16) | (0x89bc >> 2),
 326         0x00000000,
 327         (0x0e00 << 16) | (0x8900 >> 2),
 328         0x00000000,
 329         0x3,
 330         (0x0e00 << 16) | (0xc130 >> 2),
 331         0x00000000,
 332         (0x0e00 << 16) | (0xc134 >> 2),
 333         0x00000000,
 334         (0x0e00 << 16) | (0xc1fc >> 2),
 335         0x00000000,
 336         (0x0e00 << 16) | (0xc208 >> 2),
 337         0x00000000,
 338         (0x0e00 << 16) | (0xc264 >> 2),
 339         0x00000000,
 340         (0x0e00 << 16) | (0xc268 >> 2),
 341         0x00000000,
 342         (0x0e00 << 16) | (0xc26c >> 2),
 343         0x00000000,
 344         (0x0e00 << 16) | (0xc270 >> 2),
 345         0x00000000,
 346         (0x0e00 << 16) | (0xc274 >> 2),
 347         0x00000000,
 348         (0x0e00 << 16) | (0xc278 >> 2),
 349         0x00000000,
 350         (0x0e00 << 16) | (0xc27c >> 2),
 351         0x00000000,
 352         (0x0e00 << 16) | (0xc280 >> 2),
 353         0x00000000,
 354         (0x0e00 << 16) | (0xc284 >> 2),
 355         0x00000000,
 356         (0x0e00 << 16) | (0xc288 >> 2),
 357         0x00000000,
 358         (0x0e00 << 16) | (0xc28c >> 2),
 359         0x00000000,
 360         (0x0e00 << 16) | (0xc290 >> 2),
 361         0x00000000,
 362         (0x0e00 << 16) | (0xc294 >> 2),
 363         0x00000000,
 364         (0x0e00 << 16) | (0xc298 >> 2),
 365         0x00000000,
 366         (0x0e00 << 16) | (0xc29c >> 2),
 367         0x00000000,
 368         (0x0e00 << 16) | (0xc2a0 >> 2),
 369         0x00000000,
 370         (0x0e00 << 16) | (0xc2a4 >> 2),
 371         0x00000000,
 372         (0x0e00 << 16) | (0xc2a8 >> 2),
 373         0x00000000,
 374         (0x0e00 << 16) | (0xc2ac  >> 2),
 375         0x00000000,
 376         (0x0e00 << 16) | (0xc2b0 >> 2),
 377         0x00000000,
 378         (0x0e00 << 16) | (0x301d0 >> 2),
 379         0x00000000,
 380         (0x0e00 << 16) | (0x30238 >> 2),
 381         0x00000000,
 382         (0x0e00 << 16) | (0x30250 >> 2),
 383         0x00000000,
 384         (0x0e00 << 16) | (0x30254 >> 2),
 385         0x00000000,
 386         (0x0e00 << 16) | (0x30258 >> 2),
 387         0x00000000,
 388         (0x0e00 << 16) | (0x3025c >> 2),
 389         0x00000000,
 390         (0x4e00 << 16) | (0xc900 >> 2),
 391         0x00000000,
 392         (0x5e00 << 16) | (0xc900 >> 2),
 393         0x00000000,
 394         (0x6e00 << 16) | (0xc900 >> 2),
 395         0x00000000,
 396         (0x7e00 << 16) | (0xc900 >> 2),
 397         0x00000000,
 398         (0x8e00 << 16) | (0xc900 >> 2),
 399         0x00000000,
 400         (0x9e00 << 16) | (0xc900 >> 2),
 401         0x00000000,
 402         (0xae00 << 16) | (0xc900 >> 2),
 403         0x00000000,
 404         (0xbe00 << 16) | (0xc900 >> 2),
 405         0x00000000,
 406         (0x4e00 << 16) | (0xc904 >> 2),
 407         0x00000000,
 408         (0x5e00 << 16) | (0xc904 >> 2),
 409         0x00000000,
 410         (0x6e00 << 16) | (0xc904 >> 2),
 411         0x00000000,
 412         (0x7e00 << 16) | (0xc904 >> 2),
 413         0x00000000,
 414         (0x8e00 << 16) | (0xc904 >> 2),
 415         0x00000000,
 416         (0x9e00 << 16) | (0xc904 >> 2),
 417         0x00000000,
 418         (0xae00 << 16) | (0xc904 >> 2),
 419         0x00000000,
 420         (0xbe00 << 16) | (0xc904 >> 2),
 421         0x00000000,
 422         (0x4e00 << 16) | (0xc908 >> 2),
 423         0x00000000,
 424         (0x5e00 << 16) | (0xc908 >> 2),
 425         0x00000000,
 426         (0x6e00 << 16) | (0xc908 >> 2),
 427         0x00000000,
 428         (0x7e00 << 16) | (0xc908 >> 2),
 429         0x00000000,
 430         (0x8e00 << 16) | (0xc908 >> 2),
 431         0x00000000,
 432         (0x9e00 << 16) | (0xc908 >> 2),
 433         0x00000000,
 434         (0xae00 << 16) | (0xc908 >> 2),
 435         0x00000000,
 436         (0xbe00 << 16) | (0xc908 >> 2),
 437         0x00000000,
 438         (0x4e00 << 16) | (0xc90c >> 2),
 439         0x00000000,
 440         (0x5e00 << 16) | (0xc90c >> 2),
 441         0x00000000,
 442         (0x6e00 << 16) | (0xc90c >> 2),
 443         0x00000000,
 444         (0x7e00 << 16) | (0xc90c >> 2),
 445         0x00000000,
 446         (0x8e00 << 16) | (0xc90c >> 2),
 447         0x00000000,
 448         (0x9e00 << 16) | (0xc90c >> 2),
 449         0x00000000,
 450         (0xae00 << 16) | (0xc90c >> 2),
 451         0x00000000,
 452         (0xbe00 << 16) | (0xc90c >> 2),
 453         0x00000000,
 454         (0x4e00 << 16) | (0xc910 >> 2),
 455         0x00000000,
 456         (0x5e00 << 16) | (0xc910 >> 2),
 457         0x00000000,
 458         (0x6e00 << 16) | (0xc910 >> 2),
 459         0x00000000,
 460         (0x7e00 << 16) | (0xc910 >> 2),
 461         0x00000000,
 462         (0x8e00 << 16) | (0xc910 >> 2),
 463         0x00000000,
 464         (0x9e00 << 16) | (0xc910 >> 2),
 465         0x00000000,
 466         (0xae00 << 16) | (0xc910 >> 2),
 467         0x00000000,
 468         (0xbe00 << 16) | (0xc910 >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0xc99c >> 2),
 471         0x00000000,
 472         (0x0e00 << 16) | (0x9834 >> 2),
 473         0x00000000,
 474         (0x0000 << 16) | (0x30f00 >> 2),
 475         0x00000000,
 476         (0x0001 << 16) | (0x30f00 >> 2),
 477         0x00000000,
 478         (0x0000 << 16) | (0x30f04 >> 2),
 479         0x00000000,
 480         (0x0001 << 16) | (0x30f04 >> 2),
 481         0x00000000,
 482         (0x0000 << 16) | (0x30f08 >> 2),
 483         0x00000000,
 484         (0x0001 << 16) | (0x30f08 >> 2),
 485         0x00000000,
 486         (0x0000 << 16) | (0x30f0c >> 2),
 487         0x00000000,
 488         (0x0001 << 16) | (0x30f0c >> 2),
 489         0x00000000,
 490         (0x0600 << 16) | (0x9b7c >> 2),
 491         0x00000000,
 492         (0x0e00 << 16) | (0x8a14 >> 2),
 493         0x00000000,
 494         (0x0e00 << 16) | (0x8a18 >> 2),
 495         0x00000000,
 496         (0x0600 << 16) | (0x30a00 >> 2),
 497         0x00000000,
 498         (0x0e00 << 16) | (0x8bf0 >> 2),
 499         0x00000000,
 500         (0x0e00 << 16) | (0x8bcc >> 2),
 501         0x00000000,
 502         (0x0e00 << 16) | (0x8b24 >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0x30a04 >> 2),
 505         0x00000000,
 506         (0x0600 << 16) | (0x30a10 >> 2),
 507         0x00000000,
 508         (0x0600 << 16) | (0x30a14 >> 2),
 509         0x00000000,
 510         (0x0600 << 16) | (0x30a18 >> 2),
 511         0x00000000,
 512         (0x0600 << 16) | (0x30a2c >> 2),
 513         0x00000000,
 514         (0x0e00 << 16) | (0xc700 >> 2),
 515         0x00000000,
 516         (0x0e00 << 16) | (0xc704 >> 2),
 517         0x00000000,
 518         (0x0e00 << 16) | (0xc708 >> 2),
 519         0x00000000,
 520         (0x0e00 << 16) | (0xc768 >> 2),
 521         0x00000000,
 522         (0x0400 << 16) | (0xc770 >> 2),
 523         0x00000000,
 524         (0x0400 << 16) | (0xc774 >> 2),
 525         0x00000000,
 526         (0x0400 << 16) | (0xc778 >> 2),
 527         0x00000000,
 528         (0x0400 << 16) | (0xc77c >> 2),
 529         0x00000000,
 530         (0x0400 << 16) | (0xc780 >> 2),
 531         0x00000000,
 532         (0x0400 << 16) | (0xc784 >> 2),
 533         0x00000000,
 534         (0x0400 << 16) | (0xc788 >> 2),
 535         0x00000000,
 536         (0x0400 << 16) | (0xc78c >> 2),
 537         0x00000000,
 538         (0x0400 << 16) | (0xc798 >> 2),
 539         0x00000000,
 540         (0x0400 << 16) | (0xc79c >> 2),
 541         0x00000000,
 542         (0x0400 << 16) | (0xc7a0 >> 2),
 543         0x00000000,
 544         (0x0400 << 16) | (0xc7a4 >> 2),
 545         0x00000000,
 546         (0x0400 << 16) | (0xc7a8 >> 2),
 547         0x00000000,
 548         (0x0400 << 16) | (0xc7ac >> 2),
 549         0x00000000,
 550         (0x0400 << 16) | (0xc7b0 >> 2),
 551         0x00000000,
 552         (0x0400 << 16) | (0xc7b4 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0x9100 >> 2),
 555         0x00000000,
 556         (0x0e00 << 16) | (0x3c010 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0x92a8 >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0x92ac >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0x92b4 >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0x92b8 >> 2),
 565         0x00000000,
 566         (0x0e00 << 16) | (0x92bc >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0x92c0 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0x92c4 >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x92c8 >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x92cc >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x92d0 >> 2),
 577         0x00000000,
 578         (0x0e00 << 16) | (0x8c00 >> 2),
 579         0x00000000,
 580         (0x0e00 << 16) | (0x8c04 >> 2),
 581         0x00000000,
 582         (0x0e00 << 16) | (0x8c20 >> 2),
 583         0x00000000,
 584         (0x0e00 << 16) | (0x8c38 >> 2),
 585         0x00000000,
 586         (0x0e00 << 16) | (0x8c3c >> 2),
 587         0x00000000,
 588         (0x0e00 << 16) | (0xae00 >> 2),
 589         0x00000000,
 590         (0x0e00 << 16) | (0x9604 >> 2),
 591         0x00000000,
 592         (0x0e00 << 16) | (0xac08 >> 2),
 593         0x00000000,
 594         (0x0e00 << 16) | (0xac0c >> 2),
 595         0x00000000,
 596         (0x0e00 << 16) | (0xac10 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0xac14 >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0xac58 >> 2),
 601         0x00000000,
 602         (0x0e00 << 16) | (0xac68 >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0xac6c >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0xac70 >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0xac74 >> 2),
 609         0x00000000,
 610         (0x0e00 << 16) | (0xac78 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0xac7c >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0xac80 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0xac84 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0xac88 >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0xac8c >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0x970c >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0x9714 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0x9718 >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0x971c >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0x31068 >> 2),
 631         0x00000000,
 632         (0x4e00 << 16) | (0x31068 >> 2),
 633         0x00000000,
 634         (0x5e00 << 16) | (0x31068 >> 2),
 635         0x00000000,
 636         (0x6e00 << 16) | (0x31068 >> 2),
 637         0x00000000,
 638         (0x7e00 << 16) | (0x31068 >> 2),
 639         0x00000000,
 640         (0x8e00 << 16) | (0x31068 >> 2),
 641         0x00000000,
 642         (0x9e00 << 16) | (0x31068 >> 2),
 643         0x00000000,
 644         (0xae00 << 16) | (0x31068 >> 2),
 645         0x00000000,
 646         (0xbe00 << 16) | (0x31068 >> 2),
 647         0x00000000,
 648         (0x0e00 << 16) | (0xcd10 >> 2),
 649         0x00000000,
 650         (0x0e00 << 16) | (0xcd14 >> 2),
 651         0x00000000,
 652         (0x0e00 << 16) | (0x88b0 >> 2),
 653         0x00000000,
 654         (0x0e00 << 16) | (0x88b4 >> 2),
 655         0x00000000,
 656         (0x0e00 << 16) | (0x88b8 >> 2),
 657         0x00000000,
 658         (0x0e00 << 16) | (0x88bc >> 2),
 659         0x00000000,
 660         (0x0400 << 16) | (0x89c0 >> 2),
 661         0x00000000,
 662         (0x0e00 << 16) | (0x88c4 >> 2),
 663         0x00000000,
 664         (0x0e00 << 16) | (0x88c8 >> 2),
 665         0x00000000,
 666         (0x0e00 << 16) | (0x88d0 >> 2),
 667         0x00000000,
 668         (0x0e00 << 16) | (0x88d4 >> 2),
 669         0x00000000,
 670         (0x0e00 << 16) | (0x88d8 >> 2),
 671         0x00000000,
 672         (0x0e00 << 16) | (0x8980 >> 2),
 673         0x00000000,
 674         (0x0e00 << 16) | (0x30938 >> 2),
 675         0x00000000,
 676         (0x0e00 << 16) | (0x3093c >> 2),
 677         0x00000000,
 678         (0x0e00 << 16) | (0x30940 >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0x89a0 >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0x30900 >> 2),
 683         0x00000000,
 684         (0x0e00 << 16) | (0x30904 >> 2),
 685         0x00000000,
 686         (0x0e00 << 16) | (0x89b4 >> 2),
 687         0x00000000,
 688         (0x0e00 << 16) | (0x3c210 >> 2),
 689         0x00000000,
 690         (0x0e00 << 16) | (0x3c214 >> 2),
 691         0x00000000,
 692         (0x0e00 << 16) | (0x3c218 >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0x8904 >> 2),
 695         0x00000000,
 696         0x5,
 697         (0x0e00 << 16) | (0x8c28 >> 2),
 698         (0x0e00 << 16) | (0x8c2c >> 2),
 699         (0x0e00 << 16) | (0x8c30 >> 2),
 700         (0x0e00 << 16) | (0x8c34 >> 2),
 701         (0x0e00 << 16) | (0x9600 >> 2),
 702 };
 703
 704 static const u32 kalindi_rlc_save_restore_register_list[] =
 705 {
 706         (0x0e00 << 16) | (0xc12c >> 2),
 707         0x00000000,
 708         (0x0e00 << 16) | (0xc140 >> 2),
 709         0x00000000,
 710         (0x0e00 << 16) | (0xc150 >> 2),
 711         0x00000000,
 712         (0x0e00 << 16) | (0xc15c >> 2),
 713         0x00000000,
 714         (0x0e00 << 16) | (0xc168 >> 2),
 715         0x00000000,
 716         (0x0e00 << 16) | (0xc170 >> 2),
 717         0x00000000,
 718         (0x0e00 << 16) | (0xc204 >> 2),
 719         0x00000000,
 720         (0x0e00 << 16) | (0xc2b4 >> 2),
 721         0x00000000,
 722         (0x0e00 << 16) | (0xc2b8 >> 2),
 723         0x00000000,
 724         (0x0e00 << 16) | (0xc2bc >> 2),
 725         0x00000000,
 726         (0x0e00 << 16) | (0xc2c0 >> 2),
 727         0x00000000,
 728         (0x0e00 << 16) | (0x8228 >> 2),
 729         0x00000000,
 730         (0x0e00 << 16) | (0x829c >> 2),
 731         0x00000000,
 732         (0x0e00 << 16) | (0x869c >> 2),
 733         0x00000000,
 734         (0x0600 << 16) | (0x98f4 >> 2),
 735         0x00000000,
 736         (0x0e00 << 16) | (0x98f8 >> 2),
 737         0x00000000,
 738         (0x0e00 << 16) | (0x9900 >> 2),
 739         0x00000000,
 740         (0x0e00 << 16) | (0xc260 >> 2),
 741         0x00000000,
 742         (0x0e00 << 16) | (0x90e8 >> 2),
 743         0x00000000,
 744         (0x0e00 << 16) | (0x3c000 >> 2),
 745         0x00000000,
 746         (0x0e00 << 16) | (0x3c00c >> 2),
 747         0x00000000,
 748         (0x0e00 << 16) | (0x8c1c >> 2),
 749         0x00000000,
 750         (0x0e00 << 16) | (0x9700 >> 2),
 751         0x00000000,
 752         (0x0e00 << 16) | (0xcd20 >> 2),
 753         0x00000000,
 754         (0x4e00 << 16) | (0xcd20 >> 2),
 755         0x00000000,
 756         (0x5e00 << 16) | (0xcd20 >> 2),
 757         0x00000000,
 758         (0x6e00 << 16) | (0xcd20 >> 2),
 759         0x00000000,
 760         (0x7e00 << 16) | (0xcd20 >> 2),
 761         0x00000000,
 762         (0x0e00 << 16) | (0x89bc >> 2),
 763         0x00000000,
 764         (0x0e00 << 16) | (0x8900 >> 2),
 765         0x00000000,
 766         0x3,
 767         (0x0e00 << 16) | (0xc130 >> 2),
 768         0x00000000,
 769         (0x0e00 << 16) | (0xc134 >> 2),
 770         0x00000000,
 771         (0x0e00 << 16) | (0xc1fc >> 2),
 772         0x00000000,
 773         (0x0e00 << 16) | (0xc208 >> 2),
 774         0x00000000,
 775         (0x0e00 << 16) | (0xc264 >> 2),
 776         0x00000000,
 777         (0x0e00 << 16) | (0xc268 >> 2),
 778         0x00000000,
 779         (0x0e00 << 16) | (0xc26c >> 2),
 780         0x00000000,
 781         (0x0e00 << 16) | (0xc270 >> 2),
 782         0x00000000,
 783         (0x0e00 << 16) | (0xc274 >> 2),
 784         0x00000000,
 785         (0x0e00 << 16) | (0xc28c >> 2),
 786         0x00000000,
 787         (0x0e00 << 16) | (0xc290 >> 2),
 788         0x00000000,
 789         (0x0e00 << 16) | (0xc294 >> 2),
 790         0x00000000,
 791         (0x0e00 << 16) | (0xc298 >> 2),
 792         0x00000000,
 793         (0x0e00 << 16) | (0xc2a0 >> 2),
 794         0x00000000,
 795         (0x0e00 << 16) | (0xc2a4 >> 2),
 796         0x00000000,
 797         (0x0e00 << 16) | (0xc2a8 >> 2),
 798         0x00000000,
 799         (0x0e00 << 16) | (0xc2ac >> 2),
 800         0x00000000,
 801         (0x0e00 << 16) | (0x301d0 >> 2),
 802         0x00000000,
 803         (0x0e00 << 16) | (0x30238 >> 2),
 804         0x00000000,
 805         (0x0e00 << 16) | (0x30250 >> 2),
 806         0x00000000,
 807         (0x0e00 << 16) | (0x30254 >> 2),
 808         0x00000000,
 809         (0x0e00 << 16) | (0x30258 >> 2),
 810         0x00000000,
 811         (0x0e00 << 16) | (0x3025c >> 2),
 812         0x00000000,
 813         (0x4e00 << 16) | (0xc900 >> 2),
 814         0x00000000,
 815         (0x5e00 << 16) | (0xc900 >> 2),
 816         0x00000000,
 817         (0x6e00 << 16) | (0xc900 >> 2),
 818         0x00000000,
 819         (0x7e00 << 16) | (0xc900 >> 2),
 820         0x00000000,
 821         (0x4e00 << 16) | (0xc904 >> 2),
 822         0x00000000,
 823         (0x5e00 << 16) | (0xc904 >> 2),
 824         0x00000000,
 825         (0x6e00 << 16) | (0xc904 >> 2),
 826         0x00000000,
 827         (0x7e00 << 16) | (0xc904 >> 2),
 828         0x00000000,
 829         (0x4e00 << 16) | (0xc908 >> 2),
 830         0x00000000,
 831         (0x5e00 << 16) | (0xc908 >> 2),
 832         0x00000000,
 833         (0x6e00 << 16) | (0xc908 >> 2),
 834         0x00000000,
 835         (0x7e00 << 16) | (0xc908 >> 2),
 836         0x00000000,
 837         (0x4e00 << 16) | (0xc90c >> 2),
 838         0x00000000,
 839         (0x5e00 << 16) | (0xc90c >> 2),
 840         0x00000000,
 841         (0x6e00 << 16) | (0xc90c >> 2),
 842         0x00000000,
 843         (0x7e00 << 16) | (0xc90c >> 2),
 844         0x00000000,
 845         (0x4e00 << 16) | (0xc910 >> 2),
 846         0x00000000,
 847         (0x5e00 << 16) | (0xc910 >> 2),
 848         0x00000000,
 849         (0x6e00 << 16) | (0xc910 >> 2),
 850         0x00000000,
 851         (0x7e00 << 16) | (0xc910 >> 2),
 852         0x00000000,
 853         (0x0e00 << 16) | (0xc99c >> 2),
 854         0x00000000,
 855         (0x0e00 << 16) | (0x9834 >> 2),
 856         0x00000000,
 857         (0x0000 << 16) | (0x30f00 >> 2),
 858         0x00000000,
 859         (0x0000 << 16) | (0x30f04 >> 2),
 860         0x00000000,
 861         (0x0000 << 16) | (0x30f08 >> 2),
 862         0x00000000,
 863         (0x0000 << 16) | (0x30f0c >> 2),
 864         0x00000000,
 865         (0x0600 << 16) | (0x9b7c >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0x8a14 >> 2),
 868         0x00000000,
 869         (0x0e00 << 16) | (0x8a18 >> 2),
 870         0x00000000,
 871         (0x0600 << 16) | (0x30a00 >> 2),
 872         0x00000000,
 873         (0x0e00 << 16) | (0x8bf0 >> 2),
 874         0x00000000,
 875         (0x0e00 << 16) | (0x8bcc >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0x8b24 >> 2),
 878         0x00000000,
 879         (0x0e00 << 16) | (0x30a04 >> 2),
 880         0x00000000,
 881         (0x0600 << 16) | (0x30a10 >> 2),
 882         0x00000000,
 883         (0x0600 << 16) | (0x30a14 >> 2),
 884         0x00000000,
 885         (0x0600 << 16) | (0x30a18 >> 2),
 886         0x00000000,
 887         (0x0600 << 16) | (0x30a2c >> 2),
 888         0x00000000,
 889         (0x0e00 << 16) | (0xc700 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0xc704 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0xc708 >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0xc768 >> 2),
 896         0x00000000,
 897         (0x0400 << 16) | (0xc770 >> 2),
 898         0x00000000,
 899         (0x0400 << 16) | (0xc774 >> 2),
 900         0x00000000,
 901         (0x0400 << 16) | (0xc798 >> 2),
 902         0x00000000,
 903         (0x0400 << 16) | (0xc79c >> 2),
 904         0x00000000,
 905         (0x0e00 << 16) | (0x9100 >> 2),
 906         0x00000000,
 907         (0x0e00 << 16) | (0x3c010 >> 2),
 908         0x00000000,
 909         (0x0e00 << 16) | (0x8c00 >> 2),
 910         0x00000000,
 911         (0x0e00 << 16) | (0x8c04 >> 2),
 912         0x00000000,
 913         (0x0e00 << 16) | (0x8c20 >> 2),
 914         0x00000000,
 915         (0x0e00 << 16) | (0x8c38 >> 2),
 916         0x00000000,
 917         (0x0e00 << 16) | (0x8c3c >> 2),
 918         0x00000000,
 919         (0x0e00 << 16) | (0xae00 >> 2),
 920         0x00000000,
 921         (0x0e00 << 16) | (0x9604 >> 2),
 922         0x00000000,
 923         (0x0e00 << 16) | (0xac08 >> 2),
 924         0x00000000,
 925         (0x0e00 << 16) | (0xac0c >> 2),
 926         0x00000000,
 927         (0x0e00 << 16) | (0xac10 >> 2),
 928         0x00000000,
 929         (0x0e00 << 16) | (0xac14 >> 2),
 930         0x00000000,
 931         (0x0e00 << 16) | (0xac58 >> 2),
 932         0x00000000,
 933         (0x0e00 << 16) | (0xac68 >> 2),
 934         0x00000000,
 935         (0x0e00 << 16) | (0xac6c >> 2),
 936         0x00000000,
 937         (0x0e00 << 16) | (0xac70 >> 2),
 938         0x00000000,
 939         (0x0e00 << 16) | (0xac74 >> 2),
 940         0x00000000,
 941         (0x0e00 << 16) | (0xac78 >> 2),
 942         0x00000000,
 943         (0x0e00 << 16) | (0xac7c >> 2),
 944         0x00000000,
 945         (0x0e00 << 16) | (0xac80 >> 2),
 946         0x00000000,
 947         (0x0e00 << 16) | (0xac84 >> 2),
 948         0x00000000,
 949         (0x0e00 << 16) | (0xac88 >> 2),
 950         0x00000000,
 951         (0x0e00 << 16) | (0xac8c >> 2),
 952         0x00000000,
 953         (0x0e00 << 16) | (0x970c >> 2),
 954         0x00000000,
 955         (0x0e00 << 16) | (0x9714 >> 2),
 956         0x00000000,
 957         (0x0e00 << 16) | (0x9718 >> 2),
 958         0x00000000,
 959         (0x0e00 << 16) | (0x971c >> 2),
 960         0x00000000,
 961         (0x0e00 << 16) | (0x31068 >> 2),
 962         0x00000000,
 963         (0x4e00 << 16) | (0x31068 >> 2),
 964         0x00000000,
 965         (0x5e00 << 16) | (0x31068 >> 2),
 966         0x00000000,
 967         (0x6e00 << 16) | (0x31068 >> 2),
 968         0x00000000,
 969         (0x7e00 << 16) | (0x31068 >> 2),
 970         0x00000000,
 971         (0x0e00 << 16) | (0xcd10 >> 2),
 972         0x00000000,
 973         (0x0e00 << 16) | (0xcd14 >> 2),
 974         0x00000000,
 975         (0x0e00 << 16) | (0x88b0 >> 2),
 976         0x00000000,
 977         (0x0e00 << 16) | (0x88b4 >> 2),
 978         0x00000000,
 979         (0x0e00 << 16) | (0x88b8 >> 2),
 980         0x00000000,
 981         (0x0e00 << 16) | (0x88bc >> 2),
 982         0x00000000,
 983         (0x0400 << 16) | (0x89c0 >> 2),
 984         0x00000000,
 985         (0x0e00 << 16) | (0x88c4 >> 2),
 986         0x00000000,
 987         (0x0e00 << 16) | (0x88c8 >> 2),
 988         0x00000000,
 989         (0x0e00 << 16) | (0x88d0 >> 2),
 990         0x00000000,
 991         (0x0e00 << 16) | (0x88d4 >> 2),
 992         0x00000000,
 993         (0x0e00 << 16) | (0x88d8 >> 2),
 994         0x00000000,
 995         (0x0e00 << 16) | (0x8980 >> 2),
 996         0x00000000,
 997         (0x0e00 << 16) | (0x30938 >> 2),
 998         0x00000000,
 999         (0x0e00 << 16) | (0x3093c >> 2),
1000         0x00000000,
1001         (0x0e00 << 16) | (0x30940 >> 2),
1002         0x00000000,
1003         (0x0e00 << 16) | (0x89a0 >> 2),
1004         0x00000000,
1005         (0x0e00 << 16) | (0x30900 >> 2),
1006         0x00000000,
1007         (0x0e00 << 16) | (0x30904 >> 2),
1008         0x00000000,
1009         (0x0e00 << 16) | (0x89b4 >> 2),
1010         0x00000000,
1011         (0x0e00 << 16) | (0x3e1fc >> 2),
1012         0x00000000,
1013         (0x0e00 << 16) | (0x3c210 >> 2),
1014         0x00000000,
1015         (0x0e00 << 16) | (0x3c214 >> 2),
1016         0x00000000,
1017         (0x0e00 << 16) | (0x3c218 >> 2),
1018         0x00000000,
1019         (0x0e00 << 16) | (0x8904 >> 2),
1020         0x00000000,
1021         0x5,
1022         (0x0e00 << 16) | (0x8c28 >> 2),
1023         (0x0e00 << 16) | (0x8c2c >> 2),
1024         (0x0e00 << 16) | (0x8c30 >> 2),
1025         (0x0e00 << 16) | (0x8c34 >> 2),
1026         (0x0e00 << 16) | (0x9600 >> 2),
1027 };
1028
1029 static const u32 bonaire_golden_spm_registers[] =
1030 {
1031         0x30800, 0xe0ffffff, 0xe0000000
1032 };
1033
1034 static const u32 bonaire_golden_common_registers[] =
1035 {
1036         0xc770, 0xffffffff, 0x00000800,
1037         0xc774, 0xffffffff, 0x00000800,
1038         0xc798, 0xffffffff, 0x00007fbf,
1039         0xc79c, 0xffffffff, 0x00007faf
1040 };
1041
1042 static const u32 bonaire_golden_registers[] =
1043 {
1044         0x3354, 0x00000333, 0x00000333,
1045         0x3350, 0x000c0fc0, 0x00040200,
1046         0x9a10, 0x00010000, 0x00058208,
1047         0x3c000, 0xffff1fff, 0x00140000,
1048         0x3c200, 0xfdfc0fff, 0x00000100,
1049         0x3c234, 0x40000000, 0x40000200,
1050         0x9830, 0xffffffff, 0x00000000,
1051         0x9834, 0xf00fffff, 0x00000400,
1052         0x9838, 0x0002021c, 0x00020200,
1053         0xc78, 0x00000080, 0x00000000,
1054         0x5bb0, 0x000000f0, 0x00000070,
1055         0x5bc0, 0xf0311fff, 0x80300000,
1056         0x98f8, 0x73773777, 0x12010001,
1057         0x350c, 0x00810000, 0x408af000,
1058         0x7030, 0x31000111, 0x00000011,
1059         0x2f48, 0x73773777, 0x12010001,
1060         0x220c, 0x00007fb6, 0x0021a1b1,
1061         0x2210, 0x00007fb6, 0x002021b1,
1062         0x2180, 0x00007fb6, 0x00002191,
1063         0x2218, 0x00007fb6, 0x002121b1,
1064         0x221c, 0x00007fb6, 0x002021b1,
1065         0x21dc, 0x00007fb6, 0x00002191,
1066         0x21e0, 0x00007fb6, 0x00002191,
1067         0x3628, 0x0000003f, 0x0000000a,
1068         0x362c, 0x0000003f, 0x0000000a,
1069         0x2ae4, 0x00073ffe, 0x000022a2,
1070         0x240c, 0x000007ff, 0x00000000,
1071         0x8a14, 0xf000003f, 0x00000007,
1072         0x8bf0, 0x00002001, 0x00000001,
1073         0x8b24, 0xffffffff, 0x00ffffff,
1074         0x30a04, 0x0000ff0f, 0x00000000,
1075         0x28a4c, 0x07ffffff, 0x06000000,
1076         0x4d8, 0x00000fff, 0x00000100,
1077         0x3e78, 0x00000001, 0x00000002,
1078         0x9100, 0x03000000, 0x0362c688,
1079         0x8c00, 0x000000ff, 0x00000001,
1080         0xe40, 0x00001fff, 0x00001fff,
1081         0x9060, 0x0000007f, 0x00000020,
1082         0x9508, 0x00010000, 0x00010000,
1083         0xac14, 0x000003ff, 0x000000f3,
1084         0xac0c, 0xffffffff, 0x00001032
1085 };
1086
1087 static const u32 bonaire_mgcg_cgcg_init[] =
1088 {
1089         0xc420, 0xffffffff, 0xfffffffc,
1090         0x30800, 0xffffffff, 0xe0000000,
1091         0x3c2a0, 0xffffffff, 0x00000100,
1092         0x3c208, 0xffffffff, 0x00000100,
1093         0x3c2c0, 0xffffffff, 0xc0000100,
1094         0x3c2c8, 0xffffffff, 0xc0000100,
1095         0x3c2c4, 0xffffffff, 0xc0000100,
1096         0x55e4, 0xffffffff, 0x00600100,
1097         0x3c280, 0xffffffff, 0x00000100,
1098         0x3c214, 0xffffffff, 0x06000100,
1099         0x3c220, 0xffffffff, 0x00000100,
1100         0x3c218, 0xffffffff, 0x06000100,
1101         0x3c204, 0xffffffff, 0x00000100,
1102         0x3c2e0, 0xffffffff, 0x00000100,
1103         0x3c224, 0xffffffff, 0x00000100,
1104         0x3c200, 0xffffffff, 0x00000100,
1105         0x3c230, 0xffffffff, 0x00000100,
1106         0x3c234, 0xffffffff, 0x00000100,
1107         0x3c250, 0xffffffff, 0x00000100,
1108         0x3c254, 0xffffffff, 0x00000100,
1109         0x3c258, 0xffffffff, 0x00000100,
1110         0x3c25c, 0xffffffff, 0x00000100,
1111         0x3c260, 0xffffffff, 0x00000100,
1112         0x3c27c, 0xffffffff, 0x00000100,
1113         0x3c278, 0xffffffff, 0x00000100,
1114         0x3c210, 0xffffffff, 0x06000100,
1115         0x3c290, 0xffffffff, 0x00000100,
1116         0x3c274, 0xffffffff, 0x00000100,
1117         0x3c2b4, 0xffffffff, 0x00000100,
1118         0x3c2b0, 0xffffffff, 0x00000100,
1119         0x3c270, 0xffffffff, 0x00000100,
1120         0x30800, 0xffffffff, 0xe0000000,
1121         0x3c020, 0xffffffff, 0x00010000,
1122         0x3c024, 0xffffffff, 0x00030002,
1123         0x3c028, 0xffffffff, 0x00040007,
1124         0x3c02c, 0xffffffff, 0x00060005,
1125         0x3c030, 0xffffffff, 0x00090008,
1126         0x3c034, 0xffffffff, 0x00010000,
1127         0x3c038, 0xffffffff, 0x00030002,
1128         0x3c03c, 0xffffffff, 0x00040007,
1129         0x3c040, 0xffffffff, 0x00060005,
1130         0x3c044, 0xffffffff, 0x00090008,
1131         0x3c048, 0xffffffff, 0x00010000,
1132         0x3c04c, 0xffffffff, 0x00030002,
1133         0x3c050, 0xffffffff, 0x00040007,
1134         0x3c054, 0xffffffff, 0x00060005,
1135         0x3c058, 0xffffffff, 0x00090008,
1136         0x3c05c, 0xffffffff, 0x00010000,
1137         0x3c060, 0xffffffff, 0x00030002,
1138         0x3c064, 0xffffffff, 0x00040007,
1139         0x3c068, 0xffffffff, 0x00060005,
1140         0x3c06c, 0xffffffff, 0x00090008,
1141         0x3c070, 0xffffffff, 0x00010000,
1142         0x3c074, 0xffffffff, 0x00030002,
1143         0x3c078, 0xffffffff, 0x00040007,
1144         0x3c07c, 0xffffffff, 0x00060005,
1145         0x3c080, 0xffffffff, 0x00090008,
1146         0x3c084, 0xffffffff, 0x00010000,
1147         0x3c088, 0xffffffff, 0x00030002,
1148         0x3c08c, 0xffffffff, 0x00040007,
1149         0x3c090, 0xffffffff, 0x00060005,
1150         0x3c094, 0xffffffff, 0x00090008,
1151         0x3c098, 0xffffffff, 0x00010000,
1152         0x3c09c, 0xffffffff, 0x00030002,
1153         0x3c0a0, 0xffffffff, 0x00040007,
1154         0x3c0a4, 0xffffffff, 0x00060005,
1155         0x3c0a8, 0xffffffff, 0x00090008,
1156         0x3c000, 0xffffffff, 0x96e00200,
1157         0x8708, 0xffffffff, 0x00900100,
1158         0xc424, 0xffffffff, 0x0020003f,
1159         0x38, 0xffffffff, 0x0140001c,
1160         0x3c, 0x000f0000, 0x000f0000,
1161         0x220, 0xffffffff, 0xC060000C,
1162         0x224, 0xc0000fff, 0x00000100,
1163         0xf90, 0xffffffff, 0x00000100,
1164         0xf98, 0x00000101, 0x00000000,
1165         0x20a8, 0xffffffff, 0x00000104,
1166         0x55e4, 0xff000fff, 0x00000100,
1167         0x30cc, 0xc0000fff, 0x00000104,
1168         0xc1e4, 0x00000001, 0x00000001,
1169         0xd00c, 0xff000ff0, 0x00000100,
1170         0xd80c, 0xff000ff0, 0x00000100
1171 };
1172
1173 static const u32 spectre_golden_spm_registers[] =
1174 {
1175         0x30800, 0xe0ffffff, 0xe0000000
1176 };
1177
1178 static const u32 spectre_golden_common_registers[] =
1179 {
1180         0xc770, 0xffffffff, 0x00000800,
1181         0xc774, 0xffffffff, 0x00000800,
1182         0xc798, 0xffffffff, 0x00007fbf,
1183         0xc79c, 0xffffffff, 0x00007faf
1184 };
1185
1186 static const u32 spectre_golden_registers[] =
1187 {
1188         0x3c000, 0xffff1fff, 0x96940200,
1189         0x3c00c, 0xffff0001, 0xff000000,
1190         0x3c200, 0xfffc0fff, 0x00000100,
1191         0x6ed8, 0x00010101, 0x00010000,
1192         0x9834, 0xf00fffff, 0x00000400,
1193         0x9838, 0xfffffffc, 0x00020200,
1194         0x5bb0, 0x000000f0, 0x00000070,
1195         0x5bc0, 0xf0311fff, 0x80300000,
1196         0x98f8, 0x73773777, 0x12010001,
1197         0x9b7c, 0x00ff0000, 0x00fc0000,
1198         0x2f48, 0x73773777, 0x12010001,
1199         0x8a14, 0xf000003f, 0x00000007,
1200         0x8b24, 0xffffffff, 0x00ffffff,
1201         0x28350, 0x3f3f3fff, 0x00000082,
1202         0x28354, 0x0000003f, 0x00000000,
1203         0x3e78, 0x00000001, 0x00000002,
1204         0x913c, 0xffff03df, 0x00000004,
1205         0xc768, 0x00000008, 0x00000008,
1206         0x8c00, 0x000008ff, 0x00000800,
1207         0x9508, 0x00010000, 0x00010000,
1208         0xac0c, 0xffffffff, 0x54763210,
1209         0x214f8, 0x01ff01ff, 0x00000002,
1210         0x21498, 0x007ff800, 0x00200000,
1211         0x2015c, 0xffffffff, 0x00000f40,
1212         0x30934, 0xffffffff, 0x00000001
1213 };
1214
1215 static const u32 spectre_mgcg_cgcg_init[] =
1216 {
1217         0xc420, 0xffffffff, 0xfffffffc,
1218         0x30800, 0xffffffff, 0xe0000000,
1219         0x3c2a0, 0xffffffff, 0x00000100,
1220         0x3c208, 0xffffffff, 0x00000100,
1221         0x3c2c0, 0xffffffff, 0x00000100,
1222         0x3c2c8, 0xffffffff, 0x00000100,
1223         0x3c2c4, 0xffffffff, 0x00000100,
1224         0x55e4, 0xffffffff, 0x00600100,
1225         0x3c280, 0xffffffff, 0x00000100,
1226         0x3c214, 0xffffffff, 0x06000100,
1227         0x3c220, 0xffffffff, 0x00000100,
1228         0x3c218, 0xffffffff, 0x06000100,
1229         0x3c204, 0xffffffff, 0x00000100,
1230         0x3c2e0, 0xffffffff, 0x00000100,
1231         0x3c224, 0xffffffff, 0x00000100,
1232         0x3c200, 0xffffffff, 0x00000100,
1233         0x3c230, 0xffffffff, 0x00000100,
1234         0x3c234, 0xffffffff, 0x00000100,
1235         0x3c250, 0xffffffff, 0x00000100,
1236         0x3c254, 0xffffffff, 0x00000100,
1237         0x3c258, 0xffffffff, 0x00000100,
1238         0x3c25c, 0xffffffff, 0x00000100,
1239         0x3c260, 0xffffffff, 0x00000100,
1240         0x3c27c, 0xffffffff, 0x00000100,
1241         0x3c278, 0xffffffff, 0x00000100,
1242         0x3c210, 0xffffffff, 0x06000100,
1243         0x3c290, 0xffffffff, 0x00000100,
1244         0x3c274, 0xffffffff, 0x00000100,
1245         0x3c2b4, 0xffffffff, 0x00000100,
1246         0x3c2b0, 0xffffffff, 0x00000100,
1247         0x3c270, 0xffffffff, 0x00000100,
1248         0x30800, 0xffffffff, 0xe0000000,
1249         0x3c020, 0xffffffff, 0x00010000,
1250         0x3c024, 0xffffffff, 0x00030002,
1251         0x3c028, 0xffffffff, 0x00040007,
1252         0x3c02c, 0xffffffff, 0x00060005,
1253         0x3c030, 0xffffffff, 0x00090008,
1254         0x3c034, 0xffffffff, 0x00010000,
1255         0x3c038, 0xffffffff, 0x00030002,
1256         0x3c03c, 0xffffffff, 0x00040007,
1257         0x3c040, 0xffffffff, 0x00060005,
1258         0x3c044, 0xffffffff, 0x00090008,
1259         0x3c048, 0xffffffff, 0x00010000,
1260         0x3c04c, 0xffffffff, 0x00030002,
1261         0x3c050, 0xffffffff, 0x00040007,
1262         0x3c054, 0xffffffff, 0x00060005,
1263         0x3c058, 0xffffffff, 0x00090008,
1264         0x3c05c, 0xffffffff, 0x00010000,
1265         0x3c060, 0xffffffff, 0x00030002,
1266         0x3c064, 0xffffffff, 0x00040007,
1267         0x3c068, 0xffffffff, 0x00060005,
1268         0x3c06c, 0xffffffff, 0x00090008,
1269         0x3c070, 0xffffffff, 0x00010000,
1270         0x3c074, 0xffffffff, 0x00030002,
1271         0x3c078, 0xffffffff, 0x00040007,
1272         0x3c07c, 0xffffffff, 0x00060005,
1273         0x3c080, 0xffffffff, 0x00090008,
1274         0x3c084, 0xffffffff, 0x00010000,
1275         0x3c088, 0xffffffff, 0x00030002,
1276         0x3c08c, 0xffffffff, 0x00040007,
1277         0x3c090, 0xffffffff, 0x00060005,
1278         0x3c094, 0xffffffff, 0x00090008,
1279         0x3c098, 0xffffffff, 0x00010000,
1280         0x3c09c, 0xffffffff, 0x00030002,
1281         0x3c0a0, 0xffffffff, 0x00040007,
1282         0x3c0a4, 0xffffffff, 0x00060005,
1283         0x3c0a8, 0xffffffff, 0x00090008,
1284         0x3c0ac, 0xffffffff, 0x00010000,
1285         0x3c0b0, 0xffffffff, 0x00030002,
1286         0x3c0b4, 0xffffffff, 0x00040007,
1287         0x3c0b8, 0xffffffff, 0x00060005,
1288         0x3c0bc, 0xffffffff, 0x00090008,
1289         0x3c000, 0xffffffff, 0x96e00200,
1290         0x8708, 0xffffffff, 0x00900100,
1291         0xc424, 0xffffffff, 0x0020003f,
1292         0x38, 0xffffffff, 0x0140001c,
1293         0x3c, 0x000f0000, 0x000f0000,
1294         0x220, 0xffffffff, 0xC060000C,
1295         0x224, 0xc0000fff, 0x00000100,
1296         0xf90, 0xffffffff, 0x00000100,
1297         0xf98, 0x00000101, 0x00000000,
1298         0x20a8, 0xffffffff, 0x00000104,
1299         0x55e4, 0xff000fff, 0x00000100,
1300         0x30cc, 0xc0000fff, 0x00000104,
1301         0xc1e4, 0x00000001, 0x00000001,
1302         0xd00c, 0xff000ff0, 0x00000100,
1303         0xd80c, 0xff000ff0, 0x00000100
1304 };
1305
1306 static const u32 kalindi_golden_spm_registers[] =
1307 {
1308         0x30800, 0xe0ffffff, 0xe0000000
1309 };
1310
1311 static const u32 kalindi_golden_common_registers[] =
1312 {
1313         0xc770, 0xffffffff, 0x00000800,
1314         0xc774, 0xffffffff, 0x00000800,
1315         0xc798, 0xffffffff, 0x00007fbf,
1316         0xc79c, 0xffffffff, 0x00007faf
1317 };
1318
1319 static const u32 kalindi_golden_registers[] =
1320 {
1321         0x3c000, 0xffffdfff, 0x6e944040,
1322         0x55e4, 0xff607fff, 0xfc000100,
1323         0x3c220, 0xff000fff, 0x00000100,
1324         0x3c224, 0xff000fff, 0x00000100,
1325         0x3c200, 0xfffc0fff, 0x00000100,
1326         0x6ed8, 0x00010101, 0x00010000,
1327         0x9830, 0xffffffff, 0x00000000,
1328         0x9834, 0xf00fffff, 0x00000400,
1329         0x5bb0, 0x000000f0, 0x00000070,
1330         0x5bc0, 0xf0311fff, 0x80300000,
1331         0x98f8, 0x73773777, 0x12010001,
1332         0x98fc, 0xffffffff, 0x00000010,
1333         0x9b7c, 0x00ff0000, 0x00fc0000,
1334         0x8030, 0x00001f0f, 0x0000100a,
1335         0x2f48, 0x73773777, 0x12010001,
1336         0x2408, 0x000fffff, 0x000c007f,
1337         0x8a14, 0xf000003f, 0x00000007,
1338         0x8b24, 0x3fff3fff, 0x00ffcfff,
1339         0x30a04, 0x0000ff0f, 0x00000000,
1340         0x28a4c, 0x07ffffff, 0x06000000,
1341         0x4d8, 0x00000fff, 0x00000100,
1342         0x3e78, 0x00000001, 0x00000002,
1343         0xc768, 0x00000008, 0x00000008,
1344         0x8c00, 0x000000ff, 0x00000003,
1345         0x214f8, 0x01ff01ff, 0x00000002,
1346         0x21498, 0x007ff800, 0x00200000,
1347         0x2015c, 0xffffffff, 0x00000f40,
1348         0x88c4, 0x001f3ae3, 0x00000082,
1349         0x88d4, 0x0000001f, 0x00000010,
1350         0x30934, 0xffffffff, 0x00000000
1351 };
1352
1353 static const u32 kalindi_mgcg_cgcg_init[] =
1354 {
1355         0xc420, 0xffffffff, 0xfffffffc,
1356         0x30800, 0xffffffff, 0xe0000000,
1357         0x3c2a0, 0xffffffff, 0x00000100,
1358         0x3c208, 0xffffffff, 0x00000100,
1359         0x3c2c0, 0xffffffff, 0x00000100,
1360         0x3c2c8, 0xffffffff, 0x00000100,
1361         0x3c2c4, 0xffffffff, 0x00000100,
1362         0x55e4, 0xffffffff, 0x00600100,
1363         0x3c280, 0xffffffff, 0x00000100,
1364         0x3c214, 0xffffffff, 0x06000100,
1365         0x3c220, 0xffffffff, 0x00000100,
1366         0x3c218, 0xffffffff, 0x06000100,
1367         0x3c204, 0xffffffff, 0x00000100,
1368         0x3c2e0, 0xffffffff, 0x00000100,
1369         0x3c224, 0xffffffff, 0x00000100,
1370         0x3c200, 0xffffffff, 0x00000100,
1371         0x3c230, 0xffffffff, 0x00000100,
1372         0x3c234, 0xffffffff, 0x00000100,
1373         0x3c250, 0xffffffff, 0x00000100,
1374         0x3c254, 0xffffffff, 0x00000100,
1375         0x3c258, 0xffffffff, 0x00000100,
1376         0x3c25c, 0xffffffff, 0x00000100,
1377         0x3c260, 0xffffffff, 0x00000100,
1378         0x3c27c, 0xffffffff, 0x00000100,
1379         0x3c278, 0xffffffff, 0x00000100,
1380         0x3c210, 0xffffffff, 0x06000100,
1381         0x3c290, 0xffffffff, 0x00000100,
1382         0x3c274, 0xffffffff, 0x00000100,
1383         0x3c2b4, 0xffffffff, 0x00000100,
1384         0x3c2b0, 0xffffffff, 0x00000100,
1385         0x3c270, 0xffffffff, 0x00000100,
1386         0x30800, 0xffffffff, 0xe0000000,
1387         0x3c020, 0xffffffff, 0x00010000,
1388         0x3c024, 0xffffffff, 0x00030002,
1389         0x3c028, 0xffffffff, 0x00040007,
1390         0x3c02c, 0xffffffff, 0x00060005,
1391         0x3c030, 0xffffffff, 0x00090008,
1392         0x3c034, 0xffffffff, 0x00010000,
1393         0x3c038, 0xffffffff, 0x00030002,
1394         0x3c03c, 0xffffffff, 0x00040007,
1395         0x3c040, 0xffffffff, 0x00060005,
1396         0x3c044, 0xffffffff, 0x00090008,
1397         0x3c000, 0xffffffff, 0x96e00200,
1398         0x8708, 0xffffffff, 0x00900100,
1399         0xc424, 0xffffffff, 0x0020003f,
1400         0x38, 0xffffffff, 0x0140001c,
1401         0x3c, 0x000f0000, 0x000f0000,
1402         0x220, 0xffffffff, 0xC060000C,
1403         0x224, 0xc0000fff, 0x00000100,
1404         0x20a8, 0xffffffff, 0x00000104,
1405         0x55e4, 0xff000fff, 0x00000100,
1406         0x30cc, 0xc0000fff, 0x00000104,
1407         0xc1e4, 0x00000001, 0x00000001,
1408         0xd00c, 0xff000ff0, 0x00000100,
1409         0xd80c, 0xff000ff0, 0x00000100
1410 };
1411
1412 static const u32 hawaii_golden_spm_registers[] =
1413 {
1414         0x30800, 0xe0ffffff, 0xe0000000
1415 };
1416
1417 static const u32 hawaii_golden_common_registers[] =
1418 {
1419         0x30800, 0xffffffff, 0xe0000000,
1420         0x28350, 0xffffffff, 0x3a00161a,
1421         0x28354, 0xffffffff, 0x0000002e,
1422         0x9a10, 0xffffffff, 0x00018208,
1423         0x98f8, 0xffffffff, 0x12011003
1424 };
1425
1426 static const u32 hawaii_golden_registers[] =
1427 {
1428         0x3354, 0x00000333, 0x00000333,
1429         0x9a10, 0x00010000, 0x00058208,
1430         0x9830, 0xffffffff, 0x00000000,
1431         0x9834, 0xf00fffff, 0x00000400,
1432         0x9838, 0x0002021c, 0x00020200,
1433         0xc78, 0x00000080, 0x00000000,
1434         0x5bb0, 0x000000f0, 0x00000070,
1435         0x5bc0, 0xf0311fff, 0x80300000,
1436         0x350c, 0x00810000, 0x408af000,
1437         0x7030, 0x31000111, 0x00000011,
1438         0x2f48, 0x73773777, 0x12010001,
1439         0x2120, 0x0000007f, 0x0000001b,
1440         0x21dc, 0x00007fb6, 0x00002191,
1441         0x3628, 0x0000003f, 0x0000000a,
1442         0x362c, 0x0000003f, 0x0000000a,
1443         0x2ae4, 0x00073ffe, 0x000022a2,
1444         0x240c, 0x000007ff, 0x00000000,
1445         0x8bf0, 0x00002001, 0x00000001,
1446         0x8b24, 0xffffffff, 0x00ffffff,
1447         0x30a04, 0x0000ff0f, 0x00000000,
1448         0x28a4c, 0x07ffffff, 0x06000000,
1449         0x3e78, 0x00000001, 0x00000002,
1450         0xc768, 0x00000008, 0x00000008,
1451         0xc770, 0x00000f00, 0x00000800,
1452         0xc774, 0x00000f00, 0x00000800,
1453         0xc798, 0x00ffffff, 0x00ff7fbf,
1454         0xc79c, 0x00ffffff, 0x00ff7faf,
1455         0x8c00, 0x000000ff, 0x00000800,
1456         0xe40, 0x00001fff, 0x00001fff,
1457         0x9060, 0x0000007f, 0x00000020,
1458         0x9508, 0x00010000, 0x00010000,
1459         0xae00, 0x00100000, 0x000ff07c,
1460         0xac14, 0x000003ff, 0x0000000f,
1461         0xac10, 0xffffffff, 0x7564fdec,
1462         0xac0c, 0xffffffff, 0x3120b9a8,
1463         0xac08, 0x20000000, 0x0f9c0000
1464 };
1465
1466 static const u32 hawaii_mgcg_cgcg_init[] =
1467 {
1468         0xc420, 0xffffffff, 0xfffffffd,
1469         0x30800, 0xffffffff, 0xe0000000,
1470         0x3c2a0, 0xffffffff, 0x00000100,
1471         0x3c208, 0xffffffff, 0x00000100,
1472         0x3c2c0, 0xffffffff, 0x00000100,
1473         0x3c2c8, 0xffffffff, 0x00000100,
1474         0x3c2c4, 0xffffffff, 0x00000100,
1475         0x55e4, 0xffffffff, 0x00200100,
1476         0x3c280, 0xffffffff, 0x00000100,
1477         0x3c214, 0xffffffff, 0x06000100,
1478         0x3c220, 0xffffffff, 0x00000100,
1479         0x3c218, 0xffffffff, 0x06000100,
1480         0x3c204, 0xffffffff, 0x00000100,
1481         0x3c2e0, 0xffffffff, 0x00000100,
1482         0x3c224, 0xffffffff, 0x00000100,
1483         0x3c200, 0xffffffff, 0x00000100,
1484         0x3c230, 0xffffffff, 0x00000100,
1485         0x3c234, 0xffffffff, 0x00000100,
1486         0x3c250, 0xffffffff, 0x00000100,
1487         0x3c254, 0xffffffff, 0x00000100,
1488         0x3c258, 0xffffffff, 0x00000100,
1489         0x3c25c, 0xffffffff, 0x00000100,
1490         0x3c260, 0xffffffff, 0x00000100,
1491         0x3c27c, 0xffffffff, 0x00000100,
1492         0x3c278, 0xffffffff, 0x00000100,
1493         0x3c210, 0xffffffff, 0x06000100,
1494         0x3c290, 0xffffffff, 0x00000100,
1495         0x3c274, 0xffffffff, 0x00000100,
1496         0x3c2b4, 0xffffffff, 0x00000100,
1497         0x3c2b0, 0xffffffff, 0x00000100,
1498         0x3c270, 0xffffffff, 0x00000100,
1499         0x30800, 0xffffffff, 0xe0000000,
1500         0x3c020, 0xffffffff, 0x00010000,
1501         0x3c024, 0xffffffff, 0x00030002,
1502         0x3c028, 0xffffffff, 0x00040007,
1503         0x3c02c, 0xffffffff, 0x00060005,
1504         0x3c030, 0xffffffff, 0x00090008,
1505         0x3c034, 0xffffffff, 0x00010000,
1506         0x3c038, 0xffffffff, 0x00030002,
1507         0x3c03c, 0xffffffff, 0x00040007,
1508         0x3c040, 0xffffffff, 0x00060005,
1509         0x3c044, 0xffffffff, 0x00090008,
1510         0x3c048, 0xffffffff, 0x00010000,
1511         0x3c04c, 0xffffffff, 0x00030002,
1512         0x3c050, 0xffffffff, 0x00040007,
1513         0x3c054, 0xffffffff, 0x00060005,
1514         0x3c058, 0xffffffff, 0x00090008,
1515         0x3c05c, 0xffffffff, 0x00010000,
1516         0x3c060, 0xffffffff, 0x00030002,
1517         0x3c064, 0xffffffff, 0x00040007,
1518         0x3c068, 0xffffffff, 0x00060005,
1519         0x3c06c, 0xffffffff, 0x00090008,
1520         0x3c070, 0xffffffff, 0x00010000,
1521         0x3c074, 0xffffffff, 0x00030002,
1522         0x3c078, 0xffffffff, 0x00040007,
1523         0x3c07c, 0xffffffff, 0x00060005,
1524         0x3c080, 0xffffffff, 0x00090008,
1525         0x3c084, 0xffffffff, 0x00010000,
1526         0x3c088, 0xffffffff, 0x00030002,
1527         0x3c08c, 0xffffffff, 0x00040007,
1528         0x3c090, 0xffffffff, 0x00060005,
1529         0x3c094, 0xffffffff, 0x00090008,
1530         0x3c098, 0xffffffff, 0x00010000,
1531         0x3c09c, 0xffffffff, 0x00030002,
1532         0x3c0a0, 0xffffffff, 0x00040007,
1533         0x3c0a4, 0xffffffff, 0x00060005,
1534         0x3c0a8, 0xffffffff, 0x00090008,
1535         0x3c0ac, 0xffffffff, 0x00010000,
1536         0x3c0b0, 0xffffffff, 0x00030002,
1537         0x3c0b4, 0xffffffff, 0x00040007,
1538         0x3c0b8, 0xffffffff, 0x00060005,
1539         0x3c0bc, 0xffffffff, 0x00090008,
1540         0x3c0c0, 0xffffffff, 0x00010000,
1541         0x3c0c4, 0xffffffff, 0x00030002,
1542         0x3c0c8, 0xffffffff, 0x00040007,
1543         0x3c0cc, 0xffffffff, 0x00060005,
1544         0x3c0d0, 0xffffffff, 0x00090008,
1545         0x3c0d4, 0xffffffff, 0x00010000,
1546         0x3c0d8, 0xffffffff, 0x00030002,
1547         0x3c0dc, 0xffffffff, 0x00040007,
1548         0x3c0e0, 0xffffffff, 0x00060005,
1549         0x3c0e4, 0xffffffff, 0x00090008,
1550         0x3c0e8, 0xffffffff, 0x00010000,
1551         0x3c0ec, 0xffffffff, 0x00030002,
1552         0x3c0f0, 0xffffffff, 0x00040007,
1553         0x3c0f4, 0xffffffff, 0x00060005,
1554         0x3c0f8, 0xffffffff, 0x00090008,
1555         0xc318, 0xffffffff, 0x00020200,
1556         0x3350, 0xffffffff, 0x00000200,
1557         0x15c0, 0xffffffff, 0x00000400,
1558         0x55e8, 0xffffffff, 0x00000000,
1559         0x2f50, 0xffffffff, 0x00000902,
1560         0x3c000, 0xffffffff, 0x96940200,
1561         0x8708, 0xffffffff, 0x00900100,
1562         0xc424, 0xffffffff, 0x0020003f,
1563         0x38, 0xffffffff, 0x0140001c,
1564         0x3c, 0x000f0000, 0x000f0000,
1565         0x220, 0xffffffff, 0xc060000c,
1566         0x224, 0xc0000fff, 0x00000100,
1567         0xf90, 0xffffffff, 0x00000100,
1568         0xf98, 0x00000101, 0x00000000,
1569         0x20a8, 0xffffffff, 0x00000104,
1570         0x55e4, 0xff000fff, 0x00000100,
1571         0x30cc, 0xc0000fff, 0x00000104,
1572         0xc1e4, 0x00000001, 0x00000001,
1573         0xd00c, 0xff000ff0, 0x00000100,
1574         0xd80c, 0xff000ff0, 0x00000100
1575 };
1576
1577 static const u32 godavari_golden_registers[] =
1578 {
1579         0x55e4, 0xff607fff, 0xfc000100,
1580         0x6ed8, 0x00010101, 0x00010000,
1581         0x9830, 0xffffffff, 0x00000000,
1582         0x98302, 0xf00fffff, 0x00000400,
1583         0x6130, 0xffffffff, 0x00010000,
1584         0x5bb0, 0x000000f0, 0x00000070,
1585         0x5bc0, 0xf0311fff, 0x80300000,
1586         0x98f8, 0x73773777, 0x12010001,
1587         0x98fc, 0xffffffff, 0x00000010,
1588         0x8030, 0x00001f0f, 0x0000100a,
1589         0x2f48, 0x73773777, 0x12010001,
1590         0x2408, 0x000fffff, 0x000c007f,
1591         0x8a14, 0xf000003f, 0x00000007,
1592         0x8b24, 0xffffffff, 0x00ff0fff,
1593         0x30a04, 0x0000ff0f, 0x00000000,
1594         0x28a4c, 0x07ffffff, 0x06000000,
1595         0x4d8, 0x00000fff, 0x00000100,
1596         0xd014, 0x00010000, 0x00810001,
1597         0xd814, 0x00010000, 0x00810001,
1598         0x3e78, 0x00000001, 0x00000002,
1599         0xc768, 0x00000008, 0x00000008,
1600         0xc770, 0x00000f00, 0x00000800,
1601         0xc774, 0x00000f00, 0x00000800,
1602         0xc798, 0x00ffffff, 0x00ff7fbf,
1603         0xc79c, 0x00ffffff, 0x00ff7faf,
1604         0x8c00, 0x000000ff, 0x00000001,
1605         0x214f8, 0x01ff01ff, 0x00000002,
1606         0x21498, 0x007ff800, 0x00200000,
1607         0x2015c, 0xffffffff, 0x00000f40,
1608         0x88c4, 0x001f3ae3, 0x00000082,
1609         0x88d4, 0x0000001f, 0x00000010,
1610         0x30934, 0xffffffff, 0x00000000
1611 };
1612
1613
1614 static void cik_init_golden_registers(struct radeon_device *rdev)
1615 {
1616         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1617         mutex_lock(&rdev->grbm_idx_mutex);
1618         switch (rdev->family) {
1619         case CHIP_BONAIRE:
1620                 radeon_program_register_sequence(rdev,
1621                                                  bonaire_mgcg_cgcg_init,
1622                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1623                 radeon_program_register_sequence(rdev,
1624                                                  bonaire_golden_registers,
1625                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1626                 radeon_program_register_sequence(rdev,
1627                                                  bonaire_golden_common_registers,
1628                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1629                 radeon_program_register_sequence(rdev,
1630                                                  bonaire_golden_spm_registers,
1631                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1632                 break;
1633         case CHIP_KABINI:
1634                 radeon_program_register_sequence(rdev,
1635                                                  kalindi_mgcg_cgcg_init,
1636                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1637                 radeon_program_register_sequence(rdev,
1638                                                  kalindi_golden_registers,
1639                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1640                 radeon_program_register_sequence(rdev,
1641                                                  kalindi_golden_common_registers,
1642                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1643                 radeon_program_register_sequence(rdev,
1644                                                  kalindi_golden_spm_registers,
1645                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1646                 break;
1647         case CHIP_MULLINS:
1648                 radeon_program_register_sequence(rdev,
1649                                                  kalindi_mgcg_cgcg_init,
1650                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1651                 radeon_program_register_sequence(rdev,
1652                                                  godavari_golden_registers,
1653                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
1654                 radeon_program_register_sequence(rdev,
1655                                                  kalindi_golden_common_registers,
1656                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1657                 radeon_program_register_sequence(rdev,
1658                                                  kalindi_golden_spm_registers,
1659                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1660                 break;
1661         case CHIP_KAVERI:
1662                 radeon_program_register_sequence(rdev,
1663                                                  spectre_mgcg_cgcg_init,
1664                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1665                 radeon_program_register_sequence(rdev,
1666                                                  spectre_golden_registers,
1667                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1668                 radeon_program_register_sequence(rdev,
1669                                                  spectre_golden_common_registers,
1670                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1671                 radeon_program_register_sequence(rdev,
1672                                                  spectre_golden_spm_registers,
1673                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1674                 break;
1675         case CHIP_HAWAII:
1676                 radeon_program_register_sequence(rdev,
1677                                                  hawaii_mgcg_cgcg_init,
1678                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1679                 radeon_program_register_sequence(rdev,
1680                                                  hawaii_golden_registers,
1681                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1682                 radeon_program_register_sequence(rdev,
1683                                                  hawaii_golden_common_registers,
1684                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1685                 radeon_program_register_sequence(rdev,
1686                                                  hawaii_golden_spm_registers,
1687                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1688                 break;
1689         default:
1690                 break;
1691         }
1692         mutex_unlock(&rdev->grbm_idx_mutex);
1693 }
1694
1695 /**
1696  * cik_get_xclk - get the xclk
1697  *
1698  * @rdev: radeon_device pointer
1699  *
1700  * Returns the reference clock used by the gfx engine
1701  * (CIK).
1702  */
1703 u32 cik_get_xclk(struct radeon_device *rdev)
1704 {
1705         u32 reference_clock = rdev->clock.spll.reference_freq;
1706
1707         if (rdev->flags & RADEON_IS_IGP) {
1708                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1709                         return reference_clock / 2;
1710         } else {
1711                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1712                         return reference_clock / 4;
1713         }
1714         return reference_clock;
1715 }
1716
1717 /**
1718  * cik_mm_rdoorbell - read a doorbell dword
1719  *
1720  * @rdev: radeon_device pointer
1721  * @index: doorbell index
1722  *
1723  * Returns the value in the doorbell aperture at the
1724  * requested doorbell index (CIK).
1725  */
1726 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1727 {
1728         if (index < rdev->doorbell.num_doorbells) {
1729                 return readl(rdev->doorbell.ptr + index);
1730         } else {
1731                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1732                 return 0;
1733         }
1734 }
1735
1736 /**
1737  * cik_mm_wdoorbell - write a doorbell dword
1738  *
1739  * @rdev: radeon_device pointer
1740  * @index: doorbell index
1741  * @v: value to write
1742  *
1743  * Writes @v to the doorbell aperture at the
1744  * requested doorbell index (CIK).
1745  */
1746 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1747 {
1748         if (index < rdev->doorbell.num_doorbells) {
1749                 writel(v, rdev->doorbell.ptr + index);
1750         } else {
1751                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1752         }
1753 }
1754
1755 #define BONAIRE_IO_MC_REGS_SIZE 36
1756
1757 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1758 {
1759         {0x00000070, 0x04400000},
1760         {0x00000071, 0x80c01803},
1761         {0x00000072, 0x00004004},
1762         {0x00000073, 0x00000100},
1763         {0x00000074, 0x00ff0000},
1764         {0x00000075, 0x34000000},
1765         {0x00000076, 0x08000014},
1766         {0x00000077, 0x00cc08ec},
1767         {0x00000078, 0x00000400},
1768         {0x00000079, 0x00000000},
1769         {0x0000007a, 0x04090000},
1770         {0x0000007c, 0x00000000},
1771         {0x0000007e, 0x4408a8e8},
1772         {0x0000007f, 0x00000304},
1773         {0x00000080, 0x00000000},
1774         {0x00000082, 0x00000001},
1775         {0x00000083, 0x00000002},
1776         {0x00000084, 0xf3e4f400},
1777         {0x00000085, 0x052024e3},
1778         {0x00000087, 0x00000000},
1779         {0x00000088, 0x01000000},
1780         {0x0000008a, 0x1c0a0000},
1781         {0x0000008b, 0xff010000},
1782         {0x0000008d, 0xffffefff},
1783         {0x0000008e, 0xfff3efff},
1784         {0x0000008f, 0xfff3efbf},
1785         {0x00000092, 0xf7ffffff},
1786         {0x00000093, 0xffffff7f},
1787         {0x00000095, 0x00101101},
1788         {0x00000096, 0x00000fff},
1789         {0x00000097, 0x00116fff},
1790         {0x00000098, 0x60010000},
1791         {0x00000099, 0x10010000},
1792         {0x0000009a, 0x00006000},
1793         {0x0000009b, 0x00001000},
1794         {0x0000009f, 0x00b48000}
1795 };
1796
1797 #define HAWAII_IO_MC_REGS_SIZE 22
1798
1799 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1800 {
1801         {0x0000007d, 0x40000000},
1802         {0x0000007e, 0x40180304},
1803         {0x0000007f, 0x0000ff00},
1804         {0x00000081, 0x00000000},
1805         {0x00000083, 0x00000800},
1806         {0x00000086, 0x00000000},
1807         {0x00000087, 0x00000100},
1808         {0x00000088, 0x00020100},
1809         {0x00000089, 0x00000000},
1810         {0x0000008b, 0x00040000},
1811         {0x0000008c, 0x00000100},
1812         {0x0000008e, 0xff010000},
1813         {0x00000090, 0xffffefff},
1814         {0x00000091, 0xfff3efff},
1815         {0x00000092, 0xfff3efbf},
1816         {0x00000093, 0xf7ffffff},
1817         {0x00000094, 0xffffff7f},
1818         {0x00000095, 0x00000fff},
1819         {0x00000096, 0x00116fff},
1820         {0x00000097, 0x60010000},
1821         {0x00000098, 0x10010000},
1822         {0x0000009f, 0x00c79000}
1823 };
1824
1825
1826 /**
1827  * cik_srbm_select - select specific register instances
1828  *
1829  * @rdev: radeon_device pointer
1830  * @me: selected ME (micro engine)
1831  * @pipe: pipe
1832  * @queue: queue
1833  * @vmid: VMID
1834  *
1835  * Switches the currently active registers instances.  Some
1836  * registers are instanced per VMID, others are instanced per
1837  * me/pipe/queue combination.
1838  */
1839 static void cik_srbm_select(struct radeon_device *rdev,
1840                             u32 me, u32 pipe, u32 queue, u32 vmid)
1841 {
1842         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1843                              MEID(me & 0x3) |
1844                              VMID(vmid & 0xf) |
1845                              QUEUEID(queue & 0x7));
1846         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1847 }
1848
1849 /* ucode loading */
1850 /**
1851  * ci_mc_load_microcode - load MC ucode into the hw
1852  *
1853  * @rdev: radeon_device pointer
1854  *
1855  * Load the GDDR MC ucode into the hw (CIK).
1856  * Returns 0 on success, error on failure.
1857  */
1858 int ci_mc_load_microcode(struct radeon_device *rdev)
1859 {
1860         const __be32 *fw_data = NULL;
1861         const __le32 *new_fw_data = NULL;
1862         u32 running, tmp;
1863         u32 *io_mc_regs = NULL;
1864         const __le32 *new_io_mc_regs = NULL;
1865         int i, regs_size, ucode_size;
1866
1867         if (!rdev->mc_fw)
1868                 return -EINVAL;
1869
1870         if (rdev->new_fw) {
1871                 const struct mc_firmware_header_v1_0 *hdr =
1872                         (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1873
1874                 radeon_ucode_print_mc_hdr(&hdr->header);
1875
1876                 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1877                 new_io_mc_regs = (const __le32 *)
1878                         (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1879                 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1880                 new_fw_data = (const __le32 *)
1881                         (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1882         } else {
1883                 ucode_size = rdev->mc_fw->datasize / 4;
1884
1885                 switch (rdev->family) {
1886                 case CHIP_BONAIRE:
1887                         io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1888                         regs_size = BONAIRE_IO_MC_REGS_SIZE;
1889                         break;
1890                 case CHIP_HAWAII:
1891                         io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1892                         regs_size = HAWAII_IO_MC_REGS_SIZE;
1893                         break;
1894                 default:
1895                         return -EINVAL;
1896                 }
1897                 fw_data = (const __be32 *)rdev->mc_fw->data;
1898         }
1899
1900         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1901
1902         if (running == 0) {
1903                 /* reset the engine and set to writable */
1904                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1905                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1906
1907                 /* load mc io regs */
1908                 for (i = 0; i < regs_size; i++) {
1909                         if (rdev->new_fw) {
1910                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1911                                 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1912                         } else {
1913                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1914                                 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1915                         }
1916                 }
1917
1918                 tmp = RREG32(MC_SEQ_MISC0);
1919                 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1920                         WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1921                         WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1922                         WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1923                         WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1924                 }
1925
1926                 /* load the MC ucode */
1927                 for (i = 0; i < ucode_size; i++) {
1928                         if (rdev->new_fw)
1929                                 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1930                         else
1931                                 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1932                 }
1933
1934                 /* put the engine back into the active state */
1935                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1936                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1937                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1938
1939                 /* wait for training to complete */
1940                 for (i = 0; i < rdev->usec_timeout; i++) {
1941                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1942                                 break;
1943                         udelay(1);
1944                 }
1945                 for (i = 0; i < rdev->usec_timeout; i++) {
1946                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1947                                 break;
1948                         udelay(1);
1949                 }
1950         }
1951
1952         return 0;
1953 }
1954
1955 /**
1956  * cik_init_microcode - load ucode images from disk
1957  *
1958  * @rdev: radeon_device pointer
1959  *
1960  * Use the firmware interface to load the ucode images into
1961  * the driver (not loaded into hw).
1962  * Returns 0 on success, error on failure.
1963  */
1964 static int cik_init_microcode(struct radeon_device *rdev)
1965 {
1966         const char *chip_name;
1967         const char *new_chip_name;
1968         size_t pfp_req_size, me_req_size, ce_req_size,
1969                 mec_req_size, rlc_req_size, mc_req_size = 0,
1970                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1971         char fw_name[30];
1972         int new_fw = 0;
1973         int err;
1974         int num_fw;
1975         bool new_smc = false;
1976
1977         DRM_DEBUG("\n");
1978
1979         switch (rdev->family) {
1980         case CHIP_BONAIRE:
1981                 chip_name = "BONAIRE";
1982                 if ((rdev->pdev->revision == 0x80) ||
1983                     (rdev->pdev->revision == 0x81) ||
1984                     (rdev->pdev->device == 0x665f))
1985                         new_smc = true;
1986                 new_chip_name = "bonaire";
1987                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1988                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1989                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1990                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1991                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1992                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1993                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1994                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1995                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1996                 num_fw = 8;
1997                 break;
1998         case CHIP_HAWAII:
1999                 chip_name = "HAWAII";
2000                 if (rdev->pdev->revision == 0x80)
2001                         new_smc = true;
2002                 new_chip_name = "hawaii";
2003                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2004                 me_req_size = CIK_ME_UCODE_SIZE * 4;
2005                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2006                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2007                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2008                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
2009                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
2010                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2011                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
2012                 num_fw = 8;
2013                 break;
2014         case CHIP_KAVERI:
2015                 chip_name = "KAVERI";
2016                 new_chip_name = "kaveri";
2017                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2018                 me_req_size = CIK_ME_UCODE_SIZE * 4;
2019                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2020                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2021                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
2022                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2023                 num_fw = 7;
2024                 break;
2025         case CHIP_KABINI:
2026                 chip_name = "KABINI";
2027                 new_chip_name = "kabini";
2028                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2029                 me_req_size = CIK_ME_UCODE_SIZE * 4;
2030                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2031                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2032                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
2033                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2034                 num_fw = 6;
2035                 break;
2036         case CHIP_MULLINS:
2037                 chip_name = "MULLINS";
2038                 new_chip_name = "mullins";
2039                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2040                 me_req_size = CIK_ME_UCODE_SIZE * 4;
2041                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2042                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2043                 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2044                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2045                 num_fw = 6;
2046                 break;
2047         default: BUG();
2048         }
2049
2050         DRM_INFO("Loading %s Microcode\n", new_chip_name);
2051
2052         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", new_chip_name);
2053         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2054         if (err) {
2055                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
2056                 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2057                 if (err)
2058                         goto out;
2059                 if (rdev->pfp_fw->datasize != pfp_req_size) {
2060                         pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2061                                rdev->pfp_fw->datasize, fw_name);
2062                         err = -EINVAL;
2063                         goto out;
2064                 }
2065         } else {
2066                 err = radeon_ucode_validate(rdev->pfp_fw);
2067                 if (err) {
2068                         pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2069                                fw_name);
2070                         goto out;
2071                 } else {
2072                         new_fw++;
2073                 }
2074         }
2075
2076         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", new_chip_name);
2077         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2078         if (err) {
2079                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
2080                 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2081                 if (err)
2082                         goto out;
2083                 if (rdev->me_fw->datasize != me_req_size) {
2084                         pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2085                                rdev->me_fw->datasize, fw_name);
2086                         err = -EINVAL;
2087                 }
2088         } else {
2089                 err = radeon_ucode_validate(rdev->me_fw);
2090                 if (err) {
2091                         pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2092                                fw_name);
2093                         goto out;
2094                 } else {
2095                         new_fw++;
2096                 }
2097         }
2098
2099         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", new_chip_name);
2100         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2101         if (err) {
2102                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name);
2103                 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2104                 if (err)
2105                         goto out;
2106                 if (rdev->ce_fw->datasize != ce_req_size) {
2107                         pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2108                                rdev->ce_fw->datasize, fw_name);
2109                         err = -EINVAL;
2110                 }
2111         } else {
2112                 err = radeon_ucode_validate(rdev->ce_fw);
2113                 if (err) {
2114                         pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2115                                fw_name);
2116                         goto out;
2117                 } else {
2118                         new_fw++;
2119                 }
2120         }
2121
2122         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", new_chip_name);
2123         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2124         if (err) {
2125                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", chip_name);
2126                 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2127                 if (err)
2128                         goto out;
2129                 if (rdev->mec_fw->datasize != mec_req_size) {
2130                         pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
2131                                rdev->mec_fw->datasize, fw_name);
2132                         err = -EINVAL;
2133                 }
2134         } else {
2135                 err = radeon_ucode_validate(rdev->mec_fw);
2136                 if (err) {
2137                         pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2138                                fw_name);
2139                         goto out;
2140                 } else {
2141                         new_fw++;
2142                 }
2143         }
2144
2145         if (rdev->family == CHIP_KAVERI) {
2146                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec2", new_chip_name);
2147                 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2148                 if (err) {
2149                         goto out;
2150                 } else {
2151                         err = radeon_ucode_validate(rdev->mec2_fw);
2152                         if (err) {
2153                                 goto out;
2154                         } else {
2155                                 new_fw++;
2156                         }
2157                 }
2158         }
2159
2160         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", new_chip_name);
2161         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2162         if (err) {
2163                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", chip_name);
2164                 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2165                 if (err)
2166                         goto out;
2167                 if (rdev->rlc_fw->datasize != rlc_req_size) {
2168                         pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2169                                rdev->rlc_fw->datasize, fw_name);
2170                         err = -EINVAL;
2171                 }
2172         } else {
2173                 err = radeon_ucode_validate(rdev->rlc_fw);
2174                 if (err) {
2175                         pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2176                                fw_name);
2177                         goto out;
2178                 } else {
2179                         new_fw++;
2180                 }
2181         }
2182
2183         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", new_chip_name);
2184         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2185         if (err) {
2186                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", chip_name);
2187                 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2188                 if (err)
2189                         goto out;
2190                 if (rdev->sdma_fw->datasize != sdma_req_size) {
2191                         pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2192                                rdev->sdma_fw->datasize, fw_name);
2193                         err = -EINVAL;
2194                 }
2195         } else {
2196                 err = radeon_ucode_validate(rdev->sdma_fw);
2197                 if (err) {
2198                         pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2199                                fw_name);
2200                         goto out;
2201                 } else {
2202                         new_fw++;
2203                 }
2204         }
2205
2206         /* No SMC, MC ucode on APUs */
2207         if (!(rdev->flags & RADEON_IS_IGP)) {
2208                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", new_chip_name);
2209                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2210                 if (err) {
2211                         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name);
2212                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2213                         if (err) {
2214                                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
2215                                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2216                                 if (err)
2217                                         goto out;
2218                         }
2219                         if ((rdev->mc_fw->datasize != mc_req_size) &&
2220                             (rdev->mc_fw->datasize != mc2_req_size)){
2221                                 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
2222                                        rdev->mc_fw->datasize, fw_name);
2223                                 err = -EINVAL;
2224                         }
2225                         DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize);
2226                 } else {
2227                         err = radeon_ucode_validate(rdev->mc_fw);
2228                         if (err) {
2229                                 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2230                                        fw_name);
2231                                 goto out;
2232                         } else {
2233                                 new_fw++;
2234                         }
2235                 }
2236
2237                 if (new_smc)
2238                         ksnprintf(fw_name, sizeof(fw_name), "amdgpufw_%s_k_smc", new_chip_name);
2239                 else
2240                         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", new_chip_name);
2241                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2242                 if (err) {
2243                         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
2244                         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2245                         if (err) {
2246                                 pr_err("smc: error loading firmware \"%s\"\n",
2247                                        fw_name);
2248                                 release_firmware(rdev->smc_fw);
2249                                 rdev->smc_fw = NULL;
2250                                 err = 0;
2251                         } else if (rdev->smc_fw->datasize != smc_req_size) {
2252                                 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
2253                                        rdev->smc_fw->datasize, fw_name);
2254                                 err = -EINVAL;
2255                         }
2256                 } else {
2257                         err = radeon_ucode_validate(rdev->smc_fw);
2258                         if (err) {
2259                                 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
2260                                        fw_name);
2261                                 goto out;
2262                         } else {
2263                                 new_fw++;
2264                         }
2265                 }
2266         }
2267
2268         if (new_fw == 0) {
2269                 rdev->new_fw = false;
2270         } else if (new_fw < num_fw) {
2271                 pr_err("ci_fw: mixing new and old firmware!\n");
2272                 err = -EINVAL;
2273         } else {
2274                 rdev->new_fw = true;
2275         }
2276
2277 out:
2278         if (err) {
2279                 if (err != -EINVAL)
2280                         pr_err("cik_cp: Failed to load firmware \"%s\"\n",
2281                                fw_name);
2282                 release_firmware(rdev->pfp_fw);
2283                 rdev->pfp_fw = NULL;
2284                 release_firmware(rdev->me_fw);
2285                 rdev->me_fw = NULL;
2286                 release_firmware(rdev->ce_fw);
2287                 rdev->ce_fw = NULL;
2288                 release_firmware(rdev->mec_fw);
2289                 rdev->mec_fw = NULL;
2290                 release_firmware(rdev->mec2_fw);
2291                 rdev->mec2_fw = NULL;
2292                 release_firmware(rdev->rlc_fw);
2293                 rdev->rlc_fw = NULL;
2294                 release_firmware(rdev->sdma_fw);
2295                 rdev->sdma_fw = NULL;
2296                 release_firmware(rdev->mc_fw);
2297                 rdev->mc_fw = NULL;
2298                 release_firmware(rdev->smc_fw);
2299                 rdev->smc_fw = NULL;
2300         }
2301         return err;
2302 }
2303
2304 /**
2305  * cik_fini_microcode - drop the firmwares image references
2306  *
2307  * @rdev: radeon_device pointer
2308  *
2309  * Drop the pfp, me, mec, mec2, rlc, sdma, mc, smc and ce firmware image references.
2310  * Called at driver shutdown.
2311  */
2312 static void cik_fini_microcode(struct radeon_device *rdev)
2313 {
2314         release_firmware(rdev->pfp_fw);
2315         rdev->pfp_fw = NULL;
2316         release_firmware(rdev->me_fw);
2317         rdev->me_fw = NULL;
2318         release_firmware(rdev->ce_fw);
2319         rdev->ce_fw = NULL;
2320         release_firmware(rdev->mec_fw);
2321         rdev->mec_fw = NULL;
2322         release_firmware(rdev->mec2_fw);
2323         rdev->mec2_fw = NULL;
2324         release_firmware(rdev->rlc_fw);
2325         rdev->rlc_fw = NULL;
2326         release_firmware(rdev->sdma_fw);
2327         rdev->sdma_fw = NULL;
2328         release_firmware(rdev->mc_fw);
2329         rdev->mc_fw = NULL;
2330         release_firmware(rdev->smc_fw);
2331         rdev->smc_fw = NULL;
2332 }
2333
2334 /*
2335  * Core functions
2336  */
2337 /**
2338  * cik_tiling_mode_table_init - init the hw tiling table
2339  *
2340  * @rdev: radeon_device pointer
2341  *
2342  * Starting with SI, the tiling setup is done globally in a
2343  * set of 32 tiling modes.  Rather than selecting each set of
2344  * parameters per surface as on older asics, we just select
2345  * which index in the tiling table we want to use, and the
2346  * surface uses those parameters (CIK).
2347  */
2348 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2349 {
2350         u32 *tile = rdev->config.cik.tile_mode_array;
2351         u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2352         const u32 num_tile_mode_states =
2353                         ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2354         const u32 num_secondary_tile_mode_states =
2355                         ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2356         u32 reg_offset, split_equal_to_row_size;
2357         u32 num_pipe_configs;
2358         u32 num_rbs = rdev->config.cik.max_backends_per_se *
2359                 rdev->config.cik.max_shader_engines;
2360
2361         switch (rdev->config.cik.mem_row_size_in_kb) {
2362         case 1:
2363                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2364                 break;
2365         case 2:
2366         default:
2367                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2368                 break;
2369         case 4:
2370                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2371                 break;
2372         }
2373
2374         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2375         if (num_pipe_configs > 8)
2376                 num_pipe_configs = 16;
2377
2378         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2379                 tile[reg_offset] = 0;
2380         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2381                 macrotile[reg_offset] = 0;
2382
2383         switch(num_pipe_configs) {
2384         case 16:
2385                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2386                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2387                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2388                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2389                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2391                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2393                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2395                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2397                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2398                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2399                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2400                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2401                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2403                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404                            TILE_SPLIT(split_equal_to_row_size));
2405                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2406                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2408                 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2409                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2410                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2411                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2412                 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2413                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2414                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415                            TILE_SPLIT(split_equal_to_row_size));
2416                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2417                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2418                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2419                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2420                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2421                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2422                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2423                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2424                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2426                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2427                             PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2428                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2429                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2430                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2431                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2432                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2433                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2434                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2435                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2436                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2437                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2438                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2439                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2441                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2442                             PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2443                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2444                 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2445                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2446                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2447                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2448                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2449                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2450                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2451                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2453                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2454                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2455                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2456                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2457                             PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2458                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2459                 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2460                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2461                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2462                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463
2464                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2466                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2467                            NUM_BANKS(ADDR_SURF_16_BANK));
2468                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2470                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2471                            NUM_BANKS(ADDR_SURF_16_BANK));
2472                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2475                            NUM_BANKS(ADDR_SURF_16_BANK));
2476                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2479                            NUM_BANKS(ADDR_SURF_16_BANK));
2480                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2482                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2483                            NUM_BANKS(ADDR_SURF_8_BANK));
2484                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487                            NUM_BANKS(ADDR_SURF_4_BANK));
2488                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491                            NUM_BANKS(ADDR_SURF_2_BANK));
2492                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2493                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2494                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2495                            NUM_BANKS(ADDR_SURF_16_BANK));
2496                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497                            BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2498                            MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2499                            NUM_BANKS(ADDR_SURF_16_BANK));
2500                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501                             BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2502                             MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2503                             NUM_BANKS(ADDR_SURF_16_BANK));
2504                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505                             BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2506                             MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2507                             NUM_BANKS(ADDR_SURF_8_BANK));
2508                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509                             BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2510                             MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2511                             NUM_BANKS(ADDR_SURF_4_BANK));
2512                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2513                             BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2514                             MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2515                             NUM_BANKS(ADDR_SURF_2_BANK));
2516                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517                             BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2518                             MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2519                             NUM_BANKS(ADDR_SURF_2_BANK));
2520
2521                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2522                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2523                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2524                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2525                 break;
2526
2527         case 8:
2528                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2529                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2530                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2532                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2534                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2536                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2540                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2541                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2542                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2544                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2545                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2546                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547                            TILE_SPLIT(split_equal_to_row_size));
2548                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2549                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2550                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2551                 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2552                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2553                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2554                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2555                 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2556                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2557                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2558                            TILE_SPLIT(split_equal_to_row_size));
2559                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2560                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2561                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2562                            PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2564                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2566                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2567                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2568                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2569                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2570                             PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2572                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2573                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2574                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2576                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2579                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2580                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2581                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2582                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2583                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2584                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2585                             PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2586                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2587                 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2588                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2589                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2590                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2592                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2593                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2594                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2595                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2596                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2597                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2598                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2599                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2600                             PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2601                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2602                 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2603                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2604                             PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2605                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2606
2607                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2609                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2610                                 NUM_BANKS(ADDR_SURF_16_BANK));
2611                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2613                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2614                                 NUM_BANKS(ADDR_SURF_16_BANK));
2615                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2617                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2618                                 NUM_BANKS(ADDR_SURF_16_BANK));
2619                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2621                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2622                                 NUM_BANKS(ADDR_SURF_16_BANK));
2623                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2626                                 NUM_BANKS(ADDR_SURF_8_BANK));
2627                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2630                                 NUM_BANKS(ADDR_SURF_4_BANK));
2631                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2634                                 NUM_BANKS(ADDR_SURF_2_BANK));
2635                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2637                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2638                                 NUM_BANKS(ADDR_SURF_16_BANK));
2639                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2640                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2641                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2642                                 NUM_BANKS(ADDR_SURF_16_BANK));
2643                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2645                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2646                                 NUM_BANKS(ADDR_SURF_16_BANK));
2647                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2649                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2650                                 NUM_BANKS(ADDR_SURF_16_BANK));
2651                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2652                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2653                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2654                                 NUM_BANKS(ADDR_SURF_8_BANK));
2655                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2657                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2658                                 NUM_BANKS(ADDR_SURF_4_BANK));
2659                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2660                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2661                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2662                                 NUM_BANKS(ADDR_SURF_2_BANK));
2663
2664                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2665                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2666                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2667                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2668                 break;
2669
2670         case 4:
2671                 if (num_rbs == 4) {
2672                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2674                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2675                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2676                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2678                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2680                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2682                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2684                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2685                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2686                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2687                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2688                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2689                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2690                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691                            TILE_SPLIT(split_equal_to_row_size));
2692                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2693                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2694                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2695                 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2696                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2697                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2698                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2699                 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2700                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2701                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2702                            TILE_SPLIT(split_equal_to_row_size));
2703                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2704                            PIPE_CONFIG(ADDR_SURF_P4_16x16));
2705                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2706                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2707                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2708                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2709                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2710                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2711                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2712                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2713                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2714                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2715                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2716                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2717                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2718                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2719                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2720                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2721                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2722                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2723                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2724                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2725                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2726                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2727                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2728                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2729                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2730                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2731                 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2732                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2733                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2734                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2735                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2736                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2737                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2738                 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2739                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2740                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2741                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2742                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2743                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2744                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2745                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2746                 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2747                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2748                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2749                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2750
2751                 } else if (num_rbs < 4) {
2752                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2753                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2754                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2755                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2756                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2757                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2758                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2760                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2761                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2762                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2764                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2765                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2766                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2768                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2769                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2770                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771                            TILE_SPLIT(split_equal_to_row_size));
2772                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2773                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2774                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2775                 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2776                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2777                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2779                 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2780                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2781                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2782                            TILE_SPLIT(split_equal_to_row_size));
2783                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2784                            PIPE_CONFIG(ADDR_SURF_P4_8x16));
2785                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2786                            PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2788                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2789                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2790                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2791                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2792                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2793                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2794                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2796                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2797                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2798                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2799                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2800                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2802                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2803                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2804                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2805                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2806                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2807                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2808                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2809                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2810                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2811                 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2812                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2813                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2816                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2817                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2818                 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2820                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2821                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2822                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2823                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2824                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2825                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2826                 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2827                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2828                             PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2829                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2830                 }
2831
2832                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2833                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2834                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2835                                 NUM_BANKS(ADDR_SURF_16_BANK));
2836                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2837                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2838                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2839                                 NUM_BANKS(ADDR_SURF_16_BANK));
2840                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2842                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2843                                 NUM_BANKS(ADDR_SURF_16_BANK));
2844                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2846                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2847                                 NUM_BANKS(ADDR_SURF_16_BANK));
2848                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2849                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2850                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2851                                 NUM_BANKS(ADDR_SURF_16_BANK));
2852                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2854                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2855                                 NUM_BANKS(ADDR_SURF_8_BANK));
2856                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2859                                 NUM_BANKS(ADDR_SURF_4_BANK));
2860                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2861                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2862                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2863                                 NUM_BANKS(ADDR_SURF_16_BANK));
2864                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2865                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2866                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2867                                 NUM_BANKS(ADDR_SURF_16_BANK));
2868                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2869                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2870                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2871                                 NUM_BANKS(ADDR_SURF_16_BANK));
2872                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2873                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2874                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2875                                 NUM_BANKS(ADDR_SURF_16_BANK));
2876                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2878                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2879                                 NUM_BANKS(ADDR_SURF_16_BANK));
2880                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2881                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2882                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2883                                 NUM_BANKS(ADDR_SURF_8_BANK));
2884                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2885                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2886                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2887                                 NUM_BANKS(ADDR_SURF_4_BANK));
2888
2889                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2890                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2891                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2892                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2893                 break;
2894
2895         case 2:
2896                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2897                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2898                            PIPE_CONFIG(ADDR_SURF_P2) |
2899                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2900                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2902                            PIPE_CONFIG(ADDR_SURF_P2) |
2903                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2904                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2905                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2906                            PIPE_CONFIG(ADDR_SURF_P2) |
2907                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2908                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2909                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2910                            PIPE_CONFIG(ADDR_SURF_P2) |
2911                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2912                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2913                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2914                            PIPE_CONFIG(ADDR_SURF_P2) |
2915                            TILE_SPLIT(split_equal_to_row_size));
2916                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2917                            PIPE_CONFIG(ADDR_SURF_P2) |
2918                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2919                 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2920                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2921                            PIPE_CONFIG(ADDR_SURF_P2) |
2922                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2923                 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2924                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2925                            PIPE_CONFIG(ADDR_SURF_P2) |
2926                            TILE_SPLIT(split_equal_to_row_size));
2927                 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2928                            PIPE_CONFIG(ADDR_SURF_P2);
2929                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2930                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2931                            PIPE_CONFIG(ADDR_SURF_P2));
2932                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2933                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2934                             PIPE_CONFIG(ADDR_SURF_P2) |
2935                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2936                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2937                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2938                             PIPE_CONFIG(ADDR_SURF_P2) |
2939                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2940                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2941                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2942                             PIPE_CONFIG(ADDR_SURF_P2) |
2943                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2944                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2945                             PIPE_CONFIG(ADDR_SURF_P2) |
2946                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2947                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2948                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2949                             PIPE_CONFIG(ADDR_SURF_P2) |
2950                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2952                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2953                             PIPE_CONFIG(ADDR_SURF_P2) |
2954                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2955                 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2956                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2957                             PIPE_CONFIG(ADDR_SURF_P2) |
2958                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2959                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2960                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2961                             PIPE_CONFIG(ADDR_SURF_P2));
2962                 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2963                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2964                             PIPE_CONFIG(ADDR_SURF_P2) |
2965                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2966                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2967                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2968                             PIPE_CONFIG(ADDR_SURF_P2) |
2969                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2970                 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2972                             PIPE_CONFIG(ADDR_SURF_P2) |
2973                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974
2975                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2976                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2977                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2978                                 NUM_BANKS(ADDR_SURF_16_BANK));
2979                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2980                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2981                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2982                                 NUM_BANKS(ADDR_SURF_16_BANK));
2983                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2984                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2985                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2986                                 NUM_BANKS(ADDR_SURF_16_BANK));
2987                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2988                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2989                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990                                 NUM_BANKS(ADDR_SURF_16_BANK));
2991                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2993                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994                                 NUM_BANKS(ADDR_SURF_16_BANK));
2995                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2996                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2997                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998                                 NUM_BANKS(ADDR_SURF_16_BANK));
2999                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3000                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3001                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3002                                 NUM_BANKS(ADDR_SURF_8_BANK));
3003                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3004                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3005                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3006                                 NUM_BANKS(ADDR_SURF_16_BANK));
3007                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3008                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3009                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3010                                 NUM_BANKS(ADDR_SURF_16_BANK));
3011                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3012                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3013                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3014                                 NUM_BANKS(ADDR_SURF_16_BANK));
3015                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3016                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018                                 NUM_BANKS(ADDR_SURF_16_BANK));
3019                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3020                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3021                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3022                                 NUM_BANKS(ADDR_SURF_16_BANK));
3023                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3024                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3025                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026                                 NUM_BANKS(ADDR_SURF_16_BANK));
3027                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030                                 NUM_BANKS(ADDR_SURF_8_BANK));
3031
3032                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3033                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3034                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3035                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3036                 break;
3037
3038         default:
3039                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3040         }
3041 }
3042
3043 /**
3044  * cik_select_se_sh - select which SE, SH to address
3045  *
3046  * @rdev: radeon_device pointer
3047  * @se_num: shader engine to address
3048  * @sh_num: sh block to address
3049  *
3050  * Select which SE, SH combinations to address. Certain
3051  * registers are instanced per SE or SH.  0xffffffff means
3052  * broadcast to all SEs or SHs (CIK).
3053  */
3054 static void cik_select_se_sh(struct radeon_device *rdev,
3055                              u32 se_num, u32 sh_num)
3056 {
3057         u32 data = INSTANCE_BROADCAST_WRITES;
3058
3059         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3060                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3061         else if (se_num == 0xffffffff)
3062                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3063         else if (sh_num == 0xffffffff)
3064                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3065         else
3066                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3067         WREG32(GRBM_GFX_INDEX, data);
3068 }
3069
3070 /**
3071  * cik_create_bitmask - create a bitmask
3072  *
3073  * @bit_width: length of the mask
3074  *
3075  * create a variable length bit mask (CIK).
3076  * Returns the bitmask.
3077  */
3078 static u32 cik_create_bitmask(u32 bit_width)
3079 {
3080         u32 i, mask = 0;
3081
3082         for (i = 0; i < bit_width; i++) {
3083                 mask <<= 1;
3084                 mask |= 1;
3085         }
3086         return mask;
3087 }
3088
3089 /**
3090  * cik_get_rb_disabled - computes the mask of disabled RBs
3091  *
3092  * @rdev: radeon_device pointer
3093  * @max_rb_num: max RBs (render backends) for the asic
3094  * @se_num: number of SEs (shader engines) for the asic
3095  * @sh_per_se: number of SH blocks per SE for the asic
3096  *
3097  * Calculates the bitmask of disabled RBs (CIK).
3098  * Returns the disabled RB bitmask.
3099  */
3100 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3101                               u32 max_rb_num_per_se,
3102                               u32 sh_per_se)
3103 {
3104         u32 data, mask;
3105
3106         data = RREG32(CC_RB_BACKEND_DISABLE);
3107         if (data & 1)
3108                 data &= BACKEND_DISABLE_MASK;
3109         else
3110                 data = 0;
3111         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3112
3113         data >>= BACKEND_DISABLE_SHIFT;
3114
3115         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3116
3117         return data & mask;
3118 }
3119
3120 /**
3121  * cik_setup_rb - setup the RBs on the asic
3122  *
3123  * @rdev: radeon_device pointer
3124  * @se_num: number of SEs (shader engines) for the asic
3125  * @sh_per_se: number of SH blocks per SE for the asic
3126  * @max_rb_num: max RBs (render backends) for the asic
3127  *
3128  * Configures per-SE/SH RB registers (CIK).
3129  */
3130 static void cik_setup_rb(struct radeon_device *rdev,
3131                          u32 se_num, u32 sh_per_se,
3132                          u32 max_rb_num_per_se)
3133 {
3134         int i, j;
3135         u32 data, mask;
3136         u32 disabled_rbs = 0;
3137         u32 enabled_rbs = 0;
3138
3139         mutex_lock(&rdev->grbm_idx_mutex);
3140         for (i = 0; i < se_num; i++) {
3141                 for (j = 0; j < sh_per_se; j++) {
3142                         cik_select_se_sh(rdev, i, j);
3143                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3144                         if (rdev->family == CHIP_HAWAII)
3145                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3146                         else
3147                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3148                 }
3149         }
3150         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3151         mutex_unlock(&rdev->grbm_idx_mutex);
3152
3153         mask = 1;
3154         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3155                 if (!(disabled_rbs & mask))
3156                         enabled_rbs |= mask;
3157                 mask <<= 1;
3158         }
3159
3160         rdev->config.cik.backend_enable_mask = enabled_rbs;
3161
3162         mutex_lock(&rdev->grbm_idx_mutex);
3163         for (i = 0; i < se_num; i++) {
3164                 cik_select_se_sh(rdev, i, 0xffffffff);
3165                 data = 0;
3166                 for (j = 0; j < sh_per_se; j++) {
3167                         switch (enabled_rbs & 3) {
3168                         case 0:
3169                                 if (j == 0)
3170                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3171                                 else
3172                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3173                                 break;
3174                         case 1:
3175                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3176                                 break;
3177                         case 2:
3178                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3179                                 break;
3180                         case 3:
3181                         default:
3182                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3183                                 break;
3184                         }
3185                         enabled_rbs >>= 2;
3186                 }
3187                 WREG32(PA_SC_RASTER_CONFIG, data);
3188         }
3189         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3190         mutex_unlock(&rdev->grbm_idx_mutex);
3191 }
3192
3193 /**
3194  * cik_gpu_init - setup the 3D engine
3195  *
3196  * @rdev: radeon_device pointer
3197  *
3198  * Configures the 3D engine and tiling configuration
3199  * registers so that the 3D engine is usable.
3200  */
3201 static void cik_gpu_init(struct radeon_device *rdev)
3202 {
3203         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3204         u32 mc_shared_chmap, mc_arb_ramcfg;
3205         u32 hdp_host_path_cntl;
3206         u32 tmp;
3207         int i, j;
3208
3209         switch (rdev->family) {
3210         case CHIP_BONAIRE:
3211                 rdev->config.cik.max_shader_engines = 2;
3212                 rdev->config.cik.max_tile_pipes = 4;
3213                 rdev->config.cik.max_cu_per_sh = 7;
3214                 rdev->config.cik.max_sh_per_se = 1;
3215                 rdev->config.cik.max_backends_per_se = 2;
3216                 rdev->config.cik.max_texture_channel_caches = 4;
3217                 rdev->config.cik.max_gprs = 256;
3218                 rdev->config.cik.max_gs_threads = 32;
3219                 rdev->config.cik.max_hw_contexts = 8;
3220
3221                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3222                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3223                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3224                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3225                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3226                 break;
3227         case CHIP_HAWAII:
3228                 rdev->config.cik.max_shader_engines = 4;
3229                 rdev->config.cik.max_tile_pipes = 16;
3230                 rdev->config.cik.max_cu_per_sh = 11;
3231                 rdev->config.cik.max_sh_per_se = 1;
3232                 rdev->config.cik.max_backends_per_se = 4;
3233                 rdev->config.cik.max_texture_channel_caches = 16;
3234                 rdev->config.cik.max_gprs = 256;
3235                 rdev->config.cik.max_gs_threads = 32;
3236                 rdev->config.cik.max_hw_contexts = 8;
3237
3238                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3239                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3240                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3241                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3242                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3243                 break;
3244         case CHIP_KAVERI:
3245                 rdev->config.cik.max_shader_engines = 1;
3246                 rdev->config.cik.max_tile_pipes = 4;
3247                 rdev->config.cik.max_cu_per_sh = 8;
3248                 rdev->config.cik.max_backends_per_se = 2;
3249                 rdev->config.cik.max_sh_per_se = 1;
3250                 rdev->config.cik.max_texture_channel_caches = 4;
3251                 rdev->config.cik.max_gprs = 256;
3252                 rdev->config.cik.max_gs_threads = 16;
3253                 rdev->config.cik.max_hw_contexts = 8;
3254
3255                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3256                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3257                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3258                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3259                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3260                 break;
3261         case CHIP_KABINI:
3262         case CHIP_MULLINS:
3263         default:
3264                 rdev->config.cik.max_shader_engines = 1;
3265                 rdev->config.cik.max_tile_pipes = 2;
3266                 rdev->config.cik.max_cu_per_sh = 2;
3267                 rdev->config.cik.max_sh_per_se = 1;
3268                 rdev->config.cik.max_backends_per_se = 1;
3269                 rdev->config.cik.max_texture_channel_caches = 2;
3270                 rdev->config.cik.max_gprs = 256;
3271                 rdev->config.cik.max_gs_threads = 16;
3272                 rdev->config.cik.max_hw_contexts = 8;
3273
3274                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3275                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3276                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3277                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3278                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3279                 break;
3280         }
3281
3282         /* Initialize HDP */
3283         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3284                 WREG32((0x2c14 + j), 0x00000000);
3285                 WREG32((0x2c18 + j), 0x00000000);
3286                 WREG32((0x2c1c + j), 0x00000000);
3287                 WREG32((0x2c20 + j), 0x00000000);
3288                 WREG32((0x2c24 + j), 0x00000000);
3289         }
3290
3291         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3292         WREG32(SRBM_INT_CNTL, 0x1);
3293         WREG32(SRBM_INT_ACK, 0x1);
3294
3295         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3296
3297         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3298         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3299
3300         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3301         rdev->config.cik.mem_max_burst_length_bytes = 256;
3302         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3303         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3304         if (rdev->config.cik.mem_row_size_in_kb > 4)
3305                 rdev->config.cik.mem_row_size_in_kb = 4;
3306         /* XXX use MC settings? */
3307         rdev->config.cik.shader_engine_tile_size = 32;
3308         rdev->config.cik.num_gpus = 1;
3309         rdev->config.cik.multi_gpu_tile_size = 64;
3310
3311         /* fix up row size */
3312         gb_addr_config &= ~ROW_SIZE_MASK;
3313         switch (rdev->config.cik.mem_row_size_in_kb) {
3314         case 1:
3315         default:
3316                 gb_addr_config |= ROW_SIZE(0);
3317                 break;
3318         case 2:
3319                 gb_addr_config |= ROW_SIZE(1);
3320                 break;
3321         case 4:
3322                 gb_addr_config |= ROW_SIZE(2);
3323                 break;
3324         }
3325
3326         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3327          * not have bank info, so create a custom tiling dword.
3328          * bits 3:0   num_pipes
3329          * bits 7:4   num_banks
3330          * bits 11:8  group_size
3331          * bits 15:12 row_size
3332          */
3333         rdev->config.cik.tile_config = 0;
3334         switch (rdev->config.cik.num_tile_pipes) {
3335         case 1:
3336                 rdev->config.cik.tile_config |= (0 << 0);
3337                 break;
3338         case 2:
3339                 rdev->config.cik.tile_config |= (1 << 0);
3340                 break;
3341         case 4:
3342                 rdev->config.cik.tile_config |= (2 << 0);
3343                 break;
3344         case 8:
3345         default:
3346                 /* XXX what about 12? */
3347                 rdev->config.cik.tile_config |= (3 << 0);
3348                 break;
3349         }
3350         rdev->config.cik.tile_config |=
3351                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3352         rdev->config.cik.tile_config |=
3353                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3354         rdev->config.cik.tile_config |=
3355                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3356
3357         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3358         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3359         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3360         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3361         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3362         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3363         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3364         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3365
3366         cik_tiling_mode_table_init(rdev);
3367
3368         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3369                      rdev->config.cik.max_sh_per_se,
3370                      rdev->config.cik.max_backends_per_se);
3371
3372         rdev->config.cik.active_cus = 0;
3373         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3374                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3375                         rdev->config.cik.active_cus +=
3376                                 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3377                 }
3378         }
3379
3380         /* set HW defaults for 3D engine */
3381         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3382
3383         mutex_lock(&rdev->grbm_idx_mutex);
3384         /*
3385          * making sure that the following register writes will be broadcasted
3386          * to all the shaders
3387          */
3388         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3389         WREG32(SX_DEBUG_1, 0x20);
3390
3391         WREG32(TA_CNTL_AUX, 0x00010000);
3392
3393         tmp = RREG32(SPI_CONFIG_CNTL);
3394         tmp |= 0x03000000;
3395         WREG32(SPI_CONFIG_CNTL, tmp);
3396
3397         WREG32(SQ_CONFIG, 1);
3398
3399         WREG32(DB_DEBUG, 0);
3400
3401         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3402         tmp |= 0x00000400;
3403         WREG32(DB_DEBUG2, tmp);
3404
3405         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3406         tmp |= 0x00020200;
3407         WREG32(DB_DEBUG3, tmp);
3408
3409         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3410         tmp |= 0x00018208;
3411         WREG32(CB_HW_CONTROL, tmp);
3412
3413         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3414
3415         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3416                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3417                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3418                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3419
3420         WREG32(VGT_NUM_INSTANCES, 1);
3421
3422         WREG32(CP_PERFMON_CNTL, 0);
3423
3424         WREG32(SQ_CONFIG, 0);
3425
3426         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3427                                           FORCE_EOV_MAX_REZ_CNT(255)));
3428
3429         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3430                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3431
3432         WREG32(VGT_GS_VERTEX_REUSE, 16);
3433         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3434
3435         tmp = RREG32(HDP_MISC_CNTL);
3436         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3437         WREG32(HDP_MISC_CNTL, tmp);
3438
3439         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3440         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3441
3442         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3443         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3444         mutex_unlock(&rdev->grbm_idx_mutex);
3445
3446         udelay(50);
3447 }
3448
3449 /*
3450  * GPU scratch registers helpers function.
3451  */
3452 /**
3453  * cik_scratch_init - setup driver info for CP scratch regs
3454  *
3455  * @rdev: radeon_device pointer
3456  *
3457  * Set up the number and offset of the CP scratch registers.
3458  * NOTE: use of CP scratch registers is a legacy inferface and
3459  * is not used by default on newer asics (r6xx+).  On newer asics,
3460  * memory buffers are used for fences rather than scratch regs.
3461  */
3462 static void cik_scratch_init(struct radeon_device *rdev)
3463 {
3464         int i;
3465
3466         rdev->scratch.num_reg = 7;
3467         rdev->scratch.reg_base = SCRATCH_REG0;
3468         for (i = 0; i < rdev->scratch.num_reg; i++) {
3469                 rdev->scratch.free[i] = true;
3470                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3471         }
3472 }
3473
3474 /**
3475  * cik_ring_test - basic gfx ring test
3476  *
3477  * @rdev: radeon_device pointer
3478  * @ring: radeon_ring structure holding ring information
3479  *
3480  * Allocate a scratch register and write to it using the gfx ring (CIK).
3481  * Provides a basic gfx ring test to verify that the ring is working.
3482  * Used by cik_cp_gfx_resume();
3483  * Returns 0 on success, error on failure.
3484  */
3485 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3486 {
3487         uint32_t scratch;
3488         uint32_t tmp = 0;
3489         unsigned i;
3490         int r;
3491
3492         r = radeon_scratch_get(rdev, &scratch);
3493         if (r) {
3494                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3495                 return r;
3496         }
3497         WREG32(scratch, 0xCAFEDEAD);
3498         r = radeon_ring_lock(rdev, ring, 3);
3499         if (r) {
3500                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3501                 radeon_scratch_free(rdev, scratch);
3502                 return r;
3503         }
3504         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3505         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3506         radeon_ring_write(ring, 0xDEADBEEF);
3507         radeon_ring_unlock_commit(rdev, ring, false);
3508
3509         for (i = 0; i < rdev->usec_timeout; i++) {
3510                 tmp = RREG32(scratch);
3511                 if (tmp == 0xDEADBEEF)
3512                         break;
3513                 DRM_UDELAY(1);
3514         }
3515         if (i < rdev->usec_timeout) {
3516                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3517         } else {
3518                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3519                           ring->idx, scratch, tmp);
3520                 r = -EINVAL;
3521         }
3522         radeon_scratch_free(rdev, scratch);
3523         return r;
3524 }
3525
3526 /**
3527  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3528  *
3529  * @rdev: radeon_device pointer
3530  * @ridx: radeon ring index
3531  *
3532  * Emits an hdp flush on the cp.
3533  */
3534 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3535                                        int ridx)
3536 {
3537         struct radeon_ring *ring = &rdev->ring[ridx];
3538         u32 ref_and_mask;
3539
3540         switch (ring->idx) {
3541         case CAYMAN_RING_TYPE_CP1_INDEX:
3542         case CAYMAN_RING_TYPE_CP2_INDEX:
3543         default:
3544                 switch (ring->me) {
3545                 case 0:
3546                         ref_and_mask = CP2 << ring->pipe;
3547                         break;
3548                 case 1:
3549                         ref_and_mask = CP6 << ring->pipe;
3550                         break;
3551                 default:
3552                         return;
3553                 }
3554                 break;
3555         case RADEON_RING_TYPE_GFX_INDEX:
3556                 ref_and_mask = CP0;
3557                 break;
3558         }
3559
3560         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3561         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3562                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3563                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3564         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3565         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3566         radeon_ring_write(ring, ref_and_mask);
3567         radeon_ring_write(ring, ref_and_mask);
3568         radeon_ring_write(ring, 0x20); /* poll interval */
3569 }
3570
3571 /**
3572  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3573  *
3574  * @rdev: radeon_device pointer
3575  * @fence: radeon fence object
3576  *
3577  * Emits a fence sequnce number on the gfx ring and flushes
3578  * GPU caches.
3579  */
3580 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3581                              struct radeon_fence *fence)
3582 {
3583         struct radeon_ring *ring = &rdev->ring[fence->ring];
3584         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3585
3586         /* Workaround for cache flush problems. First send a dummy EOP
3587          * event down the pipe with seq one below.
3588          */
3589         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3590         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3591                                  EOP_TC_ACTION_EN |
3592                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3593                                  EVENT_INDEX(5)));
3594         radeon_ring_write(ring, addr & 0xfffffffc);
3595         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3596                                 DATA_SEL(1) | INT_SEL(0));
3597         radeon_ring_write(ring, fence->seq - 1);
3598         radeon_ring_write(ring, 0);
3599
3600         /* Then send the real EOP event down the pipe. */
3601         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3602         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3603                                  EOP_TC_ACTION_EN |
3604                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3605                                  EVENT_INDEX(5)));
3606         radeon_ring_write(ring, addr & 0xfffffffc);
3607         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3608         radeon_ring_write(ring, fence->seq);
3609         radeon_ring_write(ring, 0);
3610 }
3611
3612 /**
3613  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3614  *
3615  * @rdev: radeon_device pointer
3616  * @fence: radeon fence object
3617  *
3618  * Emits a fence sequnce number on the compute ring and flushes
3619  * GPU caches.
3620  */
3621 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3622                                  struct radeon_fence *fence)
3623 {
3624         struct radeon_ring *ring = &rdev->ring[fence->ring];
3625         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3626
3627         /* RELEASE_MEM - flush caches, send int */
3628         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3629         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3630                                  EOP_TC_ACTION_EN |
3631                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3632                                  EVENT_INDEX(5)));
3633         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3634         radeon_ring_write(ring, addr & 0xfffffffc);
3635         radeon_ring_write(ring, upper_32_bits(addr));
3636         radeon_ring_write(ring, fence->seq);
3637         radeon_ring_write(ring, 0);
3638 }
3639
3640 /**
3641  * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3642  *
3643  * @rdev: radeon_device pointer
3644  * @ring: radeon ring buffer object
3645  * @semaphore: radeon semaphore object
3646  * @emit_wait: Is this a sempahore wait?
3647  *
3648  * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3649  * from running ahead of semaphore waits.
3650  */
3651 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3652                              struct radeon_ring *ring,
3653                              struct radeon_semaphore *semaphore,
3654                              bool emit_wait)
3655 {
3656         uint64_t addr = semaphore->gpu_addr;
3657         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3658
3659         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3660         radeon_ring_write(ring, lower_32_bits(addr));
3661         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3662
3663         if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3664                 /* Prevent the PFP from running ahead of the semaphore wait */
3665                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3666                 radeon_ring_write(ring, 0x0);
3667         }
3668
3669         return true;
3670 }
3671
3672 /**
3673  * cik_copy_cpdma - copy pages using the CP DMA engine
3674  *
3675  * @rdev: radeon_device pointer
3676  * @src_offset: src GPU address
3677  * @dst_offset: dst GPU address
3678  * @num_gpu_pages: number of GPU pages to xfer
3679  * @resv: reservation object to sync to
3680  *
3681  * Copy GPU paging using the CP DMA engine (CIK+).
3682  * Used by the radeon ttm implementation to move pages if
3683  * registered as the asic copy callback.
3684  */
3685 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3686                                     uint64_t src_offset, uint64_t dst_offset,
3687                                     unsigned num_gpu_pages,
3688                                     struct reservation_object *resv)
3689 {
3690         struct radeon_fence *fence;
3691         struct radeon_sync sync;
3692         int ring_index = rdev->asic->copy.blit_ring_index;
3693         struct radeon_ring *ring = &rdev->ring[ring_index];
3694         u32 size_in_bytes, cur_size_in_bytes, control;
3695         int i, num_loops;
3696         int r = 0;
3697
3698         radeon_sync_create(&sync);
3699
3700         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3701         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3702         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3703         if (r) {
3704                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3705                 radeon_sync_free(rdev, &sync, NULL);
3706                 return ERR_PTR(r);
3707         }
3708
3709         radeon_sync_resv(rdev, &sync, resv, false);
3710         radeon_sync_rings(rdev, &sync, ring->idx);
3711
3712         for (i = 0; i < num_loops; i++) {
3713                 cur_size_in_bytes = size_in_bytes;
3714                 if (cur_size_in_bytes > 0x1fffff)
3715                         cur_size_in_bytes = 0x1fffff;
3716                 size_in_bytes -= cur_size_in_bytes;
3717                 control = 0;
3718                 if (size_in_bytes == 0)
3719                         control |= PACKET3_DMA_DATA_CP_SYNC;
3720                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3721                 radeon_ring_write(ring, control);
3722                 radeon_ring_write(ring, lower_32_bits(src_offset));
3723                 radeon_ring_write(ring, upper_32_bits(src_offset));
3724                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3725                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3726                 radeon_ring_write(ring, cur_size_in_bytes);
3727                 src_offset += cur_size_in_bytes;
3728                 dst_offset += cur_size_in_bytes;
3729         }
3730
3731         r = radeon_fence_emit(rdev, &fence, ring->idx);
3732         if (r) {
3733                 radeon_ring_unlock_undo(rdev, ring);
3734                 radeon_sync_free(rdev, &sync, NULL);
3735                 return ERR_PTR(r);
3736         }
3737
3738         radeon_ring_unlock_commit(rdev, ring, false);
3739         radeon_sync_free(rdev, &sync, fence);
3740
3741         return fence;
3742 }
3743
3744 /*
3745  * IB stuff
3746  */
3747 /**
3748  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3749  *
3750  * @rdev: radeon_device pointer
3751  * @ib: radeon indirect buffer object
3752  *
3753  * Emits a DE (drawing engine) or CE (constant engine) IB
3754  * on the gfx ring.  IBs are usually generated by userspace
3755  * acceleration drivers and submitted to the kernel for
3756  * scheduling on the ring.  This function schedules the IB
3757  * on the gfx ring for execution by the GPU.
3758  */
3759 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3760 {
3761         struct radeon_ring *ring = &rdev->ring[ib->ring];
3762         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3763         u32 header, control = INDIRECT_BUFFER_VALID;
3764
3765         if (ib->is_const_ib) {
3766                 /* set switch buffer packet before const IB */
3767                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3768                 radeon_ring_write(ring, 0);
3769
3770                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3771         } else {
3772                 u32 next_rptr;
3773                 if (ring->rptr_save_reg) {
3774                         next_rptr = ring->wptr + 3 + 4;
3775                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3776                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3777                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3778                         radeon_ring_write(ring, next_rptr);
3779                 } else if (rdev->wb.enabled) {
3780                         next_rptr = ring->wptr + 5 + 4;
3781                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3782                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3783                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3784                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3785                         radeon_ring_write(ring, next_rptr);
3786                 }
3787
3788                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3789         }
3790
3791         control |= ib->length_dw | (vm_id << 24);
3792
3793         radeon_ring_write(ring, header);
3794         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
3795         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3796         radeon_ring_write(ring, control);
3797 }
3798
3799 /**
3800  * cik_ib_test - basic gfx ring IB test
3801  *
3802  * @rdev: radeon_device pointer
3803  * @ring: radeon_ring structure holding ring information
3804  *
3805  * Allocate an IB and execute it on the gfx ring (CIK).
3806  * Provides a basic gfx ring test to verify that IBs are working.
3807  * Returns 0 on success, error on failure.
3808  */
3809 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3810 {
3811         struct radeon_ib ib;
3812         uint32_t scratch;
3813         uint32_t tmp = 0;
3814         unsigned i;
3815         int r;
3816
3817         r = radeon_scratch_get(rdev, &scratch);
3818         if (r) {
3819                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3820                 return r;
3821         }
3822         WREG32(scratch, 0xCAFEDEAD);
3823         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3824         if (r) {
3825                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3826                 radeon_scratch_free(rdev, scratch);
3827                 return r;
3828         }
3829         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3830         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3831         ib.ptr[2] = 0xDEADBEEF;
3832         ib.length_dw = 3;
3833         r = radeon_ib_schedule(rdev, &ib, NULL, false);
3834         if (r) {
3835                 radeon_scratch_free(rdev, scratch);
3836                 radeon_ib_free(rdev, &ib);
3837                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3838                 return r;
3839         }
3840         r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3841                 RADEON_USEC_IB_TEST_TIMEOUT));
3842         if (r < 0) {
3843                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3844                 radeon_scratch_free(rdev, scratch);
3845                 radeon_ib_free(rdev, &ib);
3846                 return r;
3847         } else if (r == 0) {
3848                 DRM_ERROR("radeon: fence wait timed out.\n");
3849                 radeon_scratch_free(rdev, scratch);
3850                 radeon_ib_free(rdev, &ib);
3851                 return -ETIMEDOUT;
3852         }
3853         r = 0;
3854         for (i = 0; i < rdev->usec_timeout; i++) {
3855                 tmp = RREG32(scratch);
3856                 if (tmp == 0xDEADBEEF)
3857                         break;
3858                 DRM_UDELAY(1);
3859         }
3860         if (i < rdev->usec_timeout) {
3861                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3862         } else {
3863                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3864                           scratch, tmp);
3865                 r = -EINVAL;
3866         }
3867         radeon_scratch_free(rdev, scratch);
3868         radeon_ib_free(rdev, &ib);
3869         return r;
3870 }
3871
3872 /*
3873  * CP.
3874  * On CIK, gfx and compute now have independant command processors.
3875  *
3876  * GFX
3877  * Gfx consists of a single ring and can process both gfx jobs and
3878  * compute jobs.  The gfx CP consists of three microengines (ME):
3879  * PFP - Pre-Fetch Parser
3880  * ME - Micro Engine
3881  * CE - Constant Engine
3882  * The PFP and ME make up what is considered the Drawing Engine (DE).
3883  * The CE is an asynchronous engine used for updating buffer desciptors
3884  * used by the DE so that they can be loaded into cache in parallel
3885  * while the DE is processing state update packets.
3886  *
3887  * Compute
3888  * The compute CP consists of two microengines (ME):
3889  * MEC1 - Compute MicroEngine 1
3890  * MEC2 - Compute MicroEngine 2
3891  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3892  * The queues are exposed to userspace and are programmed directly
3893  * by the compute runtime.
3894  */
3895 /**
3896  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3897  *
3898  * @rdev: radeon_device pointer
3899  * @enable: enable or disable the MEs
3900  *
3901  * Halts or unhalts the gfx MEs.
3902  */
3903 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3904 {
3905         if (enable)
3906                 WREG32(CP_ME_CNTL, 0);
3907         else {
3908                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3909                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3910                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3911                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3912         }
3913         udelay(50);
3914 }
3915
3916 /**
3917  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3918  *
3919  * @rdev: radeon_device pointer
3920  *
3921  * Loads the gfx PFP, ME, and CE ucode.
3922  * Returns 0 for success, -EINVAL if the ucode is not available.
3923  */
3924 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3925 {
3926         int i;
3927
3928         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3929                 return -EINVAL;
3930
3931         cik_cp_gfx_enable(rdev, false);
3932
3933         if (rdev->new_fw) {
3934                 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3935                         (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3936                 const struct gfx_firmware_header_v1_0 *ce_hdr =
3937                         (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3938                 const struct gfx_firmware_header_v1_0 *me_hdr =
3939                         (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3940                 const __le32 *fw_data;
3941                 u32 fw_size;
3942
3943                 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3944                 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3945                 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3946
3947                 /* PFP */
3948                 fw_data = (const __le32 *)
3949                         (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3950                 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3951                 WREG32(CP_PFP_UCODE_ADDR, 0);
3952                 for (i = 0; i < fw_size; i++)
3953                         WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3954                 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
3955
3956                 /* CE */
3957                 fw_data = (const __le32 *)
3958                         (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3959                 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3960                 WREG32(CP_CE_UCODE_ADDR, 0);
3961                 for (i = 0; i < fw_size; i++)
3962                         WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3963                 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
3964
3965                 /* ME */
3966                 fw_data = (const __be32 *)
3967                         (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3968                 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3969                 WREG32(CP_ME_RAM_WADDR, 0);
3970                 for (i = 0; i < fw_size; i++)
3971                         WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3972                 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3973                 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
3974         } else {
3975                 const __be32 *fw_data;
3976
3977                 /* PFP */
3978                 fw_data = (const __be32 *)rdev->pfp_fw->data;
3979                 WREG32(CP_PFP_UCODE_ADDR, 0);
3980                 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3981                         WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3982                 WREG32(CP_PFP_UCODE_ADDR, 0);
3983
3984                 /* CE */
3985                 fw_data = (const __be32 *)rdev->ce_fw->data;
3986                 WREG32(CP_CE_UCODE_ADDR, 0);
3987                 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3988                         WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3989                 WREG32(CP_CE_UCODE_ADDR, 0);
3990
3991                 /* ME */
3992                 fw_data = (const __be32 *)rdev->me_fw->data;
3993                 WREG32(CP_ME_RAM_WADDR, 0);
3994                 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3995                         WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3996                 WREG32(CP_ME_RAM_WADDR, 0);
3997         }
3998
3999         return 0;
4000 }
4001
4002 /**
4003  * cik_cp_gfx_start - start the gfx ring
4004  *
4005  * @rdev: radeon_device pointer
4006  *
4007  * Enables the ring and loads the clear state context and other
4008  * packets required to init the ring.
4009  * Returns 0 for success, error for failure.
4010  */
4011 static int cik_cp_gfx_start(struct radeon_device *rdev)
4012 {
4013         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4014         int r, i;
4015
4016         /* init the CP */
4017         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4018         WREG32(CP_ENDIAN_SWAP, 0);
4019         WREG32(CP_DEVICE_ID, 1);
4020
4021         cik_cp_gfx_enable(rdev, true);
4022
4023         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4024         if (r) {
4025                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4026                 return r;
4027         }
4028
4029         /* init the CE partitions.  CE only used for gfx on CIK */
4030         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4031         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4032         radeon_ring_write(ring, 0x8000);
4033         radeon_ring_write(ring, 0x8000);
4034
4035         /* setup clear context state */
4036         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4037         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4038
4039         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4040         radeon_ring_write(ring, 0x80000000);
4041         radeon_ring_write(ring, 0x80000000);
4042
4043         for (i = 0; i < cik_default_size; i++)
4044                 radeon_ring_write(ring, cik_default_state[i]);
4045
4046         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4047         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4048
4049         /* set clear context state */
4050         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4051         radeon_ring_write(ring, 0);
4052
4053         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4054         radeon_ring_write(ring, 0x00000316);
4055         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4056         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4057
4058         radeon_ring_unlock_commit(rdev, ring, false);
4059
4060         return 0;
4061 }
4062
4063 /**
4064  * cik_cp_gfx_fini - stop the gfx ring
4065  *
4066  * @rdev: radeon_device pointer
4067  *
4068  * Stop the gfx ring and tear down the driver ring
4069  * info.
4070  */
4071 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4072 {
4073         cik_cp_gfx_enable(rdev, false);
4074         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4075 }
4076
4077 /**
4078  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4079  *
4080  * @rdev: radeon_device pointer
4081  *
4082  * Program the location and size of the gfx ring buffer
4083  * and test it to make sure it's working.
4084  * Returns 0 for success, error for failure.
4085  */
4086 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4087 {
4088         struct radeon_ring *ring;
4089         u32 tmp;
4090         u32 rb_bufsz;
4091         u64 rb_addr;
4092         int r;
4093
4094         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4095         if (rdev->family != CHIP_HAWAII)
4096                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4097
4098         /* Set the write pointer delay */
4099         WREG32(CP_RB_WPTR_DELAY, 0);
4100
4101         /* set the RB to use vmid 0 */
4102         WREG32(CP_RB_VMID, 0);
4103
4104         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4105
4106         /* ring 0 - compute and gfx */
4107         /* Set ring buffer size */
4108         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4109         rb_bufsz = order_base_2(ring->ring_size / 8);
4110         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4111 #ifdef __BIG_ENDIAN
4112         tmp |= BUF_SWAP_32BIT;
4113 #endif
4114         WREG32(CP_RB0_CNTL, tmp);
4115
4116         /* Initialize the ring buffer's read and write pointers */
4117         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4118         ring->wptr = 0;
4119         WREG32(CP_RB0_WPTR, ring->wptr);
4120
4121         /* set the wb address wether it's enabled or not */
4122         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4123         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4124
4125         /* scratch register shadowing is no longer supported */
4126         WREG32(SCRATCH_UMSK, 0);
4127
4128         if (!rdev->wb.enabled)
4129                 tmp |= RB_NO_UPDATE;
4130
4131         mdelay(1);
4132         WREG32(CP_RB0_CNTL, tmp);
4133
4134         rb_addr = ring->gpu_addr >> 8;
4135         WREG32(CP_RB0_BASE, rb_addr);
4136         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4137
4138         /* start the ring */
4139         cik_cp_gfx_start(rdev);
4140         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4141         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4142         if (r) {
4143                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4144                 return r;
4145         }
4146
4147         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4148                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4149
4150         return 0;
4151 }
4152
4153 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4154                      struct radeon_ring *ring)
4155 {
4156         u32 rptr;
4157
4158         if (rdev->wb.enabled)
4159                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4160         else
4161                 rptr = RREG32(CP_RB0_RPTR);
4162
4163         return rptr;
4164 }
4165
4166 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4167                      struct radeon_ring *ring)
4168 {
4169         return RREG32(CP_RB0_WPTR);
4170 }
4171
4172 void cik_gfx_set_wptr(struct radeon_device *rdev,
4173                       struct radeon_ring *ring)
4174 {
4175         WREG32(CP_RB0_WPTR, ring->wptr);
4176         (void)RREG32(CP_RB0_WPTR);
4177 }
4178
4179 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4180                          struct radeon_ring *ring)
4181 {
4182         u32 rptr;
4183
4184         if (rdev->wb.enabled) {
4185                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4186         } else {
4187                 mutex_lock(&rdev->srbm_mutex);
4188                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4189                 rptr = RREG32(CP_HQD_PQ_RPTR);
4190                 cik_srbm_select(rdev, 0, 0, 0, 0);
4191                 mutex_unlock(&rdev->srbm_mutex);
4192         }
4193
4194         return rptr;
4195 }
4196
4197 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4198                          struct radeon_ring *ring)
4199 {
4200         u32 wptr;
4201
4202         if (rdev->wb.enabled) {
4203                 /* XXX check if swapping is necessary on BE */
4204                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4205         } else {
4206                 mutex_lock(&rdev->srbm_mutex);
4207                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4208                 wptr = RREG32(CP_HQD_PQ_WPTR);
4209                 cik_srbm_select(rdev, 0, 0, 0, 0);
4210                 mutex_unlock(&rdev->srbm_mutex);
4211         }
4212
4213         return wptr;
4214 }
4215
4216 void cik_compute_set_wptr(struct radeon_device *rdev,
4217                           struct radeon_ring *ring)
4218 {
4219         /* XXX check if swapping is necessary on BE */
4220         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4221         WDOORBELL32(ring->doorbell_index, ring->wptr);
4222 }
4223
4224 static void cik_compute_stop(struct radeon_device *rdev,
4225                              struct radeon_ring *ring)
4226 {
4227         u32 j, tmp;
4228
4229         cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4230         /* Disable wptr polling. */
4231         tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4232         tmp &= ~WPTR_POLL_EN;
4233         WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4234         /* Disable HQD. */
4235         if (RREG32(CP_HQD_ACTIVE) & 1) {
4236                 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4237                 for (j = 0; j < rdev->usec_timeout; j++) {
4238                         if (!(RREG32(CP_HQD_ACTIVE) & 1))
4239                                 break;
4240                         udelay(1);
4241                 }
4242                 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4243                 WREG32(CP_HQD_PQ_RPTR, 0);
4244                 WREG32(CP_HQD_PQ_WPTR, 0);
4245         }
4246         cik_srbm_select(rdev, 0, 0, 0, 0);
4247 }
4248
4249 /**
4250  * cik_cp_compute_enable - enable/disable the compute CP MEs
4251  *
4252  * @rdev: radeon_device pointer
4253  * @enable: enable or disable the MEs
4254  *
4255  * Halts or unhalts the compute MEs.
4256  */
4257 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4258 {
4259         if (enable)
4260                 WREG32(CP_MEC_CNTL, 0);
4261         else {
4262                 /*
4263                  * To make hibernation reliable we need to clear compute ring
4264                  * configuration before halting the compute ring.
4265                  */
4266                 mutex_lock(&rdev->srbm_mutex);
4267                 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4268                 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4269                 mutex_unlock(&rdev->srbm_mutex);
4270
4271                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4272                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4273                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4274         }
4275         udelay(50);
4276 }
4277
4278 /**
4279  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4280  *
4281  * @rdev: radeon_device pointer
4282  *
4283  * Loads the compute MEC1&2 ucode.
4284  * Returns 0 for success, -EINVAL if the ucode is not available.
4285  */
4286 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4287 {
4288         int i;
4289
4290         if (!rdev->mec_fw)
4291                 return -EINVAL;
4292
4293         cik_cp_compute_enable(rdev, false);
4294
4295         if (rdev->new_fw) {
4296                 const struct gfx_firmware_header_v1_0 *mec_hdr =
4297                         (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4298                 const __le32 *fw_data;
4299                 u32 fw_size;
4300
4301                 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4302
4303                 /* MEC1 */
4304                 fw_data = (const __le32 *)
4305                         (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4306                 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4307                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4308                 for (i = 0; i < fw_size; i++)
4309                         WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4310                 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
4311
4312                 /* MEC2 */
4313                 if (rdev->family == CHIP_KAVERI) {
4314                         const struct gfx_firmware_header_v1_0 *mec2_hdr =
4315                                 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4316
4317                         fw_data = (const __le32 *)
4318                                 (rdev->mec2_fw->data +
4319                                  le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4320                         fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4321                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4322                         for (i = 0; i < fw_size; i++)
4323                                 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4324                         WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
4325                 }
4326         } else {
4327                 const __be32 *fw_data;
4328
4329                 /* MEC1 */
4330                 fw_data = (const __be32 *)rdev->mec_fw->data;
4331                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4332                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4333                         WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4334                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4335
4336                 if (rdev->family == CHIP_KAVERI) {
4337                         /* MEC2 */
4338                         fw_data = (const __be32 *)rdev->mec_fw->data;
4339                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4340                         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4341                                 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4342                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4343                 }
4344         }
4345
4346         return 0;
4347 }
4348
4349 /**
4350  * cik_cp_compute_start - start the compute queues
4351  *
4352  * @rdev: radeon_device pointer
4353  *
4354  * Enable the compute queues.
4355  * Returns 0 for success, error for failure.
4356  */
4357 static int cik_cp_compute_start(struct radeon_device *rdev)
4358 {
4359         cik_cp_compute_enable(rdev, true);
4360
4361         return 0;
4362 }
4363
4364 /**
4365  * cik_cp_compute_fini - stop the compute queues
4366  *
4367  * @rdev: radeon_device pointer
4368  *
4369  * Stop the compute queues and tear down the driver queue
4370  * info.
4371  */
4372 static void cik_cp_compute_fini(struct radeon_device *rdev)
4373 {
4374         int i, idx, r;
4375
4376         cik_cp_compute_enable(rdev, false);
4377
4378         for (i = 0; i < 2; i++) {
4379                 if (i == 0)
4380                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4381                 else
4382                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4383
4384                 if (rdev->ring[idx].mqd_obj) {
4385                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4386                         if (unlikely(r != 0))
4387                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4388
4389                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4390                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4391
4392                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4393                         rdev->ring[idx].mqd_obj = NULL;
4394                 }
4395         }
4396 }
4397
4398 static void cik_mec_fini(struct radeon_device *rdev)
4399 {
4400         int r;
4401
4402         if (rdev->mec.hpd_eop_obj) {
4403                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4404                 if (unlikely(r != 0))
4405                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4406                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4407                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4408
4409                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4410                 rdev->mec.hpd_eop_obj = NULL;
4411         }
4412 }
4413
4414 #define MEC_HPD_SIZE 2048
4415
4416 static int cik_mec_init(struct radeon_device *rdev)
4417 {
4418         int r;
4419         u32 *hpd;
4420
4421         /*
4422          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4423          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4424          * Nonetheless, we assign only 1 pipe because all other pipes will
4425          * be handled by KFD
4426          */
4427         rdev->mec.num_mec = 1;
4428         rdev->mec.num_pipe = 1;
4429         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4430
4431         if (rdev->mec.hpd_eop_obj == NULL) {
4432                 r = radeon_bo_create(rdev,
4433                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4434                                      PAGE_SIZE, true,
4435                                      RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
4436                                      &rdev->mec.hpd_eop_obj);
4437                 if (r) {
4438                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4439                         return r;
4440                 }
4441         }
4442
4443         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4444         if (unlikely(r != 0)) {
4445                 cik_mec_fini(rdev);
4446                 return r;
4447         }
4448         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4449                           &rdev->mec.hpd_eop_gpu_addr);
4450         if (r) {
4451                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4452                 cik_mec_fini(rdev);
4453                 return r;
4454         }
4455         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4456         if (r) {
4457                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4458                 cik_mec_fini(rdev);
4459                 return r;
4460         }
4461
4462         /* clear memory.  Not sure if this is required or not */
4463         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4464
4465         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4466         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4467
4468         return 0;
4469 }
4470
4471 struct hqd_registers
4472 {
4473         u32 cp_mqd_base_addr;
4474         u32 cp_mqd_base_addr_hi;
4475         u32 cp_hqd_active;
4476         u32 cp_hqd_vmid;
4477         u32 cp_hqd_persistent_state;
4478         u32 cp_hqd_pipe_priority;
4479         u32 cp_hqd_queue_priority;
4480         u32 cp_hqd_quantum;
4481         u32 cp_hqd_pq_base;
4482         u32 cp_hqd_pq_base_hi;
4483         u32 cp_hqd_pq_rptr;
4484         u32 cp_hqd_pq_rptr_report_addr;
4485         u32 cp_hqd_pq_rptr_report_addr_hi;
4486         u32 cp_hqd_pq_wptr_poll_addr;
4487         u32 cp_hqd_pq_wptr_poll_addr_hi;
4488         u32 cp_hqd_pq_doorbell_control;
4489         u32 cp_hqd_pq_wptr;
4490         u32 cp_hqd_pq_control;
4491         u32 cp_hqd_ib_base_addr;
4492         u32 cp_hqd_ib_base_addr_hi;
4493         u32 cp_hqd_ib_rptr;
4494         u32 cp_hqd_ib_control;
4495         u32 cp_hqd_iq_timer;
4496         u32 cp_hqd_iq_rptr;
4497         u32 cp_hqd_dequeue_request;
4498         u32 cp_hqd_dma_offload;
4499         u32 cp_hqd_sema_cmd;
4500         u32 cp_hqd_msg_type;
4501         u32 cp_hqd_atomic0_preop_lo;
4502         u32 cp_hqd_atomic0_preop_hi;
4503         u32 cp_hqd_atomic1_preop_lo;
4504         u32 cp_hqd_atomic1_preop_hi;
4505         u32 cp_hqd_hq_scheduler0;
4506         u32 cp_hqd_hq_scheduler1;
4507         u32 cp_mqd_control;
4508 };
4509
4510 struct bonaire_mqd
4511 {
4512         u32 header;
4513         u32 dispatch_initiator;
4514         u32 dimensions[3];
4515         u32 start_idx[3];
4516         u32 num_threads[3];
4517         u32 pipeline_stat_enable;
4518         u32 perf_counter_enable;
4519         u32 pgm[2];
4520         u32 tba[2];
4521         u32 tma[2];
4522         u32 pgm_rsrc[2];
4523         u32 vmid;
4524         u32 resource_limits;
4525         u32 static_thread_mgmt01[2];
4526         u32 tmp_ring_size;
4527         u32 static_thread_mgmt23[2];
4528         u32 restart[3];
4529         u32 thread_trace_enable;
4530         u32 reserved1;
4531         u32 user_data[16];
4532         u32 vgtcs_invoke_count[2];
4533         struct hqd_registers queue_state;
4534         u32 dequeue_cntr;
4535         u32 interrupt_queue[64];
4536 };
4537
4538 /**
4539  * cik_cp_compute_resume - setup the compute queue registers
4540  *
4541  * @rdev: radeon_device pointer
4542  *
4543  * Program the compute queues and test them to make sure they
4544  * are working.
4545  * Returns 0 for success, error for failure.
4546  */
4547 static int cik_cp_compute_resume(struct radeon_device *rdev)
4548 {
4549         int r, i, j, idx;
4550         u32 tmp;
4551         bool use_doorbell = true;
4552         u64 hqd_gpu_addr;
4553         u64 mqd_gpu_addr;
4554         u64 eop_gpu_addr;
4555         u64 wb_gpu_addr;
4556         u32 *buf;
4557         struct bonaire_mqd *mqd;
4558
4559         r = cik_cp_compute_start(rdev);
4560         if (r)
4561                 return r;
4562
4563         /* fix up chicken bits */
4564         tmp = RREG32(CP_CPF_DEBUG);
4565         tmp |= (1 << 23);
4566         WREG32(CP_CPF_DEBUG, tmp);
4567
4568         /* init the pipes */
4569         mutex_lock(&rdev->srbm_mutex);
4570
4571         for (i = 0; i < rdev->mec.num_pipe; ++i) {
4572                 cik_srbm_select(rdev, 0, i, 0, 0);
4573
4574                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4575                 /* write the EOP addr */
4576                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4577                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4578
4579                 /* set the VMID assigned */
4580                 WREG32(CP_HPD_EOP_VMID, 0);
4581
4582                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4583                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4584                 tmp &= ~EOP_SIZE_MASK;
4585                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4586                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4587
4588         }
4589         mutex_unlock(&rdev->srbm_mutex);
4590
4591         /* init the queues.  Just two for now. */
4592         for (i = 0; i < 2; i++) {
4593                 if (i == 0)
4594                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4595                 else
4596                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4597
4598                 if (rdev->ring[idx].mqd_obj == NULL) {
4599                         r = radeon_bo_create(rdev,
4600                                              sizeof(struct bonaire_mqd),
4601                                              PAGE_SIZE, true,
4602                                              RADEON_GEM_DOMAIN_GTT, 0, NULL,
4603                                              NULL, &rdev->ring[idx].mqd_obj);
4604                         if (r) {
4605                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4606                                 return r;
4607                         }
4608                 }
4609
4610                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4611                 if (unlikely(r != 0)) {
4612                         cik_cp_compute_fini(rdev);
4613                         return r;
4614                 }
4615                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4616                                   &mqd_gpu_addr);
4617                 if (r) {
4618                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4619                         cik_cp_compute_fini(rdev);
4620                         return r;
4621                 }
4622                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4623                 if (r) {
4624                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4625                         cik_cp_compute_fini(rdev);
4626                         return r;
4627                 }
4628
4629                 /* init the mqd struct */
4630                 memset(buf, 0, sizeof(struct bonaire_mqd));
4631
4632                 mqd = (struct bonaire_mqd *)buf;
4633                 mqd->header = 0xC0310800;
4634                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4635                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4636                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4637                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4638
4639                 mutex_lock(&rdev->srbm_mutex);
4640                 cik_srbm_select(rdev, rdev->ring[idx].me,
4641                                 rdev->ring[idx].pipe,
4642                                 rdev->ring[idx].queue, 0);
4643
4644                 /* disable wptr polling */
4645                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4646                 tmp &= ~WPTR_POLL_EN;
4647                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4648
4649                 /* enable doorbell? */
4650                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4651                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4652                 if (use_doorbell)
4653                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4654                 else
4655                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4656                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4657                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4658
4659                 /* disable the queue if it's active */
4660                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4661                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4662                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4663                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4664                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4665                         for (j = 0; j < rdev->usec_timeout; j++) {
4666                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4667                                         break;
4668                                 udelay(1);
4669                         }
4670                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4671                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4672                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4673                 }
4674
4675                 /* set the pointer to the MQD */
4676                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4677                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4678                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4679                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4680                 /* set MQD vmid to 0 */
4681                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4682                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4683                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4684
4685                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4686                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4687                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4688                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4689                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4690                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4691
4692                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4693                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4694                 mqd->queue_state.cp_hqd_pq_control &=
4695                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4696
4697                 mqd->queue_state.cp_hqd_pq_control |=
4698                         order_base_2(rdev->ring[idx].ring_size / 8);
4699                 mqd->queue_state.cp_hqd_pq_control |=
4700                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4701 #ifdef __BIG_ENDIAN
4702                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4703 #endif
4704                 mqd->queue_state.cp_hqd_pq_control &=
4705                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4706                 mqd->queue_state.cp_hqd_pq_control |=
4707                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4708                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4709
4710                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4711                 if (i == 0)
4712                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4713                 else
4714                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4715                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4716                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4717                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4718                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4719                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4720
4721                 /* set the wb address wether it's enabled or not */
4722                 if (i == 0)
4723                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4724                 else
4725                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4726                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4727                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4728                         upper_32_bits(wb_gpu_addr) & 0xffff;
4729                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4730                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4731                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4732                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4733
4734                 /* enable the doorbell if requested */
4735                 if (use_doorbell) {
4736                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4737                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4738                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4739                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4740                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4741                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4742                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4743                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4744
4745                 } else {
4746                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4747                 }
4748                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4749                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4750
4751                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4752                 rdev->ring[idx].wptr = 0;
4753                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4754                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4755                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4756
4757                 /* set the vmid for the queue */
4758                 mqd->queue_state.cp_hqd_vmid = 0;
4759                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4760
4761                 /* activate the queue */
4762                 mqd->queue_state.cp_hqd_active = 1;
4763                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4764
4765                 cik_srbm_select(rdev, 0, 0, 0, 0);
4766                 mutex_unlock(&rdev->srbm_mutex);
4767
4768                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4769                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4770
4771                 rdev->ring[idx].ready = true;
4772                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4773                 if (r)
4774                         rdev->ring[idx].ready = false;
4775         }
4776
4777         return 0;
4778 }
4779
4780 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4781 {
4782         cik_cp_gfx_enable(rdev, enable);
4783         cik_cp_compute_enable(rdev, enable);
4784 }
4785
4786 static int cik_cp_load_microcode(struct radeon_device *rdev)
4787 {
4788         int r;
4789
4790         r = cik_cp_gfx_load_microcode(rdev);
4791         if (r)
4792                 return r;
4793         r = cik_cp_compute_load_microcode(rdev);
4794         if (r)
4795                 return r;
4796
4797         return 0;
4798 }
4799
4800 static void cik_cp_fini(struct radeon_device *rdev)
4801 {
4802         cik_cp_gfx_fini(rdev);
4803         cik_cp_compute_fini(rdev);
4804 }
4805
4806 static int cik_cp_resume(struct radeon_device *rdev)
4807 {
4808         int r;
4809
4810         cik_enable_gui_idle_interrupt(rdev, false);
4811
4812         r = cik_cp_load_microcode(rdev);
4813         if (r)
4814                 return r;
4815
4816         r = cik_cp_gfx_resume(rdev);
4817         if (r)
4818                 return r;
4819         r = cik_cp_compute_resume(rdev);
4820         if (r)
4821                 return r;
4822
4823         cik_enable_gui_idle_interrupt(rdev, true);
4824
4825         return 0;
4826 }
4827
4828 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4829 {
4830         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4831                 RREG32(GRBM_STATUS));
4832         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4833                 RREG32(GRBM_STATUS2));
4834         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4835                 RREG32(GRBM_STATUS_SE0));
4836         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4837                 RREG32(GRBM_STATUS_SE1));
4838         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4839                 RREG32(GRBM_STATUS_SE2));
4840         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4841                 RREG32(GRBM_STATUS_SE3));
4842         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4843                 RREG32(SRBM_STATUS));
4844         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4845                 RREG32(SRBM_STATUS2));
4846         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4847                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4848         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4849                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4850         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4851         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4852                  RREG32(CP_STALLED_STAT1));
4853         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4854                  RREG32(CP_STALLED_STAT2));
4855         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4856                  RREG32(CP_STALLED_STAT3));
4857         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4858                  RREG32(CP_CPF_BUSY_STAT));
4859         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4860                  RREG32(CP_CPF_STALLED_STAT1));
4861         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4862         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4863         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4864                  RREG32(CP_CPC_STALLED_STAT1));
4865         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4866 }
4867
4868 /**
4869  * cik_gpu_check_soft_reset - check which blocks are busy
4870  *
4871  * @rdev: radeon_device pointer
4872  *
4873  * Check which blocks are busy and return the relevant reset
4874  * mask to be used by cik_gpu_soft_reset().
4875  * Returns a mask of the blocks to be reset.
4876  */
4877 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4878 {
4879         u32 reset_mask = 0;
4880         u32 tmp;
4881
4882         /* GRBM_STATUS */
4883         tmp = RREG32(GRBM_STATUS);
4884         if (tmp & (PA_BUSY | SC_BUSY |
4885                    BCI_BUSY | SX_BUSY |
4886                    TA_BUSY | VGT_BUSY |
4887                    DB_BUSY | CB_BUSY |
4888                    GDS_BUSY | SPI_BUSY |
4889                    IA_BUSY | IA_BUSY_NO_DMA))
4890                 reset_mask |= RADEON_RESET_GFX;
4891
4892         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4893                 reset_mask |= RADEON_RESET_CP;
4894
4895         /* GRBM_STATUS2 */
4896         tmp = RREG32(GRBM_STATUS2);
4897         if (tmp & RLC_BUSY)
4898                 reset_mask |= RADEON_RESET_RLC;
4899
4900         /* SDMA0_STATUS_REG */
4901         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4902         if (!(tmp & SDMA_IDLE))
4903                 reset_mask |= RADEON_RESET_DMA;
4904
4905         /* SDMA1_STATUS_REG */
4906         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4907         if (!(tmp & SDMA_IDLE))
4908                 reset_mask |= RADEON_RESET_DMA1;
4909
4910         /* SRBM_STATUS2 */
4911         tmp = RREG32(SRBM_STATUS2);
4912         if (tmp & SDMA_BUSY)
4913                 reset_mask |= RADEON_RESET_DMA;
4914
4915         if (tmp & SDMA1_BUSY)
4916                 reset_mask |= RADEON_RESET_DMA1;
4917
4918         /* SRBM_STATUS */
4919         tmp = RREG32(SRBM_STATUS);
4920
4921         if (tmp & IH_BUSY)
4922                 reset_mask |= RADEON_RESET_IH;
4923
4924         if (tmp & SEM_BUSY)
4925                 reset_mask |= RADEON_RESET_SEM;
4926
4927         if (tmp & GRBM_RQ_PENDING)
4928                 reset_mask |= RADEON_RESET_GRBM;
4929
4930         if (tmp & VMC_BUSY)
4931                 reset_mask |= RADEON_RESET_VMC;
4932
4933         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4934                    MCC_BUSY | MCD_BUSY))
4935                 reset_mask |= RADEON_RESET_MC;
4936
4937         if (evergreen_is_display_hung(rdev))
4938                 reset_mask |= RADEON_RESET_DISPLAY;
4939
4940         /* Skip MC reset as it's mostly likely not hung, just busy */
4941         if (reset_mask & RADEON_RESET_MC) {
4942                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4943                 reset_mask &= ~RADEON_RESET_MC;
4944         }
4945
4946         return reset_mask;
4947 }
4948
4949 /**
4950  * cik_gpu_soft_reset - soft reset GPU
4951  *
4952  * @rdev: radeon_device pointer
4953  * @reset_mask: mask of which blocks to reset
4954  *
4955  * Soft reset the blocks specified in @reset_mask.
4956  */
4957 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4958 {
4959         struct evergreen_mc_save save;
4960         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4961         u32 tmp;
4962
4963         if (reset_mask == 0)
4964                 return;
4965
4966         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4967
4968         cik_print_gpu_status_regs(rdev);
4969         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4970                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4971         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4972                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4973
4974         /* disable CG/PG */
4975         cik_fini_pg(rdev);
4976         cik_fini_cg(rdev);
4977
4978         /* stop the rlc */
4979         cik_rlc_stop(rdev);
4980
4981         /* Disable GFX parsing/prefetching */
4982         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4983
4984         /* Disable MEC parsing/prefetching */
4985         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4986
4987         if (reset_mask & RADEON_RESET_DMA) {
4988                 /* sdma0 */
4989                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4990                 tmp |= SDMA_HALT;
4991                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4992         }
4993         if (reset_mask & RADEON_RESET_DMA1) {
4994                 /* sdma1 */
4995                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4996                 tmp |= SDMA_HALT;
4997                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4998         }
4999
5000         evergreen_mc_stop(rdev, &save);
5001         if (evergreen_mc_wait_for_idle(rdev)) {
5002                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5003         }
5004
5005         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5006                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5007
5008         if (reset_mask & RADEON_RESET_CP) {
5009                 grbm_soft_reset |= SOFT_RESET_CP;
5010
5011                 srbm_soft_reset |= SOFT_RESET_GRBM;
5012         }
5013
5014         if (reset_mask & RADEON_RESET_DMA)
5015                 srbm_soft_reset |= SOFT_RESET_SDMA;
5016
5017         if (reset_mask & RADEON_RESET_DMA1)
5018                 srbm_soft_reset |= SOFT_RESET_SDMA1;
5019
5020         if (reset_mask & RADEON_RESET_DISPLAY)
5021                 srbm_soft_reset |= SOFT_RESET_DC;
5022
5023         if (reset_mask & RADEON_RESET_RLC)
5024                 grbm_soft_reset |= SOFT_RESET_RLC;
5025
5026         if (reset_mask & RADEON_RESET_SEM)
5027                 srbm_soft_reset |= SOFT_RESET_SEM;
5028
5029         if (reset_mask & RADEON_RESET_IH)
5030                 srbm_soft_reset |= SOFT_RESET_IH;
5031
5032         if (reset_mask & RADEON_RESET_GRBM)
5033                 srbm_soft_reset |= SOFT_RESET_GRBM;
5034
5035         if (reset_mask & RADEON_RESET_VMC)
5036                 srbm_soft_reset |= SOFT_RESET_VMC;
5037
5038         if (!(rdev->flags & RADEON_IS_IGP)) {
5039                 if (reset_mask & RADEON_RESET_MC)
5040                         srbm_soft_reset |= SOFT_RESET_MC;
5041         }
5042
5043         if (grbm_soft_reset) {
5044                 tmp = RREG32(GRBM_SOFT_RESET);
5045                 tmp |= grbm_soft_reset;
5046                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5047                 WREG32(GRBM_SOFT_RESET, tmp);
5048                 tmp = RREG32(GRBM_SOFT_RESET);
5049
5050                 udelay(50);
5051
5052                 tmp &= ~grbm_soft_reset;
5053                 WREG32(GRBM_SOFT_RESET, tmp);
5054                 tmp = RREG32(GRBM_SOFT_RESET);
5055         }
5056
5057         if (srbm_soft_reset) {
5058                 tmp = RREG32(SRBM_SOFT_RESET);
5059                 tmp |= srbm_soft_reset;
5060                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5061                 WREG32(SRBM_SOFT_RESET, tmp);
5062                 tmp = RREG32(SRBM_SOFT_RESET);
5063
5064                 udelay(50);
5065
5066                 tmp &= ~srbm_soft_reset;
5067                 WREG32(SRBM_SOFT_RESET, tmp);
5068                 tmp = RREG32(SRBM_SOFT_RESET);
5069         }
5070
5071         /* Wait a little for things to settle down */
5072         udelay(50);
5073
5074         evergreen_mc_resume(rdev, &save);
5075         udelay(50);
5076
5077         cik_print_gpu_status_regs(rdev);
5078 }
5079
5080 struct kv_reset_save_regs {
5081         u32 gmcon_reng_execute;
5082         u32 gmcon_misc;
5083         u32 gmcon_misc3;
5084 };
5085
5086 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5087                                    struct kv_reset_save_regs *save)
5088 {
5089         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5090         save->gmcon_misc = RREG32(GMCON_MISC);
5091         save->gmcon_misc3 = RREG32(GMCON_MISC3);
5092
5093         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5094         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5095                                                 STCTRL_STUTTER_EN));
5096 }
5097
5098 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5099                                       struct kv_reset_save_regs *save)
5100 {
5101         int i;
5102
5103         WREG32(GMCON_PGFSM_WRITE, 0);
5104         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5105
5106         for (i = 0; i < 5; i++)
5107                 WREG32(GMCON_PGFSM_WRITE, 0);
5108
5109         WREG32(GMCON_PGFSM_WRITE, 0);
5110         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5111
5112         for (i = 0; i < 5; i++)
5113                 WREG32(GMCON_PGFSM_WRITE, 0);
5114
5115         WREG32(GMCON_PGFSM_WRITE, 0x210000);
5116         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5117
5118         for (i = 0; i < 5; i++)
5119                 WREG32(GMCON_PGFSM_WRITE, 0);
5120
5121         WREG32(GMCON_PGFSM_WRITE, 0x21003);
5122         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5123
5124         for (i = 0; i < 5; i++)
5125                 WREG32(GMCON_PGFSM_WRITE, 0);
5126
5127         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5128         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5129
5130         for (i = 0; i < 5; i++)
5131                 WREG32(GMCON_PGFSM_WRITE, 0);
5132
5133         WREG32(GMCON_PGFSM_WRITE, 0);
5134         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5135
5136         for (i = 0; i < 5; i++)
5137                 WREG32(GMCON_PGFSM_WRITE, 0);
5138
5139         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5140         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5141
5142         for (i = 0; i < 5; i++)
5143                 WREG32(GMCON_PGFSM_WRITE, 0);
5144
5145         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5146         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5147
5148         for (i = 0; i < 5; i++)
5149                 WREG32(GMCON_PGFSM_WRITE, 0);
5150
5151         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5152         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5153
5154         for (i = 0; i < 5; i++)
5155                 WREG32(GMCON_PGFSM_WRITE, 0);
5156
5157         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5158         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5159
5160         for (i = 0; i < 5; i++)
5161                 WREG32(GMCON_PGFSM_WRITE, 0);
5162
5163         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5164         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5165
5166         WREG32(GMCON_MISC3, save->gmcon_misc3);
5167         WREG32(GMCON_MISC, save->gmcon_misc);
5168         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5169 }
5170
5171 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5172 {
5173         struct evergreen_mc_save save;
5174         struct kv_reset_save_regs kv_save = { 0 };
5175         u32 tmp, i;
5176
5177         dev_info(rdev->dev, "GPU pci config reset\n");
5178
5179         /* disable dpm? */
5180
5181         /* disable cg/pg */
5182         cik_fini_pg(rdev);
5183         cik_fini_cg(rdev);
5184
5185         /* Disable GFX parsing/prefetching */
5186         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5187
5188         /* Disable MEC parsing/prefetching */
5189         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5190
5191         /* sdma0 */
5192         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5193         tmp |= SDMA_HALT;
5194         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5195         /* sdma1 */
5196         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5197         tmp |= SDMA_HALT;
5198         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5199         /* XXX other engines? */
5200
5201         /* halt the rlc, disable cp internal ints */
5202         cik_rlc_stop(rdev);
5203
5204         udelay(50);
5205
5206         /* disable mem access */
5207         evergreen_mc_stop(rdev, &save);
5208         if (evergreen_mc_wait_for_idle(rdev)) {
5209                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5210         }
5211
5212         if (rdev->flags & RADEON_IS_IGP)
5213                 kv_save_regs_for_reset(rdev, &kv_save);
5214
5215         /* disable BM */
5216         pci_clear_master(rdev->pdev);
5217         /* reset */
5218         radeon_pci_config_reset(rdev);
5219
5220         udelay(100);
5221
5222         /* wait for asic to come out of reset */
5223         for (i = 0; i < rdev->usec_timeout; i++) {
5224                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5225                         break;
5226                 udelay(1);
5227         }
5228
5229         /* does asic init need to be run first??? */
5230         if (rdev->flags & RADEON_IS_IGP)
5231                 kv_restore_regs_for_reset(rdev, &kv_save);
5232 }
5233
5234 /**
5235  * cik_asic_reset - soft reset GPU
5236  *
5237  * @rdev: radeon_device pointer
5238  * @hard: force hard reset
5239  *
5240  * Look up which blocks are hung and attempt
5241  * to reset them.
5242  * Returns 0 for success.
5243  */
5244 int cik_asic_reset(struct radeon_device *rdev, bool hard)
5245 {
5246         u32 reset_mask;
5247
5248         if (hard) {
5249                 cik_gpu_pci_config_reset(rdev);
5250                 return 0;
5251         }
5252
5253         reset_mask = cik_gpu_check_soft_reset(rdev);
5254
5255         if (reset_mask)
5256                 r600_set_bios_scratch_engine_hung(rdev, true);
5257
5258         /* try soft reset */
5259         cik_gpu_soft_reset(rdev, reset_mask);
5260
5261         reset_mask = cik_gpu_check_soft_reset(rdev);
5262
5263         /* try pci config reset */
5264         if (reset_mask && radeon_hard_reset)
5265                 cik_gpu_pci_config_reset(rdev);
5266
5267         reset_mask = cik_gpu_check_soft_reset(rdev);
5268
5269         if (!reset_mask)
5270                 r600_set_bios_scratch_engine_hung(rdev, false);
5271
5272         return 0;
5273 }
5274
5275 /**
5276  * cik_gfx_is_lockup - check if the 3D engine is locked up
5277  *
5278  * @rdev: radeon_device pointer
5279  * @ring: radeon_ring structure holding ring information
5280  *
5281  * Check if the 3D engine is locked up (CIK).
5282  * Returns true if the engine is locked, false if not.
5283  */
5284 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5285 {
5286         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5287
5288         if (!(reset_mask & (RADEON_RESET_GFX |
5289                             RADEON_RESET_COMPUTE |
5290                             RADEON_RESET_CP))) {
5291                 radeon_ring_lockup_update(rdev, ring);
5292                 return false;
5293         }
5294         return radeon_ring_test_lockup(rdev, ring);
5295 }
5296
5297 /* MC */
5298 /**
5299  * cik_mc_program - program the GPU memory controller
5300  *
5301  * @rdev: radeon_device pointer
5302  *
5303  * Set the location of vram, gart, and AGP in the GPU's
5304  * physical address space (CIK).
5305  */
5306 static void cik_mc_program(struct radeon_device *rdev)
5307 {
5308         struct evergreen_mc_save save;
5309         u32 tmp;
5310         int i, j;
5311
5312         /* Initialize HDP */
5313         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5314                 WREG32((0x2c14 + j), 0x00000000);
5315                 WREG32((0x2c18 + j), 0x00000000);
5316                 WREG32((0x2c1c + j), 0x00000000);
5317                 WREG32((0x2c20 + j), 0x00000000);
5318                 WREG32((0x2c24 + j), 0x00000000);
5319         }
5320         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5321
5322         evergreen_mc_stop(rdev, &save);
5323         if (radeon_mc_wait_for_idle(rdev)) {
5324                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5325         }
5326         /* Lockout access through VGA aperture*/
5327         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5328         /* Update configuration */
5329         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5330                rdev->mc.vram_start >> 12);
5331         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5332                rdev->mc.vram_end >> 12);
5333         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5334                rdev->vram_scratch.gpu_addr >> 12);
5335         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5336         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5337         WREG32(MC_VM_FB_LOCATION, tmp);
5338         /* XXX double check these! */
5339         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5340         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5341         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5342         WREG32(MC_VM_AGP_BASE, 0);
5343         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5344         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5345         if (radeon_mc_wait_for_idle(rdev)) {
5346                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5347         }
5348         evergreen_mc_resume(rdev, &save);
5349         /* we need to own VRAM, so turn off the VGA renderer here
5350          * to stop it overwriting our objects */
5351         rv515_vga_render_disable(rdev);
5352 }
5353
5354 /**
5355  * cik_mc_init - initialize the memory controller driver params
5356  *
5357  * @rdev: radeon_device pointer
5358  *
5359  * Look up the amount of vram, vram width, and decide how to place
5360  * vram and gart within the GPU's physical address space (CIK).
5361  * Returns 0 for success.
5362  */
5363 static int cik_mc_init(struct radeon_device *rdev)
5364 {
5365         u32 tmp;
5366         int chansize, numchan;
5367
5368         /* Get VRAM informations */
5369         rdev->mc.vram_is_ddr = true;
5370         tmp = RREG32(MC_ARB_RAMCFG);
5371         if (tmp & CHANSIZE_MASK) {
5372                 chansize = 64;
5373         } else {
5374                 chansize = 32;
5375         }
5376         tmp = RREG32(MC_SHARED_CHMAP);
5377         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5378         case 0:
5379         default:
5380                 numchan = 1;
5381                 break;
5382         case 1:
5383                 numchan = 2;
5384                 break;
5385         case 2:
5386                 numchan = 4;
5387                 break;
5388         case 3:
5389                 numchan = 8;
5390                 break;
5391         case 4:
5392                 numchan = 3;
5393                 break;
5394         case 5:
5395                 numchan = 6;
5396                 break;
5397         case 6:
5398                 numchan = 10;
5399                 break;
5400         case 7:
5401                 numchan = 12;
5402                 break;
5403         case 8:
5404                 numchan = 16;
5405                 break;
5406         }
5407         rdev->mc.vram_width = numchan * chansize;
5408         /* Could aper size report 0 ? */
5409         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5410         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5411         /* size in MB on si */
5412         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5413         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5414         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5415         si_vram_gtt_location(rdev, &rdev->mc);
5416         radeon_update_bandwidth_info(rdev);
5417
5418         return 0;
5419 }
5420
5421 /*
5422  * GART
5423  * VMID 0 is the physical GPU addresses as used by the kernel.
5424  * VMIDs 1-15 are used for userspace clients and are handled
5425  * by the radeon vm/hsa code.
5426  */
5427 /**
5428  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5429  *
5430  * @rdev: radeon_device pointer
5431  *
5432  * Flush the TLB for the VMID 0 page table (CIK).
5433  */
5434 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5435 {
5436         /* flush hdp cache */
5437         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5438
5439         /* bits 0-15 are the VM contexts0-15 */
5440         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5441 }
5442
5443 /**
5444  * cik_pcie_gart_enable - gart enable
5445  *
5446  * @rdev: radeon_device pointer
5447  *
5448  * This sets up the TLBs, programs the page tables for VMID0,
5449  * sets up the hw for VMIDs 1-15 which are allocated on
5450  * demand, and sets up the global locations for the LDS, GDS,
5451  * and GPUVM for FSA64 clients (CIK).
5452  * Returns 0 for success, errors for failure.
5453  */
5454 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5455 {
5456         int r, i;
5457
5458         if (rdev->gart.robj == NULL) {
5459                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5460                 return -EINVAL;
5461         }
5462         r = radeon_gart_table_vram_pin(rdev);
5463         if (r)
5464                 return r;
5465         /* Setup TLB control */
5466         WREG32(MC_VM_MX_L1_TLB_CNTL,
5467                (0xA << 7) |
5468                ENABLE_L1_TLB |
5469                ENABLE_L1_FRAGMENT_PROCESSING |
5470                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5471                ENABLE_ADVANCED_DRIVER_MODEL |
5472                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5473         /* Setup L2 cache */
5474         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5475                ENABLE_L2_FRAGMENT_PROCESSING |
5476                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5477                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5478                EFFECTIVE_L2_QUEUE_SIZE(7) |
5479                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5480         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5481         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5482                BANK_SELECT(4) |
5483                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5484         /* setup context0 */
5485         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5486         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5487         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5488         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5489                         (u32)(rdev->dummy_page.addr >> 12));
5490         WREG32(VM_CONTEXT0_CNTL2, 0);
5491         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5492                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5493
5494         WREG32(0x15D4, 0);
5495         WREG32(0x15D8, 0);
5496         WREG32(0x15DC, 0);
5497
5498         /* restore context1-15 */
5499         /* set vm size, must be a multiple of 4 */
5500         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5501         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
5502         for (i = 1; i < 16; i++) {
5503                 if (i < 8)
5504                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5505                                rdev->vm_manager.saved_table_addr[i]);
5506                 else
5507                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5508                                rdev->vm_manager.saved_table_addr[i]);
5509         }
5510
5511         /* enable context1-15 */
5512         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5513                (u32)(rdev->dummy_page.addr >> 12));
5514         WREG32(VM_CONTEXT1_CNTL2, 4);
5515         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5516                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5517                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5518                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5519                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5520                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5521                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5522                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5523                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5524                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5525                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5526                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5527                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5528                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5529
5530         if (rdev->family == CHIP_KAVERI) {
5531                 u32 tmp = RREG32(CHUB_CONTROL);
5532                 tmp &= ~BYPASS_VM;
5533                 WREG32(CHUB_CONTROL, tmp);
5534         }
5535
5536         /* XXX SH_MEM regs */
5537         /* where to put LDS, scratch, GPUVM in FSA64 space */
5538         mutex_lock(&rdev->srbm_mutex);
5539         for (i = 0; i < 16; i++) {
5540                 cik_srbm_select(rdev, 0, 0, 0, i);
5541                 /* CP and shaders */
5542                 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
5543                 WREG32(SH_MEM_APE1_BASE, 1);
5544                 WREG32(SH_MEM_APE1_LIMIT, 0);
5545                 WREG32(SH_MEM_BASES, 0);
5546                 /* SDMA GFX */
5547                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5548                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5549                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5550                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5551                 /* XXX SDMA RLC - todo */
5552         }
5553         cik_srbm_select(rdev, 0, 0, 0, 0);
5554         mutex_unlock(&rdev->srbm_mutex);
5555
5556         cik_pcie_gart_tlb_flush(rdev);
5557         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5558                  (unsigned)(rdev->mc.gtt_size >> 20),
5559                  (unsigned long long)rdev->gart.table_addr);
5560         rdev->gart.ready = true;
5561         return 0;
5562 }
5563
5564 /**
5565  * cik_pcie_gart_disable - gart disable
5566  *
5567  * @rdev: radeon_device pointer
5568  *
5569  * This disables all VM page table (CIK).
5570  */
5571 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5572 {
5573         unsigned i;
5574
5575         for (i = 1; i < 16; ++i) {
5576                 uint32_t reg;
5577                 if (i < 8)
5578                         reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5579                 else
5580                         reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5581                 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5582         }
5583
5584         /* Disable all tables */
5585         WREG32(VM_CONTEXT0_CNTL, 0);
5586         WREG32(VM_CONTEXT1_CNTL, 0);
5587         /* Setup TLB control */
5588         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5589                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5590         /* Setup L2 cache */
5591         WREG32(VM_L2_CNTL,
5592                ENABLE_L2_FRAGMENT_PROCESSING |
5593                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5594                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5595                EFFECTIVE_L2_QUEUE_SIZE(7) |
5596                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5597         WREG32(VM_L2_CNTL2, 0);
5598         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5599                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5600         radeon_gart_table_vram_unpin(rdev);
5601 }
5602
5603 /**
5604  * cik_pcie_gart_fini - vm fini callback
5605  *
5606  * @rdev: radeon_device pointer
5607  *
5608  * Tears down the driver GART/VM setup (CIK).
5609  */
5610 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5611 {
5612         cik_pcie_gart_disable(rdev);
5613         radeon_gart_table_vram_free(rdev);
5614         radeon_gart_fini(rdev);
5615 }
5616
5617 /* vm parser */
5618 /**
5619  * cik_ib_parse - vm ib_parse callback
5620  *
5621  * @rdev: radeon_device pointer
5622  * @ib: indirect buffer pointer
5623  *
5624  * CIK uses hw IB checking so this is a nop (CIK).
5625  */
5626 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5627 {
5628         return 0;
5629 }
5630
5631 /*
5632  * vm
5633  * VMID 0 is the physical GPU addresses as used by the kernel.
5634  * VMIDs 1-15 are used for userspace clients and are handled
5635  * by the radeon vm/hsa code.
5636  */
5637 /**
5638  * cik_vm_init - cik vm init callback
5639  *
5640  * @rdev: radeon_device pointer
5641  *
5642  * Inits cik specific vm parameters (number of VMs, base of vram for
5643  * VMIDs 1-15) (CIK).
5644  * Returns 0 for success.
5645  */
5646 int cik_vm_init(struct radeon_device *rdev)
5647 {
5648         /*
5649          * number of VMs
5650          * VMID 0 is reserved for System
5651          * radeon graphics/compute will use VMIDs 1-15
5652          */
5653         rdev->vm_manager.nvm = 16;
5654         /* base offset of vram pages */
5655         if (rdev->flags & RADEON_IS_IGP) {
5656                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5657                 tmp <<= 22;
5658                 rdev->vm_manager.vram_base_offset = tmp;
5659         } else
5660                 rdev->vm_manager.vram_base_offset = 0;
5661
5662         return 0;
5663 }
5664
5665 /**
5666  * cik_vm_fini - cik vm fini callback
5667  *
5668  * @rdev: radeon_device pointer
5669  *
5670  * Tear down any asic specific VM setup (CIK).
5671  */
5672 void cik_vm_fini(struct radeon_device *rdev)
5673 {
5674 }
5675
5676 /**
5677  * cik_vm_decode_fault - print human readable fault info
5678  *
5679  * @rdev: radeon_device pointer
5680  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5681  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5682  *
5683  * Print human readable fault information (CIK).
5684  */
5685 static void cik_vm_decode_fault(struct radeon_device *rdev,
5686                                 u32 status, u32 addr, u32 mc_client)
5687 {
5688         u32 mc_id;
5689         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5690         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5691         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5692                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5693
5694         if (rdev->family == CHIP_HAWAII)
5695                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5696         else
5697                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5698
5699         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5700                protections, vmid, addr,
5701                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5702                block, mc_client, mc_id);
5703 }
5704
5705 /**
5706  * cik_vm_flush - cik vm flush using the CP
5707  *
5708  * @rdev: radeon_device pointer
5709  *
5710  * Update the page table base and flush the VM TLB
5711  * using the CP (CIK).
5712  */
5713 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5714                   unsigned vm_id, uint64_t pd_addr)
5715 {
5716         int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
5717
5718         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5719         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5720                                  WRITE_DATA_DST_SEL(0)));
5721         if (vm_id < 8) {
5722                 radeon_ring_write(ring,
5723                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5724         } else {
5725                 radeon_ring_write(ring,
5726                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5727         }
5728         radeon_ring_write(ring, 0);
5729         radeon_ring_write(ring, pd_addr >> 12);
5730
5731         /* update SH_MEM_* regs */
5732         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5733         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5734                                  WRITE_DATA_DST_SEL(0)));
5735         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5736         radeon_ring_write(ring, 0);
5737         radeon_ring_write(ring, VMID(vm_id));
5738
5739         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5740         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5741                                  WRITE_DATA_DST_SEL(0)));
5742         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5743         radeon_ring_write(ring, 0);
5744
5745         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5746         radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
5747         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5748         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5749
5750         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5751         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5752                                  WRITE_DATA_DST_SEL(0)));
5753         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5754         radeon_ring_write(ring, 0);
5755         radeon_ring_write(ring, VMID(0));
5756
5757         /* HDP flush */
5758         cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
5759
5760         /* bits 0-15 are the VM contexts0-15 */
5761         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5762         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5763                                  WRITE_DATA_DST_SEL(0)));
5764         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5765         radeon_ring_write(ring, 0);
5766         radeon_ring_write(ring, 1 << vm_id);
5767
5768         /* wait for the invalidate to complete */
5769         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5770         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5771                                  WAIT_REG_MEM_FUNCTION(0) |  /* always */
5772                                  WAIT_REG_MEM_ENGINE(0))); /* me */
5773         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5774         radeon_ring_write(ring, 0);
5775         radeon_ring_write(ring, 0); /* ref */
5776         radeon_ring_write(ring, 0); /* mask */
5777         radeon_ring_write(ring, 0x20); /* poll interval */
5778
5779         /* compute doesn't have PFP */
5780         if (usepfp) {
5781                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5782                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5783                 radeon_ring_write(ring, 0x0);
5784         }
5785 }
5786
5787 /*
5788  * RLC
5789  * The RLC is a multi-purpose microengine that handles a
5790  * variety of functions, the most important of which is
5791  * the interrupt controller.
5792  */
5793 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5794                                           bool enable)
5795 {
5796         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5797
5798         if (enable)
5799                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5800         else
5801                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5802         WREG32(CP_INT_CNTL_RING0, tmp);
5803 }
5804
5805 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5806 {
5807         u32 tmp;
5808
5809         tmp = RREG32(RLC_LB_CNTL);
5810         if (enable)
5811                 tmp |= LOAD_BALANCE_ENABLE;
5812         else
5813                 tmp &= ~LOAD_BALANCE_ENABLE;
5814         WREG32(RLC_LB_CNTL, tmp);
5815 }
5816
5817 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5818 {
5819         u32 i, j, k;
5820         u32 mask;
5821
5822         mutex_lock(&rdev->grbm_idx_mutex);
5823         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5824                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5825                         cik_select_se_sh(rdev, i, j);
5826                         for (k = 0; k < rdev->usec_timeout; k++) {
5827                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5828                                         break;
5829                                 udelay(1);
5830                         }
5831                 }
5832         }
5833         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5834         mutex_unlock(&rdev->grbm_idx_mutex);
5835
5836         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5837         for (k = 0; k < rdev->usec_timeout; k++) {
5838                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5839                         break;
5840                 udelay(1);
5841         }
5842 }
5843
5844 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5845 {
5846         u32 tmp;
5847
5848         tmp = RREG32(RLC_CNTL);
5849         if (tmp != rlc)
5850                 WREG32(RLC_CNTL, rlc);
5851 }
5852
5853 static u32 cik_halt_rlc(struct radeon_device *rdev)
5854 {
5855         u32 data, orig;
5856
5857         orig = data = RREG32(RLC_CNTL);
5858
5859         if (data & RLC_ENABLE) {
5860                 u32 i;
5861
5862                 data &= ~RLC_ENABLE;
5863                 WREG32(RLC_CNTL, data);
5864
5865                 for (i = 0; i < rdev->usec_timeout; i++) {
5866                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5867                                 break;
5868                         udelay(1);
5869                 }
5870
5871                 cik_wait_for_rlc_serdes(rdev);
5872         }
5873
5874         return orig;
5875 }
5876
5877 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5878 {
5879         u32 tmp, i, mask;
5880
5881         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5882         WREG32(RLC_GPR_REG2, tmp);
5883
5884         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5885         for (i = 0; i < rdev->usec_timeout; i++) {
5886                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5887                         break;
5888                 udelay(1);
5889         }
5890
5891         for (i = 0; i < rdev->usec_timeout; i++) {
5892                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5893                         break;
5894                 udelay(1);
5895         }
5896 }
5897
5898 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5899 {
5900         u32 tmp;
5901
5902         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5903         WREG32(RLC_GPR_REG2, tmp);
5904 }
5905
5906 /**
5907  * cik_rlc_stop - stop the RLC ME
5908  *
5909  * @rdev: radeon_device pointer
5910  *
5911  * Halt the RLC ME (MicroEngine) (CIK).
5912  */
5913 static void cik_rlc_stop(struct radeon_device *rdev)
5914 {
5915         WREG32(RLC_CNTL, 0);
5916
5917         cik_enable_gui_idle_interrupt(rdev, false);
5918
5919         cik_wait_for_rlc_serdes(rdev);
5920 }
5921
5922 /**
5923  * cik_rlc_start - start the RLC ME
5924  *
5925  * @rdev: radeon_device pointer
5926  *
5927  * Unhalt the RLC ME (MicroEngine) (CIK).
5928  */
5929 static void cik_rlc_start(struct radeon_device *rdev)
5930 {
5931         WREG32(RLC_CNTL, RLC_ENABLE);
5932
5933         cik_enable_gui_idle_interrupt(rdev, true);
5934
5935         udelay(50);
5936 }
5937
5938 /**
5939  * cik_rlc_resume - setup the RLC hw
5940  *
5941  * @rdev: radeon_device pointer
5942  *
5943  * Initialize the RLC registers, load the ucode,
5944  * and start the RLC (CIK).
5945  * Returns 0 for success, -EINVAL if the ucode is not available.
5946  */
5947 static int cik_rlc_resume(struct radeon_device *rdev)
5948 {
5949         u32 i, size, tmp;
5950
5951         if (!rdev->rlc_fw)
5952                 return -EINVAL;
5953
5954         cik_rlc_stop(rdev);
5955
5956         /* disable CG */
5957         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5958         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5959
5960         si_rlc_reset(rdev);
5961
5962         cik_init_pg(rdev);
5963
5964         cik_init_cg(rdev);
5965
5966         WREG32(RLC_LB_CNTR_INIT, 0);
5967         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5968
5969         mutex_lock(&rdev->grbm_idx_mutex);
5970         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5971         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5972         WREG32(RLC_LB_PARAMS, 0x00600408);
5973         WREG32(RLC_LB_CNTL, 0x80000004);
5974         mutex_unlock(&rdev->grbm_idx_mutex);
5975
5976         WREG32(RLC_MC_CNTL, 0);
5977         WREG32(RLC_UCODE_CNTL, 0);
5978
5979         if (rdev->new_fw) {
5980                 const struct rlc_firmware_header_v1_0 *hdr =
5981                         (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5982                 const __le32 *fw_data = (const __le32 *)
5983                         (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5984
5985                 radeon_ucode_print_rlc_hdr(&hdr->header);
5986
5987                 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5988                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5989                 for (i = 0; i < size; i++)
5990                         WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
5991                 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
5992         } else {
5993                 const __be32 *fw_data;
5994
5995                 switch (rdev->family) {
5996                 case CHIP_BONAIRE:
5997                 case CHIP_HAWAII:
5998                 default:
5999                         size = BONAIRE_RLC_UCODE_SIZE;
6000                         break;
6001                 case CHIP_KAVERI:
6002                         size = KV_RLC_UCODE_SIZE;
6003                         break;
6004                 case CHIP_KABINI:
6005                         size = KB_RLC_UCODE_SIZE;
6006                         break;
6007                 case CHIP_MULLINS:
6008                         size = ML_RLC_UCODE_SIZE;
6009                         break;
6010                 }
6011
6012                 fw_data = (const __be32 *)rdev->rlc_fw->data;
6013                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6014                 for (i = 0; i < size; i++)
6015                         WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6016                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6017         }
6018
6019         /* XXX - find out what chips support lbpw */
6020         cik_enable_lbpw(rdev, false);
6021
6022         if (rdev->family == CHIP_BONAIRE)
6023                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6024
6025         cik_rlc_start(rdev);
6026
6027         return 0;
6028 }
6029
6030 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6031 {
6032         u32 data, orig, tmp, tmp2;
6033
6034         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6035
6036         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6037                 cik_enable_gui_idle_interrupt(rdev, true);
6038
6039                 tmp = cik_halt_rlc(rdev);
6040
6041                 mutex_lock(&rdev->grbm_idx_mutex);
6042                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6043                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6044                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6045                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6046                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6047                 mutex_unlock(&rdev->grbm_idx_mutex);
6048
6049                 cik_update_rlc(rdev, tmp);
6050
6051                 data |= CGCG_EN | CGLS_EN;
6052         } else {
6053                 cik_enable_gui_idle_interrupt(rdev, false);
6054
6055                 RREG32(CB_CGTT_SCLK_CTRL);
6056                 RREG32(CB_CGTT_SCLK_CTRL);
6057                 RREG32(CB_CGTT_SCLK_CTRL);
6058                 RREG32(CB_CGTT_SCLK_CTRL);
6059
6060                 data &= ~(CGCG_EN | CGLS_EN);
6061         }
6062
6063         if (orig != data)
6064                 WREG32(RLC_CGCG_CGLS_CTRL, data);
6065
6066 }
6067
6068 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6069 {
6070         u32 data, orig, tmp = 0;
6071
6072         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6073                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6074                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6075                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
6076                                 data |= CP_MEM_LS_EN;
6077                                 if (orig != data)
6078                                         WREG32(CP_MEM_SLP_CNTL, data);
6079                         }
6080                 }
6081
6082                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6083                 data |= 0x00000001;
6084                 data &= 0xfffffffd;
6085                 if (orig != data)
6086                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6087
6088                 tmp = cik_halt_rlc(rdev);
6089
6090                 mutex_lock(&rdev->grbm_idx_mutex);
6091                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6092                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6093                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6094                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6095                 WREG32(RLC_SERDES_WR_CTRL, data);
6096                 mutex_unlock(&rdev->grbm_idx_mutex);
6097
6098                 cik_update_rlc(rdev, tmp);
6099
6100                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6101                         orig = data = RREG32(CGTS_SM_CTRL_REG);
6102                         data &= ~SM_MODE_MASK;
6103                         data |= SM_MODE(0x2);
6104                         data |= SM_MODE_ENABLE;
6105                         data &= ~CGTS_OVERRIDE;
6106                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6107                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6108                                 data &= ~CGTS_LS_OVERRIDE;
6109                         data &= ~ON_MONITOR_ADD_MASK;
6110                         data |= ON_MONITOR_ADD_EN;
6111                         data |= ON_MONITOR_ADD(0x96);
6112                         if (orig != data)
6113                                 WREG32(CGTS_SM_CTRL_REG, data);
6114                 }
6115         } else {
6116                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6117                 data |= 0x00000003;
6118                 if (orig != data)
6119                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6120
6121                 data = RREG32(RLC_MEM_SLP_CNTL);
6122                 if (data & RLC_MEM_LS_EN) {
6123                         data &= ~RLC_MEM_LS_EN;
6124                         WREG32(RLC_MEM_SLP_CNTL, data);
6125                 }
6126
6127                 data = RREG32(CP_MEM_SLP_CNTL);
6128                 if (data & CP_MEM_LS_EN) {
6129                         data &= ~CP_MEM_LS_EN;
6130                         WREG32(CP_MEM_SLP_CNTL, data);
6131                 }
6132
6133                 orig = data = RREG32(CGTS_SM_CTRL_REG);
6134                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6135                 if (orig != data)
6136                         WREG32(CGTS_SM_CTRL_REG, data);
6137
6138                 tmp = cik_halt_rlc(rdev);
6139
6140                 mutex_lock(&rdev->grbm_idx_mutex);
6141                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6142                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6143                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6144                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6145                 WREG32(RLC_SERDES_WR_CTRL, data);
6146                 mutex_unlock(&rdev->grbm_idx_mutex);
6147
6148                 cik_update_rlc(rdev, tmp);
6149         }
6150 }
6151
6152 static const u32 mc_cg_registers[] =
6153 {
6154         MC_HUB_MISC_HUB_CG,
6155         MC_HUB_MISC_SIP_CG,
6156         MC_HUB_MISC_VM_CG,
6157         MC_XPB_CLK_GAT,
6158         ATC_MISC_CG,
6159         MC_CITF_MISC_WR_CG,
6160         MC_CITF_MISC_RD_CG,
6161         MC_CITF_MISC_VM_CG,
6162         VM_L2_CG,
6163 };
6164
6165 static void cik_enable_mc_ls(struct radeon_device *rdev,
6166                              bool enable)
6167 {
6168         int i;
6169         u32 orig, data;
6170
6171         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6172                 orig = data = RREG32(mc_cg_registers[i]);
6173                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6174                         data |= MC_LS_ENABLE;
6175                 else
6176                         data &= ~MC_LS_ENABLE;
6177                 if (data != orig)
6178                         WREG32(mc_cg_registers[i], data);
6179         }
6180 }
6181
6182 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6183                                bool enable)
6184 {
6185         int i;
6186         u32 orig, data;
6187
6188         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6189                 orig = data = RREG32(mc_cg_registers[i]);
6190                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6191                         data |= MC_CG_ENABLE;
6192                 else
6193                         data &= ~MC_CG_ENABLE;
6194                 if (data != orig)
6195                         WREG32(mc_cg_registers[i], data);
6196         }
6197 }
6198
6199 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6200                                  bool enable)
6201 {
6202         u32 orig, data;
6203
6204         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6205                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6206                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6207         } else {
6208                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6209                 data |= 0xff000000;
6210                 if (data != orig)
6211                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6212
6213                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6214                 data |= 0xff000000;
6215                 if (data != orig)
6216                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6217         }
6218 }
6219
6220 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6221                                  bool enable)
6222 {
6223         u32 orig, data;
6224
6225         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6226                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6227                 data |= 0x100;
6228                 if (orig != data)
6229                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6230
6231                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6232                 data |= 0x100;
6233                 if (orig != data)
6234                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6235         } else {
6236                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6237                 data &= ~0x100;
6238                 if (orig != data)
6239                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6240
6241                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6242                 data &= ~0x100;
6243                 if (orig != data)
6244                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6245         }
6246 }
6247
6248 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6249                                 bool enable)
6250 {
6251         u32 orig, data;
6252
6253         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6254                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6255                 data = 0xfff;
6256                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6257
6258                 orig = data = RREG32(UVD_CGC_CTRL);
6259                 data |= DCM;
6260                 if (orig != data)
6261                         WREG32(UVD_CGC_CTRL, data);
6262         } else {
6263                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6264                 data &= ~0xfff;
6265                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6266
6267                 orig = data = RREG32(UVD_CGC_CTRL);
6268                 data &= ~DCM;
6269                 if (orig != data)
6270                         WREG32(UVD_CGC_CTRL, data);
6271         }
6272 }
6273
6274 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6275                                bool enable)
6276 {
6277         u32 orig, data;
6278
6279         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6280
6281         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6282                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6283                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6284         else
6285                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6286                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6287
6288         if (orig != data)
6289                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6290 }
6291
6292 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6293                                 bool enable)
6294 {
6295         u32 orig, data;
6296
6297         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6298
6299         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6300                 data &= ~CLOCK_GATING_DIS;
6301         else
6302                 data |= CLOCK_GATING_DIS;
6303
6304         if (orig != data)
6305                 WREG32(HDP_HOST_PATH_CNTL, data);
6306 }
6307
6308 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6309                               bool enable)
6310 {
6311         u32 orig, data;
6312
6313         orig = data = RREG32(HDP_MEM_POWER_LS);
6314
6315         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6316                 data |= HDP_LS_ENABLE;
6317         else
6318                 data &= ~HDP_LS_ENABLE;
6319
6320         if (orig != data)
6321                 WREG32(HDP_MEM_POWER_LS, data);
6322 }
6323
6324 void cik_update_cg(struct radeon_device *rdev,
6325                    u32 block, bool enable)
6326 {
6327
6328         if (block & RADEON_CG_BLOCK_GFX) {
6329                 cik_enable_gui_idle_interrupt(rdev, false);
6330                 /* order matters! */
6331                 if (enable) {
6332                         cik_enable_mgcg(rdev, true);
6333                         cik_enable_cgcg(rdev, true);
6334                 } else {
6335                         cik_enable_cgcg(rdev, false);
6336                         cik_enable_mgcg(rdev, false);
6337                 }
6338                 cik_enable_gui_idle_interrupt(rdev, true);
6339         }
6340
6341         if (block & RADEON_CG_BLOCK_MC) {
6342                 if (!(rdev->flags & RADEON_IS_IGP)) {
6343                         cik_enable_mc_mgcg(rdev, enable);
6344                         cik_enable_mc_ls(rdev, enable);
6345                 }
6346         }
6347
6348         if (block & RADEON_CG_BLOCK_SDMA) {
6349                 cik_enable_sdma_mgcg(rdev, enable);
6350                 cik_enable_sdma_mgls(rdev, enable);
6351         }
6352
6353         if (block & RADEON_CG_BLOCK_BIF) {
6354                 cik_enable_bif_mgls(rdev, enable);
6355         }
6356
6357         if (block & RADEON_CG_BLOCK_UVD) {
6358                 if (rdev->has_uvd)
6359                         cik_enable_uvd_mgcg(rdev, enable);
6360         }
6361
6362         if (block & RADEON_CG_BLOCK_HDP) {
6363                 cik_enable_hdp_mgcg(rdev, enable);
6364                 cik_enable_hdp_ls(rdev, enable);
6365         }
6366
6367         if (block & RADEON_CG_BLOCK_VCE) {
6368                 vce_v2_0_enable_mgcg(rdev, enable);
6369         }
6370 }
6371
6372 static void cik_init_cg(struct radeon_device *rdev)
6373 {
6374
6375         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6376
6377         if (rdev->has_uvd)
6378                 si_init_uvd_internal_cg(rdev);
6379
6380         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6381                              RADEON_CG_BLOCK_SDMA |
6382                              RADEON_CG_BLOCK_BIF |
6383                              RADEON_CG_BLOCK_UVD |
6384                              RADEON_CG_BLOCK_HDP), true);
6385 }
6386
6387 static void cik_fini_cg(struct radeon_device *rdev)
6388 {
6389         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6390                              RADEON_CG_BLOCK_SDMA |
6391                              RADEON_CG_BLOCK_BIF |
6392                              RADEON_CG_BLOCK_UVD |
6393                              RADEON_CG_BLOCK_HDP), false);
6394
6395         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6396 }
6397
6398 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6399                                           bool enable)
6400 {
6401         u32 data, orig;
6402
6403         orig = data = RREG32(RLC_PG_CNTL);
6404         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6405                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6406         else
6407                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6408         if (orig != data)
6409                 WREG32(RLC_PG_CNTL, data);
6410 }
6411
6412 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6413                                           bool enable)
6414 {
6415         u32 data, orig;
6416
6417         orig = data = RREG32(RLC_PG_CNTL);
6418         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6419                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6420         else
6421                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6422         if (orig != data)
6423                 WREG32(RLC_PG_CNTL, data);
6424 }
6425
6426 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6427 {
6428         u32 data, orig;
6429
6430         orig = data = RREG32(RLC_PG_CNTL);
6431         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6432                 data &= ~DISABLE_CP_PG;
6433         else
6434                 data |= DISABLE_CP_PG;
6435         if (orig != data)
6436                 WREG32(RLC_PG_CNTL, data);
6437 }
6438
6439 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6440 {
6441         u32 data, orig;
6442
6443         orig = data = RREG32(RLC_PG_CNTL);
6444         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6445                 data &= ~DISABLE_GDS_PG;
6446         else
6447                 data |= DISABLE_GDS_PG;
6448         if (orig != data)
6449                 WREG32(RLC_PG_CNTL, data);
6450 }
6451
6452 #define CP_ME_TABLE_SIZE    96
6453 #define CP_ME_TABLE_OFFSET  2048
6454 #define CP_MEC_TABLE_OFFSET 4096
6455
6456 void cik_init_cp_pg_table(struct radeon_device *rdev)
6457 {
6458         volatile u32 *dst_ptr;
6459         int me, i, max_me = 4;
6460         u32 bo_offset = 0;
6461         u32 table_offset, table_size;
6462
6463         if (rdev->family == CHIP_KAVERI)
6464                 max_me = 5;
6465
6466         if (rdev->rlc.cp_table_ptr == NULL)
6467                 return;
6468
6469         /* write the cp table buffer */
6470         dst_ptr = rdev->rlc.cp_table_ptr;
6471         for (me = 0; me < max_me; me++) {
6472                 if (rdev->new_fw) {
6473                         const __le32 *fw_data;
6474                         const struct gfx_firmware_header_v1_0 *hdr;
6475
6476                         if (me == 0) {
6477                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6478                                 fw_data = (const __le32 *)
6479                                         (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6480                                 table_offset = le32_to_cpu(hdr->jt_offset);
6481                                 table_size = le32_to_cpu(hdr->jt_size);
6482                         } else if (me == 1) {
6483                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6484                                 fw_data = (const __le32 *)
6485                                         (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6486                                 table_offset = le32_to_cpu(hdr->jt_offset);
6487                                 table_size = le32_to_cpu(hdr->jt_size);
6488                         } else if (me == 2) {
6489                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6490                                 fw_data = (const __le32 *)
6491                                         (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6492                                 table_offset = le32_to_cpu(hdr->jt_offset);
6493                                 table_size = le32_to_cpu(hdr->jt_size);
6494                         } else if (me == 3) {
6495                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6496                                 fw_data = (const __le32 *)
6497                                         (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6498                                 table_offset = le32_to_cpu(hdr->jt_offset);
6499                                 table_size = le32_to_cpu(hdr->jt_size);
6500                         } else {
6501                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6502                                 fw_data = (const __le32 *)
6503                                         (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6504                                 table_offset = le32_to_cpu(hdr->jt_offset);
6505                                 table_size = le32_to_cpu(hdr->jt_size);
6506                         }
6507
6508                         for (i = 0; i < table_size; i ++) {
6509                                 dst_ptr[bo_offset + i] =
6510                                         cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6511                         }
6512                         bo_offset += table_size;
6513                 } else {
6514                         const __be32 *fw_data;
6515                         table_size = CP_ME_TABLE_SIZE;
6516
6517                         if (me == 0) {
6518                                 fw_data = (const __be32 *)rdev->ce_fw->data;
6519                                 table_offset = CP_ME_TABLE_OFFSET;
6520                         } else if (me == 1) {
6521                                 fw_data = (const __be32 *)rdev->pfp_fw->data;
6522                                 table_offset = CP_ME_TABLE_OFFSET;
6523                         } else if (me == 2) {
6524                                 fw_data = (const __be32 *)rdev->me_fw->data;
6525                                 table_offset = CP_ME_TABLE_OFFSET;
6526                         } else {
6527                                 fw_data = (const __be32 *)rdev->mec_fw->data;
6528                                 table_offset = CP_MEC_TABLE_OFFSET;
6529                         }
6530
6531                         for (i = 0; i < table_size; i ++) {
6532                                 dst_ptr[bo_offset + i] =
6533                                         cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6534                         }
6535                         bo_offset += table_size;
6536                 }
6537         }
6538 }
6539
6540 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6541                                 bool enable)
6542 {
6543         u32 data, orig;
6544
6545         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6546                 orig = data = RREG32(RLC_PG_CNTL);
6547                 data |= GFX_PG_ENABLE;
6548                 if (orig != data)
6549                         WREG32(RLC_PG_CNTL, data);
6550
6551                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6552                 data |= AUTO_PG_EN;
6553                 if (orig != data)
6554                         WREG32(RLC_AUTO_PG_CTRL, data);
6555         } else {
6556                 orig = data = RREG32(RLC_PG_CNTL);
6557                 data &= ~GFX_PG_ENABLE;
6558                 if (orig != data)
6559                         WREG32(RLC_PG_CNTL, data);
6560
6561                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6562                 data &= ~AUTO_PG_EN;
6563                 if (orig != data)
6564                         WREG32(RLC_AUTO_PG_CTRL, data);
6565
6566                 data = RREG32(DB_RENDER_CONTROL);
6567         }
6568 }
6569
6570 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6571 {
6572         u32 mask = 0, tmp, tmp1;
6573         int i;
6574
6575         mutex_lock(&rdev->grbm_idx_mutex);
6576         cik_select_se_sh(rdev, se, sh);
6577         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6578         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6579         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6580         mutex_unlock(&rdev->grbm_idx_mutex);
6581
6582         tmp &= 0xffff0000;
6583
6584         tmp |= tmp1;
6585         tmp >>= 16;
6586
6587         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6588                 mask <<= 1;
6589                 mask |= 1;
6590         }
6591
6592         return (~tmp) & mask;
6593 }
6594
6595 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6596 {
6597         u32 i, j, k, active_cu_number = 0;
6598         u32 mask, counter, cu_bitmap;
6599         u32 tmp = 0;
6600
6601         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6602                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6603                         mask = 1;
6604                         cu_bitmap = 0;
6605                         counter = 0;
6606                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6607                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6608                                         if (counter < 2)
6609                                                 cu_bitmap |= mask;
6610                                         counter ++;
6611                                 }
6612                                 mask <<= 1;
6613                         }
6614
6615                         active_cu_number += counter;
6616                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6617                 }
6618         }
6619
6620         WREG32(RLC_PG_AO_CU_MASK, tmp);
6621
6622         tmp = RREG32(RLC_MAX_PG_CU);
6623         tmp &= ~MAX_PU_CU_MASK;
6624         tmp |= MAX_PU_CU(active_cu_number);
6625         WREG32(RLC_MAX_PG_CU, tmp);
6626 }
6627
6628 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6629                                        bool enable)
6630 {
6631         u32 data, orig;
6632
6633         orig = data = RREG32(RLC_PG_CNTL);
6634         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6635                 data |= STATIC_PER_CU_PG_ENABLE;
6636         else
6637                 data &= ~STATIC_PER_CU_PG_ENABLE;
6638         if (orig != data)
6639                 WREG32(RLC_PG_CNTL, data);
6640 }
6641
6642 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6643                                         bool enable)
6644 {
6645         u32 data, orig;
6646
6647         orig = data = RREG32(RLC_PG_CNTL);
6648         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6649                 data |= DYN_PER_CU_PG_ENABLE;
6650         else
6651                 data &= ~DYN_PER_CU_PG_ENABLE;
6652         if (orig != data)
6653                 WREG32(RLC_PG_CNTL, data);
6654 }
6655
6656 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6657 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6658
6659 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6660 {
6661         u32 data, orig;
6662         u32 i;
6663
6664         if (rdev->rlc.cs_data) {
6665                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6666                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6667                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6668                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6669         } else {
6670                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6671                 for (i = 0; i < 3; i++)
6672                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6673         }
6674         if (rdev->rlc.reg_list) {
6675                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6676                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6677                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6678         }
6679
6680         orig = data = RREG32(RLC_PG_CNTL);
6681         data |= GFX_PG_SRC;
6682         if (orig != data)
6683                 WREG32(RLC_PG_CNTL, data);
6684
6685         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6686         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6687
6688         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6689         data &= ~IDLE_POLL_COUNT_MASK;
6690         data |= IDLE_POLL_COUNT(0x60);
6691         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6692
6693         data = 0x10101010;
6694         WREG32(RLC_PG_DELAY, data);
6695
6696         data = RREG32(RLC_PG_DELAY_2);
6697         data &= ~0xff;
6698         data |= 0x3;
6699         WREG32(RLC_PG_DELAY_2, data);
6700
6701         data = RREG32(RLC_AUTO_PG_CTRL);
6702         data &= ~GRBM_REG_SGIT_MASK;
6703         data |= GRBM_REG_SGIT(0x700);
6704         WREG32(RLC_AUTO_PG_CTRL, data);
6705
6706 }
6707
6708 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6709 {
6710         cik_enable_gfx_cgpg(rdev, enable);
6711         cik_enable_gfx_static_mgpg(rdev, enable);
6712         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6713 }
6714
6715 u32 cik_get_csb_size(struct radeon_device *rdev)
6716 {
6717         u32 count = 0;
6718         const struct cs_section_def *sect = NULL;
6719         const struct cs_extent_def *ext = NULL;
6720
6721         if (rdev->rlc.cs_data == NULL)
6722                 return 0;
6723
6724         /* begin clear state */
6725         count += 2;
6726         /* context control state */
6727         count += 3;
6728
6729         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6730                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6731                         if (sect->id == SECT_CONTEXT)
6732                                 count += 2 + ext->reg_count;
6733                         else
6734                                 return 0;
6735                 }
6736         }
6737         /* pa_sc_raster_config/pa_sc_raster_config1 */
6738         count += 4;
6739         /* end clear state */
6740         count += 2;
6741         /* clear state */
6742         count += 2;
6743
6744         return count;
6745 }
6746
6747 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6748 {
6749         u32 count = 0, i;
6750         const struct cs_section_def *sect = NULL;
6751         const struct cs_extent_def *ext = NULL;
6752
6753         if (rdev->rlc.cs_data == NULL)
6754                 return;
6755         if (buffer == NULL)
6756                 return;
6757
6758         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6759         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6760
6761         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6762         buffer[count++] = cpu_to_le32(0x80000000);
6763         buffer[count++] = cpu_to_le32(0x80000000);
6764
6765         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6766                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6767                         if (sect->id == SECT_CONTEXT) {
6768                                 buffer[count++] =
6769                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6770                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6771                                 for (i = 0; i < ext->reg_count; i++)
6772                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6773                         } else {
6774                                 return;
6775                         }
6776                 }
6777         }
6778
6779         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6780         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6781         switch (rdev->family) {
6782         case CHIP_BONAIRE:
6783                 buffer[count++] = cpu_to_le32(0x16000012);
6784                 buffer[count++] = cpu_to_le32(0x00000000);
6785                 break;
6786         case CHIP_KAVERI:
6787                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6788                 buffer[count++] = cpu_to_le32(0x00000000);
6789                 break;
6790         case CHIP_KABINI:
6791         case CHIP_MULLINS:
6792                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6793                 buffer[count++] = cpu_to_le32(0x00000000);
6794                 break;
6795         case CHIP_HAWAII:
6796                 buffer[count++] = cpu_to_le32(0x3a00161a);
6797                 buffer[count++] = cpu_to_le32(0x0000002e);
6798                 break;
6799         default:
6800                 buffer[count++] = cpu_to_le32(0x00000000);
6801                 buffer[count++] = cpu_to_le32(0x00000000);
6802                 break;
6803         }
6804
6805         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6806         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6807
6808         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6809         buffer[count++] = cpu_to_le32(0);
6810 }
6811
6812 static void cik_init_pg(struct radeon_device *rdev)
6813 {
6814         if (rdev->pg_flags) {
6815                 cik_enable_sck_slowdown_on_pu(rdev, true);
6816                 cik_enable_sck_slowdown_on_pd(rdev, true);
6817                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6818                         cik_init_gfx_cgpg(rdev);
6819                         cik_enable_cp_pg(rdev, true);
6820                         cik_enable_gds_pg(rdev, true);
6821                 }
6822                 cik_init_ao_cu_mask(rdev);
6823                 cik_update_gfx_pg(rdev, true);
6824         }
6825 }
6826
6827 static void cik_fini_pg(struct radeon_device *rdev)
6828 {
6829         if (rdev->pg_flags) {
6830                 cik_update_gfx_pg(rdev, false);
6831                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6832                         cik_enable_cp_pg(rdev, false);
6833                         cik_enable_gds_pg(rdev, false);
6834                 }
6835         }
6836 }
6837
6838 /*
6839  * Interrupts
6840  * Starting with r6xx, interrupts are handled via a ring buffer.
6841  * Ring buffers are areas of GPU accessible memory that the GPU
6842  * writes interrupt vectors into and the host reads vectors out of.
6843  * There is a rptr (read pointer) that determines where the
6844  * host is currently reading, and a wptr (write pointer)
6845  * which determines where the GPU has written.  When the
6846  * pointers are equal, the ring is idle.  When the GPU
6847  * writes vectors to the ring buffer, it increments the
6848  * wptr.  When there is an interrupt, the host then starts
6849  * fetching commands and processing them until the pointers are
6850  * equal again at which point it updates the rptr.
6851  */
6852
6853 /**
6854  * cik_enable_interrupts - Enable the interrupt ring buffer
6855  *
6856  * @rdev: radeon_device pointer
6857  *
6858  * Enable the interrupt ring buffer (CIK).
6859  */
6860 static void cik_enable_interrupts(struct radeon_device *rdev)
6861 {
6862         u32 ih_cntl = RREG32(IH_CNTL);
6863         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6864
6865         ih_cntl |= ENABLE_INTR;
6866         ih_rb_cntl |= IH_RB_ENABLE;
6867         WREG32(IH_CNTL, ih_cntl);
6868         WREG32(IH_RB_CNTL, ih_rb_cntl);
6869         rdev->ih.enabled = true;
6870 }
6871
6872 /**
6873  * cik_disable_interrupts - Disable the interrupt ring buffer
6874  *
6875  * @rdev: radeon_device pointer
6876  *
6877  * Disable the interrupt ring buffer (CIK).
6878  */
6879 static void cik_disable_interrupts(struct radeon_device *rdev)
6880 {
6881         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6882         u32 ih_cntl = RREG32(IH_CNTL);
6883
6884         ih_rb_cntl &= ~IH_RB_ENABLE;
6885         ih_cntl &= ~ENABLE_INTR;
6886         WREG32(IH_RB_CNTL, ih_rb_cntl);
6887         WREG32(IH_CNTL, ih_cntl);
6888         /* set rptr, wptr to 0 */
6889         WREG32(IH_RB_RPTR, 0);
6890         WREG32(IH_RB_WPTR, 0);
6891         rdev->ih.enabled = false;
6892         rdev->ih.rptr = 0;
6893 }
6894
6895 /**
6896  * cik_disable_interrupt_state - Disable all interrupt sources
6897  *
6898  * @rdev: radeon_device pointer
6899  *
6900  * Clear all interrupt enable bits used by the driver (CIK).
6901  */
6902 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6903 {
6904         u32 tmp;
6905
6906         /* gfx ring */
6907         tmp = RREG32(CP_INT_CNTL_RING0) &
6908                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6909         WREG32(CP_INT_CNTL_RING0, tmp);
6910         /* sdma */
6911         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6912         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6913         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6914         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6915         /* compute queues */
6916         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6917         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6918         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6919         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6920         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6921         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6922         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6923         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6924         /* grbm */
6925         WREG32(GRBM_INT_CNTL, 0);
6926         /* SRBM */
6927         WREG32(SRBM_INT_CNTL, 0);
6928         /* vline/vblank, etc. */
6929         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6930         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6931         if (rdev->num_crtc >= 4) {
6932                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6933                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6934         }
6935         if (rdev->num_crtc >= 6) {
6936                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6937                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6938         }
6939         /* pflip */
6940         if (rdev->num_crtc >= 2) {
6941                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6942                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6943         }
6944         if (rdev->num_crtc >= 4) {
6945                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6946                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6947         }
6948         if (rdev->num_crtc >= 6) {
6949                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6950                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6951         }
6952
6953         /* dac hotplug */
6954         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6955
6956         /* digital hotplug */
6957         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6958         WREG32(DC_HPD1_INT_CONTROL, tmp);
6959         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6960         WREG32(DC_HPD2_INT_CONTROL, tmp);
6961         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6962         WREG32(DC_HPD3_INT_CONTROL, tmp);
6963         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6964         WREG32(DC_HPD4_INT_CONTROL, tmp);
6965         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6966         WREG32(DC_HPD5_INT_CONTROL, tmp);
6967         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6968         WREG32(DC_HPD6_INT_CONTROL, tmp);
6969
6970 }
6971
6972 /**
6973  * cik_irq_init - init and enable the interrupt ring
6974  *
6975  * @rdev: radeon_device pointer
6976  *
6977  * Allocate a ring buffer for the interrupt controller,
6978  * enable the RLC, disable interrupts, enable the IH
6979  * ring buffer and enable it (CIK).
6980  * Called at device load and reume.
6981  * Returns 0 for success, errors for failure.
6982  */
6983 static int cik_irq_init(struct radeon_device *rdev)
6984 {
6985         int ret = 0;
6986         int rb_bufsz;
6987         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6988
6989         /* allocate ring */
6990         ret = r600_ih_ring_alloc(rdev);
6991         if (ret)
6992                 return ret;
6993
6994         /* disable irqs */
6995         cik_disable_interrupts(rdev);
6996
6997         /* init rlc */
6998         ret = cik_rlc_resume(rdev);
6999         if (ret) {
7000                 r600_ih_ring_fini(rdev);
7001                 return ret;
7002         }
7003
7004         /* setup interrupt control */
7005         /* XXX this should actually be a bus address, not an MC address. same on older asics */
7006         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7007         interrupt_cntl = RREG32(INTERRUPT_CNTL);
7008         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7009          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7010          */
7011         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7012         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7013         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7014         WREG32(INTERRUPT_CNTL, interrupt_cntl);
7015
7016         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7017         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7018
7019         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7020                       IH_WPTR_OVERFLOW_CLEAR |
7021                       (rb_bufsz << 1));
7022
7023         if (rdev->wb.enabled)
7024                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7025
7026         /* set the writeback address whether it's enabled or not */
7027         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7028         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7029
7030         WREG32(IH_RB_CNTL, ih_rb_cntl);
7031
7032         /* set rptr, wptr to 0 */
7033         WREG32(IH_RB_RPTR, 0);
7034         WREG32(IH_RB_WPTR, 0);
7035
7036         /* Default settings for IH_CNTL (disabled at first) */
7037         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7038         /* RPTR_REARM only works if msi's are enabled */
7039         if (rdev->msi_enabled)
7040                 ih_cntl |= RPTR_REARM;
7041         WREG32(IH_CNTL, ih_cntl);
7042
7043         /* force the active interrupt state to all disabled */
7044         cik_disable_interrupt_state(rdev);
7045
7046         pci_set_master(rdev->pdev);
7047
7048         /* enable irqs */
7049         cik_enable_interrupts(rdev);
7050
7051         return ret;
7052 }
7053
7054 /**
7055  * cik_irq_set - enable/disable interrupt sources
7056  *
7057  * @rdev: radeon_device pointer
7058  *
7059  * Enable interrupt sources on the GPU (vblanks, hpd,
7060  * etc.) (CIK).
7061  * Returns 0 for success, errors for failure.
7062  */
7063 int cik_irq_set(struct radeon_device *rdev)
7064 {
7065         u32 cp_int_cntl;
7066         u32 cp_m1p0;
7067         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7068         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7069         u32 grbm_int_cntl = 0;
7070         u32 dma_cntl, dma_cntl1;
7071
7072         if (!rdev->irq.installed) {
7073                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7074                 return -EINVAL;
7075         }
7076         /* don't enable anything if the ih is disabled */
7077         if (!rdev->ih.enabled) {
7078                 cik_disable_interrupts(rdev);
7079                 /* force the active interrupt state to all disabled */
7080                 cik_disable_interrupt_state(rdev);
7081                 return 0;
7082         }
7083
7084         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7085                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7086         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7087
7088         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7089         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7090         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7091         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7092         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7093         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7094
7095         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7096         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7097
7098         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7099
7100         /* enable CP interrupts on all rings */
7101         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7102                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7103                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7104         }
7105         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7106                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7107                 DRM_DEBUG("si_irq_set: sw int cp1\n");
7108                 if (ring->me == 1) {
7109                         switch (ring->pipe) {
7110                         case 0:
7111                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7112                                 break;
7113                         default:
7114                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7115                                 break;
7116                         }
7117                 } else {
7118                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7119                 }
7120         }
7121         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7122                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7123                 DRM_DEBUG("si_irq_set: sw int cp2\n");
7124                 if (ring->me == 1) {
7125                         switch (ring->pipe) {
7126                         case 0:
7127                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7128                                 break;
7129                         default:
7130                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7131                                 break;
7132                         }
7133                 } else {
7134                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7135                 }
7136         }
7137
7138         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7139                 DRM_DEBUG("cik_irq_set: sw int dma\n");
7140                 dma_cntl |= TRAP_ENABLE;
7141         }
7142
7143         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7144                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7145                 dma_cntl1 |= TRAP_ENABLE;
7146         }
7147
7148         if (rdev->irq.crtc_vblank_int[0] ||
7149             atomic_read(&rdev->irq.pflip[0])) {
7150                 DRM_DEBUG("cik_irq_set: vblank 0\n");
7151                 crtc1 |= VBLANK_INTERRUPT_MASK;
7152         }
7153         if (rdev->irq.crtc_vblank_int[1] ||
7154             atomic_read(&rdev->irq.pflip[1])) {
7155                 DRM_DEBUG("cik_irq_set: vblank 1\n");
7156                 crtc2 |= VBLANK_INTERRUPT_MASK;
7157         }
7158         if (rdev->irq.crtc_vblank_int[2] ||
7159             atomic_read(&rdev->irq.pflip[2])) {
7160                 DRM_DEBUG("cik_irq_set: vblank 2\n");
7161                 crtc3 |= VBLANK_INTERRUPT_MASK;
7162         }
7163         if (rdev->irq.crtc_vblank_int[3] ||
7164             atomic_read(&rdev->irq.pflip[3])) {
7165                 DRM_DEBUG("cik_irq_set: vblank 3\n");
7166                 crtc4 |= VBLANK_INTERRUPT_MASK;
7167         }
7168         if (rdev->irq.crtc_vblank_int[4] ||
7169             atomic_read(&rdev->irq.pflip[4])) {
7170                 DRM_DEBUG("cik_irq_set: vblank 4\n");
7171                 crtc5 |= VBLANK_INTERRUPT_MASK;
7172         }
7173         if (rdev->irq.crtc_vblank_int[5] ||
7174             atomic_read(&rdev->irq.pflip[5])) {
7175                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7176                 crtc6 |= VBLANK_INTERRUPT_MASK;
7177         }
7178         if (rdev->irq.hpd[0]) {
7179                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7180                 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7181         }
7182         if (rdev->irq.hpd[1]) {
7183                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7184                 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7185         }
7186         if (rdev->irq.hpd[2]) {
7187                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7188                 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7189         }
7190         if (rdev->irq.hpd[3]) {
7191                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7192                 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7193         }
7194         if (rdev->irq.hpd[4]) {
7195                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7196                 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7197         }
7198         if (rdev->irq.hpd[5]) {
7199                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7200                 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
7201         }
7202
7203         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7204
7205         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7206         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7207
7208         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7209
7210         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7211
7212         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7213         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7214         if (rdev->num_crtc >= 4) {
7215                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7216                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7217         }
7218         if (rdev->num_crtc >= 6) {
7219                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7220                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7221         }
7222
7223         if (rdev->num_crtc >= 2) {
7224                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7225                        GRPH_PFLIP_INT_MASK);
7226                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7227                        GRPH_PFLIP_INT_MASK);
7228         }
7229         if (rdev->num_crtc >= 4) {
7230                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7231                        GRPH_PFLIP_INT_MASK);
7232                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7233                        GRPH_PFLIP_INT_MASK);
7234         }
7235         if (rdev->num_crtc >= 6) {
7236                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7237                        GRPH_PFLIP_INT_MASK);
7238                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7239                        GRPH_PFLIP_INT_MASK);
7240         }
7241
7242         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7243         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7244         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7245         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7246         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7247         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7248
7249         /* posting read */
7250         RREG32(SRBM_STATUS);
7251
7252         return 0;
7253 }
7254
7255 /**
7256  * cik_irq_ack - ack interrupt sources
7257  *
7258  * @rdev: radeon_device pointer
7259  *
7260  * Ack interrupt sources on the GPU (vblanks, hpd,
7261  * etc.) (CIK).  Certain interrupts sources are sw
7262  * generated and do not require an explicit ack.
7263  */
7264 static inline void cik_irq_ack(struct radeon_device *rdev)
7265 {
7266         u32 tmp;
7267
7268         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7269         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7270         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7271         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7272         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7273         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7274         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7275
7276         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7277                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7278         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7279                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7280         if (rdev->num_crtc >= 4) {
7281                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7282                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7283                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7284                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7285         }
7286         if (rdev->num_crtc >= 6) {
7287                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7288                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7289                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7290                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7291         }
7292
7293         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7294                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7295                        GRPH_PFLIP_INT_CLEAR);
7296         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7297                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7298                        GRPH_PFLIP_INT_CLEAR);
7299         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7300                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7301         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7302                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7303         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7304                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7305         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7306                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7307
7308         if (rdev->num_crtc >= 4) {
7309                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7310                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7311                                GRPH_PFLIP_INT_CLEAR);
7312                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7313                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7314                                GRPH_PFLIP_INT_CLEAR);
7315                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7316                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7317                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7318                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7319                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7320                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7321                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7322                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7323         }
7324
7325         if (rdev->num_crtc >= 6) {
7326                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7327                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7328                                GRPH_PFLIP_INT_CLEAR);
7329                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7330                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7331                                GRPH_PFLIP_INT_CLEAR);
7332                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7333                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7334                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7335                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7336                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7337                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7338                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7339                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7340         }
7341
7342         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7343                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7344                 tmp |= DC_HPDx_INT_ACK;
7345                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7346         }
7347         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7348                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7349                 tmp |= DC_HPDx_INT_ACK;
7350                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7351         }
7352         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7353                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7354                 tmp |= DC_HPDx_INT_ACK;
7355                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7356         }
7357         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7358                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7359                 tmp |= DC_HPDx_INT_ACK;
7360                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7361         }
7362         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7363                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7364                 tmp |= DC_HPDx_INT_ACK;
7365                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7366         }
7367         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7368                 tmp = RREG32(DC_HPD6_INT_CONTROL);
7369                 tmp |= DC_HPDx_INT_ACK;
7370                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7371         }
7372         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7373                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7374                 tmp |= DC_HPDx_RX_INT_ACK;
7375                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7376         }
7377         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7378                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7379                 tmp |= DC_HPDx_RX_INT_ACK;
7380                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7381         }
7382         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7383                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7384                 tmp |= DC_HPDx_RX_INT_ACK;
7385                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7386         }
7387         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7388                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7389                 tmp |= DC_HPDx_RX_INT_ACK;
7390                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7391         }
7392         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7393                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7394                 tmp |= DC_HPDx_RX_INT_ACK;
7395                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7396         }
7397         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7398                 tmp = RREG32(DC_HPD6_INT_CONTROL);
7399                 tmp |= DC_HPDx_RX_INT_ACK;
7400                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7401         }
7402 }
7403
7404 /**
7405  * cik_irq_disable - disable interrupts
7406  *
7407  * @rdev: radeon_device pointer
7408  *
7409  * Disable interrupts on the hw (CIK).
7410  */
7411 static void cik_irq_disable(struct radeon_device *rdev)
7412 {
7413         cik_disable_interrupts(rdev);
7414         /* Wait and acknowledge irq */
7415         mdelay(1);
7416         cik_irq_ack(rdev);
7417         cik_disable_interrupt_state(rdev);
7418 }
7419
7420 /**
7421  * cik_irq_disable - disable interrupts for suspend
7422  *
7423  * @rdev: radeon_device pointer
7424  *
7425  * Disable interrupts and stop the RLC (CIK).
7426  * Used for suspend.
7427  */
7428 static void cik_irq_suspend(struct radeon_device *rdev)
7429 {
7430         cik_irq_disable(rdev);
7431         cik_rlc_stop(rdev);
7432 }
7433
7434 /**
7435  * cik_irq_fini - tear down interrupt support
7436  *
7437  * @rdev: radeon_device pointer
7438  *
7439  * Disable interrupts on the hw and free the IH ring
7440  * buffer (CIK).
7441  * Used for driver unload.
7442  */
7443 static void cik_irq_fini(struct radeon_device *rdev)
7444 {
7445         cik_irq_suspend(rdev);
7446         r600_ih_ring_fini(rdev);
7447 }
7448
7449 /**
7450  * cik_get_ih_wptr - get the IH ring buffer wptr
7451  *
7452  * @rdev: radeon_device pointer
7453  *
7454  * Get the IH ring buffer wptr from either the register
7455  * or the writeback memory buffer (CIK).  Also check for
7456  * ring buffer overflow and deal with it.
7457  * Used by cik_irq_process().
7458  * Returns the value of the wptr.
7459  */
7460 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7461 {
7462         u32 wptr, tmp;
7463
7464         if (rdev->wb.enabled)
7465                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7466         else
7467                 wptr = RREG32(IH_RB_WPTR);
7468
7469         if (wptr & RB_OVERFLOW) {
7470                 wptr &= ~RB_OVERFLOW;
7471                 /* When a ring buffer overflow happen start parsing interrupt
7472                  * from the last not overwritten vector (wptr + 16). Hopefully
7473                  * this should allow us to catchup.
7474                  */
7475                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7476                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7477                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7478                 tmp = RREG32(IH_RB_CNTL);
7479                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7480                 WREG32(IH_RB_CNTL, tmp);
7481         }
7482         return (wptr & rdev->ih.ptr_mask);
7483 }
7484
7485 /*        CIK IV Ring
7486  * Each IV ring entry is 128 bits:
7487  * [7:0]    - interrupt source id
7488  * [31:8]   - reserved
7489  * [59:32]  - interrupt source data
7490  * [63:60]  - reserved
7491  * [71:64]  - RINGID
7492  *            CP:
7493  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7494  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7495  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7496  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7497  *            PIPE_ID - ME0 0=3D
7498  *                    - ME1&2 compute dispatcher (4 pipes each)
7499  *            SDMA:
7500  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7501  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7502  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7503  * [79:72]  - VMID
7504  * [95:80]  - PASID
7505  * [127:96] - reserved
7506  */
7507 /**
7508  * cik_irq_process - interrupt handler
7509  *
7510  * @rdev: radeon_device pointer
7511  *
7512  * Interrupt hander (CIK).  Walk the IH ring,
7513  * ack interrupts and schedule work to handle
7514  * interrupt events.
7515  * Returns irq process return code.
7516  */
7517 irqreturn_t cik_irq_process(struct radeon_device *rdev)
7518 {
7519         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7520         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7521         u32 wptr;
7522         u32 rptr;
7523         u32 src_id, src_data, ring_id;
7524         u8 me_id, pipe_id, queue_id;
7525         u32 ring_index;
7526         bool queue_hotplug = false;
7527         bool queue_dp = false;
7528         bool queue_reset = false;
7529         u32 addr, status, mc_client;
7530         bool queue_thermal = false;
7531
7532         if (!rdev->ih.enabled || rdev->shutdown)
7533                 return IRQ_NONE;
7534
7535         wptr = cik_get_ih_wptr(rdev);
7536
7537 restart_ih:
7538         /* is somebody else already processing irqs? */
7539         if (atomic_xchg(&rdev->ih.lock, 1))
7540                 return IRQ_NONE;
7541
7542         rptr = rdev->ih.rptr;
7543         DRM_DEBUG_VBLANK("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7544
7545         /* Order reading of wptr vs. reading of IH ring data */
7546         rmb();
7547
7548         /* display interrupts */
7549         cik_irq_ack(rdev);
7550
7551         while (rptr != wptr) {
7552                 /* wptr/rptr are in bytes! */
7553                 ring_index = rptr / 4;
7554
7555                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7556                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7557                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7558
7559                 switch (src_id) {
7560                 case 1: /* D1 vblank/vline */
7561                         switch (src_data) {
7562                         case 0: /* D1 vblank */
7563                                 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7564                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7565
7566                                 if (rdev->irq.crtc_vblank_int[0]) {
7567                                         drm_handle_vblank(rdev->ddev, 0);
7568                                         rdev->pm.vblank_sync = true;
7569                                         wake_up(&rdev->irq.vblank_queue);
7570                                 }
7571                                 if (atomic_read(&rdev->irq.pflip[0]))
7572                                         radeon_crtc_handle_vblank(rdev, 0);
7573                                 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7574                                 DRM_DEBUG_VBLANK("IH: D1 vblank\n");
7575
7576                                 break;
7577                         case 1: /* D1 vline */
7578                                 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7579                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7580
7581                                 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7582                                 DRM_DEBUG_VBLANK("IH: D1 vline\n");
7583
7584                                 break;
7585                         default:
7586                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7587                                 break;
7588                         }
7589                         break;
7590                 case 2: /* D2 vblank/vline */
7591                         switch (src_data) {
7592                         case 0: /* D2 vblank */
7593                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7594                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7595
7596                                 if (rdev->irq.crtc_vblank_int[1]) {
7597                                         drm_handle_vblank(rdev->ddev, 1);
7598                                         rdev->pm.vblank_sync = true;
7599                                         wake_up(&rdev->irq.vblank_queue);
7600                                 }
7601                                 if (atomic_read(&rdev->irq.pflip[1]))
7602                                         radeon_crtc_handle_vblank(rdev, 1);
7603                                 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7604                                 DRM_DEBUG_VBLANK("IH: D2 vblank\n");
7605
7606                                 break;
7607                         case 1: /* D2 vline */
7608                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7609                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7610
7611                                 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7612                                 DRM_DEBUG_VBLANK("IH: D2 vline\n");
7613
7614                                 break;
7615                         default:
7616                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7617                                 break;
7618                         }
7619                         break;
7620                 case 3: /* D3 vblank/vline */
7621                         switch (src_data) {
7622                         case 0: /* D3 vblank */
7623                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7624                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7625
7626                                 if (rdev->irq.crtc_vblank_int[2]) {
7627                                         drm_handle_vblank(rdev->ddev, 2);
7628                                         rdev->pm.vblank_sync = true;
7629                                         wake_up(&rdev->irq.vblank_queue);
7630                                 }
7631                                 if (atomic_read(&rdev->irq.pflip[2]))
7632                                         radeon_crtc_handle_vblank(rdev, 2);
7633                                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7634                                 DRM_DEBUG_VBLANK("IH: D3 vblank\n");
7635
7636                                 break;
7637                         case 1: /* D3 vline */
7638                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7639                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7640
7641                                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7642                                 DRM_DEBUG_VBLANK("IH: D3 vline\n");
7643
7644                                 break;
7645                         default:
7646                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7647                                 break;
7648                         }
7649                         break;
7650                 case 4: /* D4 vblank/vline */
7651                         switch (src_data) {
7652                         case 0: /* D4 vblank */
7653                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7654                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7655
7656                                 if (rdev->irq.crtc_vblank_int[3]) {
7657                                         drm_handle_vblank(rdev->ddev, 3);
7658                                         rdev->pm.vblank_sync = true;
7659                                         wake_up(&rdev->irq.vblank_queue);
7660                                 }
7661                                 if (atomic_read(&rdev->irq.pflip[3]))
7662                                         radeon_crtc_handle_vblank(rdev, 3);
7663                                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7664                                 DRM_DEBUG_VBLANK("IH: D4 vblank\n");
7665
7666                                 break;
7667                         case 1: /* D4 vline */
7668                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7669                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7670
7671                                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7672                                 DRM_DEBUG_VBLANK("IH: D4 vline\n");
7673
7674                                 break;
7675                         default:
7676                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7677                                 break;
7678                         }
7679                         break;
7680                 case 5: /* D5 vblank/vline */
7681                         switch (src_data) {
7682                         case 0: /* D5 vblank */
7683                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7684                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7685
7686                                 if (rdev->irq.crtc_vblank_int[4]) {
7687                                         drm_handle_vblank(rdev->ddev, 4);
7688                                         rdev->pm.vblank_sync = true;
7689                                         wake_up(&rdev->irq.vblank_queue);
7690                                 }
7691                                 if (atomic_read(&rdev->irq.pflip[4]))
7692                                         radeon_crtc_handle_vblank(rdev, 4);
7693                                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7694                                 DRM_DEBUG_VBLANK("IH: D5 vblank\n");
7695
7696                                 break;
7697                         case 1: /* D5 vline */
7698                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7699                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7700
7701                                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7702                                 DRM_DEBUG("IH: D5 vline\n");
7703
7704                                 break;
7705                         default:
7706                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7707                                 break;
7708                         }
7709                         break;
7710                 case 6: /* D6 vblank/vline */
7711                         switch (src_data) {
7712                         case 0: /* D6 vblank */
7713                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7714                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7715
7716                                 if (rdev->irq.crtc_vblank_int[5]) {
7717                                         drm_handle_vblank(rdev->ddev, 5);
7718                                         rdev->pm.vblank_sync = true;
7719                                         wake_up(&rdev->irq.vblank_queue);
7720                                 }
7721                                 if (atomic_read(&rdev->irq.pflip[5]))
7722                                         radeon_crtc_handle_vblank(rdev, 5);
7723                                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7724                                 DRM_DEBUG_VBLANK("IH: D6 vblank\n");
7725
7726                                 break;
7727                         case 1: /* D6 vline */
7728                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7729                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7730
7731                                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7732                                 DRM_DEBUG_VBLANK("IH: D6 vline\n");
7733
7734                                 break;
7735                         default:
7736                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7737                                 break;
7738                         }
7739                         break;
7740                 case 8: /* D1 page flip */
7741                 case 10: /* D2 page flip */
7742                 case 12: /* D3 page flip */
7743                 case 14: /* D4 page flip */
7744                 case 16: /* D5 page flip */
7745                 case 18: /* D6 page flip */
7746                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7747                         if (radeon_use_pflipirq > 0)
7748                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7749                         break;
7750                 case 42: /* HPD hotplug */
7751                         switch (src_data) {
7752                         case 0:
7753                                 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7754                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7755
7756                                 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7757                                 queue_hotplug = true;
7758                                 DRM_DEBUG("IH: HPD1\n");
7759
7760                                 break;
7761                         case 1:
7762                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7763                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7764
7765                                 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7766                                 queue_hotplug = true;
7767                                 DRM_DEBUG("IH: HPD2\n");
7768
7769                                 break;
7770                         case 2:
7771                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7772                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7773
7774                                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7775                                 queue_hotplug = true;
7776                                 DRM_DEBUG("IH: HPD3\n");
7777
7778                                 break;
7779                         case 3:
7780                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7781                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7782
7783                                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7784                                 queue_hotplug = true;
7785                                 DRM_DEBUG("IH: HPD4\n");
7786
7787                                 break;
7788                         case 4:
7789                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7790                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7791
7792                                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7793                                 queue_hotplug = true;
7794                                 DRM_DEBUG("IH: HPD5\n");
7795
7796                                 break;
7797                         case 5:
7798                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7799                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7800
7801                                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7802                                 queue_hotplug = true;
7803                                 DRM_DEBUG("IH: HPD6\n");
7804
7805                                 break;
7806                         case 6:
7807                                 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7808                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7809
7810                                 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7811                                 queue_dp = true;
7812                                 DRM_DEBUG("IH: HPD_RX 1\n");
7813
7814                                 break;
7815                         case 7:
7816                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7817                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7818
7819                                 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7820                                 queue_dp = true;
7821                                 DRM_DEBUG("IH: HPD_RX 2\n");
7822
7823                                 break;
7824                         case 8:
7825                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7826                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7827
7828                                 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7829                                 queue_dp = true;
7830                                 DRM_DEBUG("IH: HPD_RX 3\n");
7831
7832                                 break;
7833                         case 9:
7834                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7835                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7836
7837                                 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7838                                 queue_dp = true;
7839                                 DRM_DEBUG("IH: HPD_RX 4\n");
7840
7841                                 break;
7842                         case 10:
7843                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7844                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7845
7846                                 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7847                                 queue_dp = true;
7848                                 DRM_DEBUG("IH: HPD_RX 5\n");
7849
7850                                 break;
7851                         case 11:
7852                                 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7853                                         DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7854
7855                                 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7856                                 queue_dp = true;
7857                                 DRM_DEBUG("IH: HPD_RX 6\n");
7858
7859                                 break;
7860                         default:
7861                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7862                                 break;
7863                         }
7864                         break;
7865                 case 96:
7866                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7867                         WREG32(SRBM_INT_ACK, 0x1);
7868                         break;
7869                 case 124: /* UVD */
7870                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7871                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7872                         break;
7873                 case 146:
7874                 case 147:
7875                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7876                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7877                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7878                         /* reset addr and status */
7879                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7880                         if (addr == 0x0 && status == 0x0)
7881                                 break;
7882                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7883                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7884                                 addr);
7885                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7886                                 status);
7887                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7888                         break;
7889                 case 167: /* VCE */
7890                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7891                         switch (src_data) {
7892                         case 0:
7893                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7894                                 break;
7895                         case 1:
7896                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7897                                 break;
7898                         default:
7899                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7900                                 break;
7901                         }
7902                         break;
7903                 case 176: /* GFX RB CP_INT */
7904                 case 177: /* GFX IB CP_INT */
7905                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7906                         break;
7907                 case 181: /* CP EOP event */
7908                         DRM_DEBUG("IH: CP EOP\n");
7909                         /* XXX check the bitfield order! */
7910                         me_id = (ring_id & 0x60) >> 5;
7911                         pipe_id = (ring_id & 0x18) >> 3;
7912                         queue_id = (ring_id & 0x7) >> 0;
7913                         switch (me_id) {
7914                         case 0:
7915                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7916                                 break;
7917                         case 1:
7918                         case 2:
7919                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7920                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7921                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7922                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7923                                 break;
7924                         }
7925                         break;
7926                 case 184: /* CP Privileged reg access */
7927                         DRM_ERROR("Illegal register access in command stream\n");
7928                         /* XXX check the bitfield order! */
7929                         me_id = (ring_id & 0x60) >> 5;
7930                         pipe_id = (ring_id & 0x18) >> 3;
7931                         queue_id = (ring_id & 0x7) >> 0;
7932                         switch (me_id) {
7933                         case 0:
7934                                 /* This results in a full GPU reset, but all we need to do is soft
7935                                  * reset the CP for gfx
7936                                  */
7937                                 queue_reset = true;
7938                                 break;
7939                         case 1:
7940                                 /* XXX compute */
7941                                 queue_reset = true;
7942                                 break;
7943                         case 2:
7944                                 /* XXX compute */
7945                                 queue_reset = true;
7946                                 break;
7947                         }
7948                         break;
7949                 case 185: /* CP Privileged inst */
7950                         DRM_ERROR("Illegal instruction in command stream\n");
7951                         /* XXX check the bitfield order! */
7952                         me_id = (ring_id & 0x60) >> 5;
7953                         pipe_id = (ring_id & 0x18) >> 3;
7954                         queue_id = (ring_id & 0x7) >> 0;
7955                         switch (me_id) {
7956                         case 0:
7957                                 /* This results in a full GPU reset, but all we need to do is soft
7958                                  * reset the CP for gfx
7959                                  */
7960                                 queue_reset = true;
7961                                 break;
7962                         case 1:
7963                                 /* XXX compute */
7964                                 queue_reset = true;
7965                                 break;
7966                         case 2:
7967                                 /* XXX compute */
7968                                 queue_reset = true;
7969                                 break;
7970                         }
7971                         break;
7972                 case 224: /* SDMA trap event */
7973                         /* XXX check the bitfield order! */
7974                         me_id = (ring_id & 0x3) >> 0;
7975                         queue_id = (ring_id & 0xc) >> 2;
7976                         DRM_DEBUG("IH: SDMA trap\n");
7977                         switch (me_id) {
7978                         case 0:
7979                                 switch (queue_id) {
7980                                 case 0:
7981                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7982                                         break;
7983                                 case 1:
7984                                         /* XXX compute */
7985                                         break;
7986                                 case 2:
7987                                         /* XXX compute */
7988                                         break;
7989                                 }
7990                                 break;
7991                         case 1:
7992                                 switch (queue_id) {
7993                                 case 0:
7994                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7995                                         break;
7996                                 case 1:
7997                                         /* XXX compute */
7998                                         break;
7999                                 case 2:
8000                                         /* XXX compute */
8001                                         break;
8002                                 }
8003                                 break;
8004                         }
8005                         break;
8006                 case 230: /* thermal low to high */
8007                         DRM_DEBUG("IH: thermal low to high\n");
8008                         rdev->pm.dpm.thermal.high_to_low = false;
8009                         queue_thermal = true;
8010                         break;
8011                 case 231: /* thermal high to low */
8012                         DRM_DEBUG("IH: thermal high to low\n");
8013                         rdev->pm.dpm.thermal.high_to_low = true;
8014                         queue_thermal = true;
8015                         break;
8016                 case 233: /* GUI IDLE */
8017                         DRM_DEBUG("IH: GUI idle\n");
8018                         break;
8019                 case 241: /* SDMA Privileged inst */
8020                 case 247: /* SDMA Privileged inst */
8021                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
8022                         /* XXX check the bitfield order! */
8023                         me_id = (ring_id & 0x3) >> 0;
8024                         queue_id = (ring_id & 0xc) >> 2;
8025                         switch (me_id) {
8026                         case 0:
8027                                 switch (queue_id) {
8028                                 case 0:
8029                                         queue_reset = true;
8030                                         break;
8031                                 case 1:
8032                                         /* XXX compute */
8033                                         queue_reset = true;
8034                                         break;
8035                                 case 2:
8036                                         /* XXX compute */
8037                                         queue_reset = true;
8038                                         break;
8039                                 }
8040                                 break;
8041                         case 1:
8042                                 switch (queue_id) {
8043                                 case 0:
8044                                         queue_reset = true;
8045                                         break;
8046                                 case 1:
8047                                         /* XXX compute */
8048                                         queue_reset = true;
8049                                         break;
8050                                 case 2:
8051                                         /* XXX compute */
8052                                         queue_reset = true;
8053                                         break;
8054                                 }
8055                                 break;
8056                         }
8057                         break;
8058                 default:
8059                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8060                         break;
8061                 }
8062
8063                 /* wptr/rptr are in bytes! */
8064                 rptr += 16;
8065                 rptr &= rdev->ih.ptr_mask;
8066                 WREG32(IH_RB_RPTR, rptr);
8067         }
8068         if (queue_dp)
8069                 schedule_work(&rdev->dp_work);
8070         if (queue_hotplug)
8071                 schedule_delayed_work(&rdev->hotplug_work, 0);
8072         if (queue_reset) {
8073                 rdev->needs_reset = true;
8074                 wake_up_all(&rdev->fence_queue);
8075         }
8076         if (queue_thermal)
8077                 schedule_work(&rdev->pm.dpm.thermal.work);
8078         rdev->ih.rptr = rptr;
8079         atomic_set(&rdev->ih.lock, 0);
8080
8081         /* make sure wptr hasn't changed while processing */
8082         wptr = cik_get_ih_wptr(rdev);
8083         if (wptr != rptr)
8084                 goto restart_ih;
8085
8086         return IRQ_HANDLED;
8087 }
8088
8089 /*
8090  * startup/shutdown callbacks
8091  */
8092 static void cik_uvd_init(struct radeon_device *rdev)
8093 {
8094         int r;
8095
8096         if (!rdev->has_uvd)
8097                 return;
8098
8099         r = radeon_uvd_init(rdev);
8100         if (r) {
8101                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8102                 /*
8103                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8104                  * to early fails cik_uvd_start() and thus nothing happens
8105                  * there. So it is pointless to try to go through that code
8106                  * hence why we disable uvd here.
8107                  */
8108                 rdev->has_uvd = 0;
8109                 return;
8110         }
8111         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8112         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8113 }
8114
8115 static void cik_uvd_start(struct radeon_device *rdev)
8116 {
8117         int r;
8118
8119         if (!rdev->has_uvd)
8120                 return;
8121
8122         r = radeon_uvd_resume(rdev);
8123         if (r) {
8124                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8125                 goto error;
8126         }
8127         r = uvd_v4_2_resume(rdev);
8128         if (r) {
8129                 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8130                 goto error;
8131         }
8132         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8133         if (r) {
8134                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8135                 goto error;
8136         }
8137         return;
8138
8139 error:
8140         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8141 }
8142
8143 static void cik_uvd_resume(struct radeon_device *rdev)
8144 {
8145         struct radeon_ring *ring;
8146         int r;
8147
8148         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8149                 return;
8150
8151         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8152         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
8153         if (r) {
8154                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8155                 return;
8156         }
8157         r = uvd_v1_0_init(rdev);
8158         if (r) {
8159                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8160                 return;
8161         }
8162 }
8163
8164 static void cik_vce_init(struct radeon_device *rdev)
8165 {
8166         int r;
8167
8168         if (!rdev->has_vce)
8169                 return;
8170
8171         r = radeon_vce_init(rdev);
8172         if (r) {
8173                 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8174                 /*
8175                  * At this point rdev->vce.vcpu_bo is NULL which trickles down
8176                  * to early fails cik_vce_start() and thus nothing happens
8177                  * there. So it is pointless to try to go through that code
8178                  * hence why we disable vce here.
8179                  */
8180                 rdev->has_vce = 0;
8181                 return;
8182         }
8183         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8184         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8185         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8186         r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8187 }
8188
8189 static void cik_vce_start(struct radeon_device *rdev)
8190 {
8191         int r;
8192
8193         if (!rdev->has_vce)
8194                 return;
8195
8196         r = radeon_vce_resume(rdev);
8197         if (r) {
8198                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8199                 goto error;
8200         }
8201         r = vce_v2_0_resume(rdev);
8202         if (r) {
8203                 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8204                 goto error;
8205         }
8206         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8207         if (r) {
8208                 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8209                 goto error;
8210         }
8211         r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8212         if (r) {
8213                 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8214                 goto error;
8215         }
8216         return;
8217
8218 error:
8219         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8220         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8221 }
8222
8223 static void cik_vce_resume(struct radeon_device *rdev)
8224 {
8225         struct radeon_ring *ring;
8226         int r;
8227
8228         if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8229                 return;
8230
8231         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8232         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8233         if (r) {
8234                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8235                 return;
8236         }
8237         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8238         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8239         if (r) {
8240                 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8241                 return;
8242         }
8243         r = vce_v1_0_init(rdev);
8244         if (r) {
8245                 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8246                 return;
8247         }
8248 }
8249
8250 /**
8251  * cik_startup - program the asic to a functional state
8252  *
8253  * @rdev: radeon_device pointer
8254  *
8255  * Programs the asic to a functional state (CIK).
8256  * Called by cik_init() and cik_resume().
8257  * Returns 0 for success, error for failure.
8258  */
8259 static int cik_startup(struct radeon_device *rdev)
8260 {
8261         struct radeon_ring *ring;
8262         u32 nop;
8263         int r;
8264
8265         /* enable pcie gen2/3 link */
8266         cik_pcie_gen3_enable(rdev);
8267         /* enable aspm */
8268         cik_program_aspm(rdev);
8269
8270         /* scratch needs to be initialized before MC */
8271         r = r600_vram_scratch_init(rdev);
8272         if (r)
8273                 return r;
8274
8275         cik_mc_program(rdev);
8276
8277         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8278                 r = ci_mc_load_microcode(rdev);
8279                 if (r) {
8280                         DRM_ERROR("Failed to load MC firmware!\n");
8281                         return r;
8282                 }
8283         }
8284
8285         r = cik_pcie_gart_enable(rdev);
8286         if (r)
8287                 return r;
8288         cik_gpu_init(rdev);
8289
8290         /* allocate rlc buffers */
8291         if (rdev->flags & RADEON_IS_IGP) {
8292                 if (rdev->family == CHIP_KAVERI) {
8293                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8294                         rdev->rlc.reg_list_size =
8295                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8296                 } else {
8297                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8298                         rdev->rlc.reg_list_size =
8299                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8300                 }
8301         }
8302         rdev->rlc.cs_data = ci_cs_data;
8303         rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
8304         rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
8305         r = sumo_rlc_init(rdev);
8306         if (r) {
8307                 DRM_ERROR("Failed to init rlc BOs!\n");
8308                 return r;
8309         }
8310
8311         /* allocate wb buffer */
8312         r = radeon_wb_init(rdev);
8313         if (r)
8314                 return r;
8315
8316         /* allocate mec buffers */
8317         r = cik_mec_init(rdev);
8318         if (r) {
8319                 DRM_ERROR("Failed to init MEC BOs!\n");
8320                 return r;
8321         }
8322
8323         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8324         if (r) {
8325                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8326                 return r;
8327         }
8328
8329         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8330         if (r) {
8331                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8332                 return r;
8333         }
8334
8335         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8336         if (r) {
8337                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8338                 return r;
8339         }
8340
8341         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8342         if (r) {
8343                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8344                 return r;
8345         }
8346
8347         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8348         if (r) {
8349                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8350                 return r;
8351         }
8352
8353         cik_uvd_start(rdev);
8354         cik_vce_start(rdev);
8355
8356         /* Enable IRQ */
8357         if (!rdev->irq.installed) {
8358                 r = radeon_irq_kms_init(rdev);
8359                 if (r)
8360                         return r;
8361         }
8362
8363         r = cik_irq_init(rdev);
8364         if (r) {
8365                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8366                 radeon_irq_kms_fini(rdev);
8367                 return r;
8368         }
8369         cik_irq_set(rdev);
8370
8371         if (rdev->family == CHIP_HAWAII) {
8372                 if (rdev->new_fw)
8373                         nop = PACKET3(PACKET3_NOP, 0x3FFF);
8374                 else
8375                         nop = RADEON_CP_PACKET2;
8376         } else {
8377                 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8378         }
8379
8380         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8381         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8382                              nop);
8383         if (r)
8384                 return r;
8385
8386         /* set up the compute queues */
8387         /* type-2 packets are deprecated on MEC, use type-3 instead */
8388         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8389         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8390                              nop);
8391         if (r)
8392                 return r;
8393         ring->me = 1; /* first MEC */
8394         ring->pipe = 0; /* first pipe */
8395         ring->queue = 0; /* first queue */
8396         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8397
8398         /* type-2 packets are deprecated on MEC, use type-3 instead */
8399         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8400         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8401                              nop);
8402         if (r)
8403                 return r;
8404         /* dGPU only have 1 MEC */
8405         ring->me = 1; /* first MEC */
8406         ring->pipe = 0; /* first pipe */
8407         ring->queue = 1; /* second queue */
8408         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8409
8410         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8411         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8412                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8413         if (r)
8414                 return r;
8415
8416         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8417         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8418                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8419         if (r)
8420                 return r;
8421
8422         r = cik_cp_resume(rdev);
8423         if (r)
8424                 return r;
8425
8426         r = cik_sdma_resume(rdev);
8427         if (r)
8428                 return r;
8429
8430         cik_uvd_resume(rdev);
8431         cik_vce_resume(rdev);
8432
8433         r = radeon_ib_pool_init(rdev);
8434         if (r) {
8435                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8436                 return r;
8437         }
8438
8439         r = radeon_vm_manager_init(rdev);
8440         if (r) {
8441                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8442                 return r;
8443         }
8444
8445         r = radeon_audio_init(rdev);
8446         if (r)
8447                 return r;
8448
8449         return 0;
8450 }
8451
8452 /**
8453  * cik_resume - resume the asic to a functional state
8454  *
8455  * @rdev: radeon_device pointer
8456  *
8457  * Programs the asic to a functional state (CIK).
8458  * Called at resume.
8459  * Returns 0 for success, error for failure.
8460  */
8461 int cik_resume(struct radeon_device *rdev)
8462 {
8463         int r;
8464
8465         /* post card */
8466         atom_asic_init(rdev->mode_info.atom_context);
8467
8468         /* init golden registers */
8469         cik_init_golden_registers(rdev);
8470
8471         if (rdev->pm.pm_method == PM_METHOD_DPM)
8472                 radeon_pm_resume(rdev);
8473
8474         rdev->accel_working = true;
8475         r = cik_startup(rdev);
8476         if (r) {
8477                 DRM_ERROR("cik startup failed on resume\n");
8478                 rdev->accel_working = false;
8479                 return r;
8480         }
8481
8482         return r;
8483
8484 }
8485
8486 /**
8487  * cik_suspend - suspend the asic
8488  *
8489  * @rdev: radeon_device pointer
8490  *
8491  * Bring the chip into a state suitable for suspend (CIK).
8492  * Called at suspend.
8493  * Returns 0 for success.
8494  */
8495 int cik_suspend(struct radeon_device *rdev)
8496 {
8497         radeon_pm_suspend(rdev);
8498         radeon_audio_fini(rdev);
8499         radeon_vm_manager_fini(rdev);
8500         cik_cp_enable(rdev, false);
8501         cik_sdma_enable(rdev, false);
8502         if (rdev->has_uvd) {
8503                 uvd_v1_0_fini(rdev);
8504                 radeon_uvd_suspend(rdev);
8505         }
8506         if (rdev->has_vce)
8507                 radeon_vce_suspend(rdev);
8508         cik_fini_pg(rdev);
8509         cik_fini_cg(rdev);
8510         cik_irq_suspend(rdev);
8511         radeon_wb_disable(rdev);
8512         cik_pcie_gart_disable(rdev);
8513         return 0;
8514 }
8515
8516 /* Plan is to move initialization in that function and use
8517  * helper function so that radeon_device_init pretty much
8518  * do nothing more than calling asic specific function. This
8519  * should also allow to remove a bunch of callback function
8520  * like vram_info.
8521  */
8522 /**
8523  * cik_init - asic specific driver and hw init
8524  *
8525  * @rdev: radeon_device pointer
8526  *
8527  * Setup asic specific driver variables and program the hw
8528  * to a functional state (CIK).
8529  * Called at driver startup.
8530  * Returns 0 for success, errors for failure.
8531  */
8532 int cik_init(struct radeon_device *rdev)
8533 {
8534         struct radeon_ring *ring;
8535         int r;
8536
8537         /* Read BIOS */
8538         if (!radeon_get_bios(rdev)) {
8539                 if (ASIC_IS_AVIVO(rdev))
8540                         return -EINVAL;
8541         }
8542         /* Must be an ATOMBIOS */
8543         if (!rdev->is_atom_bios) {
8544                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8545                 return -EINVAL;
8546         }
8547         r = radeon_atombios_init(rdev);
8548         if (r)
8549                 return r;
8550
8551         /* Post card if necessary */
8552         if (!radeon_card_posted(rdev)) {
8553                 if (!rdev->bios) {
8554                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8555                         return -EINVAL;
8556                 }
8557                 DRM_INFO("GPU not posted. posting now...\n");
8558                 atom_asic_init(rdev->mode_info.atom_context);
8559         }
8560         /* init golden registers */
8561         cik_init_golden_registers(rdev);
8562         /* Initialize scratch registers */
8563         cik_scratch_init(rdev);
8564         /* Initialize surface registers */
8565         radeon_surface_init(rdev);
8566         /* Initialize clocks */
8567         radeon_get_clock_info(rdev->ddev);
8568
8569         /* Fence driver */
8570         r = radeon_fence_driver_init(rdev);
8571         if (r)
8572                 return r;
8573
8574         /* initialize memory controller */
8575         r = cik_mc_init(rdev);
8576         if (r)
8577                 return r;
8578         /* Memory manager */
8579         r = radeon_bo_init(rdev);
8580         if (r)
8581                 return r;
8582
8583         if (rdev->flags & RADEON_IS_IGP) {
8584                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8585                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8586                         r = cik_init_microcode(rdev);
8587                         if (r) {
8588                                 DRM_ERROR("Failed to load firmware!\n");
8589                                 return r;
8590                         }
8591                 }
8592         } else {
8593                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8594                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8595                     !rdev->mc_fw) {
8596                         r = cik_init_microcode(rdev);
8597                         if (r) {
8598                                 DRM_ERROR("Failed to load firmware!\n");
8599                                 return r;
8600                         }
8601                 }
8602         }
8603
8604         /* Initialize power management */
8605         radeon_pm_init(rdev);
8606
8607         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8608         ring->ring_obj = NULL;
8609         r600_ring_init(rdev, ring, 1024 * 1024);
8610
8611         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8612         ring->ring_obj = NULL;
8613         r600_ring_init(rdev, ring, 1024 * 1024);
8614         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8615         if (r)
8616                 return r;
8617
8618         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8619         ring->ring_obj = NULL;
8620         r600_ring_init(rdev, ring, 1024 * 1024);
8621         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8622         if (r)
8623                 return r;
8624
8625         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8626         ring->ring_obj = NULL;
8627         r600_ring_init(rdev, ring, 256 * 1024);
8628
8629         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8630         ring->ring_obj = NULL;
8631         r600_ring_init(rdev, ring, 256 * 1024);
8632
8633         cik_uvd_init(rdev);
8634         cik_vce_init(rdev);
8635
8636         rdev->ih.ring_obj = NULL;
8637         r600_ih_ring_init(rdev, 64 * 1024);
8638
8639         r = r600_pcie_gart_init(rdev);
8640         if (r)
8641                 return r;
8642
8643         rdev->accel_working = true;
8644         r = cik_startup(rdev);
8645         if (r) {
8646                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8647                 cik_cp_fini(rdev);
8648                 cik_sdma_fini(rdev);
8649                 cik_irq_fini(rdev);
8650                 sumo_rlc_fini(rdev);
8651                 cik_mec_fini(rdev);
8652                 radeon_wb_fini(rdev);
8653                 radeon_ib_pool_fini(rdev);
8654                 radeon_vm_manager_fini(rdev);
8655                 radeon_irq_kms_fini(rdev);
8656                 cik_pcie_gart_fini(rdev);
8657                 rdev->accel_working = false;
8658         }
8659
8660         /* Don't start up if the MC ucode is missing.
8661          * The default clocks and voltages before the MC ucode
8662          * is loaded are not suffient for advanced operations.
8663          */
8664         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8665                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8666                 return -EINVAL;
8667         }
8668
8669         return 0;
8670 }
8671
8672 /**
8673  * cik_fini - asic specific driver and hw fini
8674  *
8675  * @rdev: radeon_device pointer
8676  *
8677  * Tear down the asic specific driver variables and program the hw
8678  * to an idle state (CIK).
8679  * Called at driver unload.
8680  */
8681 void cik_fini(struct radeon_device *rdev)
8682 {
8683         radeon_pm_fini(rdev);
8684         cik_cp_fini(rdev);
8685         cik_sdma_fini(rdev);
8686         cik_fini_pg(rdev);
8687         cik_fini_cg(rdev);
8688         cik_irq_fini(rdev);
8689         sumo_rlc_fini(rdev);
8690         cik_mec_fini(rdev);
8691         radeon_wb_fini(rdev);
8692         radeon_vm_manager_fini(rdev);
8693         radeon_ib_pool_fini(rdev);
8694         radeon_irq_kms_fini(rdev);
8695         uvd_v1_0_fini(rdev);
8696         radeon_uvd_fini(rdev);
8697         radeon_vce_fini(rdev);
8698         cik_pcie_gart_fini(rdev);
8699         r600_vram_scratch_fini(rdev);
8700         radeon_gem_fini(rdev);
8701         radeon_fence_driver_fini(rdev);
8702         radeon_bo_fini(rdev);
8703         radeon_atombios_fini(rdev);
8704         cik_fini_microcode(rdev);
8705         kfree(rdev->bios);
8706         rdev->bios = NULL;
8707 }
8708
8709 void dce8_program_fmt(struct drm_encoder *encoder)
8710 {
8711         struct drm_device *dev = encoder->dev;
8712         struct radeon_device *rdev = dev->dev_private;
8713         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8714         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8715         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8716         int bpc = 0;
8717         u32 tmp = 0;
8718         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8719
8720         if (connector) {
8721                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8722                 bpc = radeon_get_monitor_bpc(connector);
8723                 dither = radeon_connector->dither;
8724         }
8725
8726         /* LVDS/eDP FMT is set up by atom */
8727         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8728                 return;
8729
8730         /* not needed for analog */
8731         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8732             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8733                 return;
8734
8735         if (bpc == 0)
8736                 return;
8737
8738         switch (bpc) {
8739         case 6:
8740                 if (dither == RADEON_FMT_DITHER_ENABLE)
8741                         /* XXX sort out optimal dither settings */
8742                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8743                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8744                 else
8745                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8746                 break;
8747         case 8:
8748                 if (dither == RADEON_FMT_DITHER_ENABLE)
8749                         /* XXX sort out optimal dither settings */
8750                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8751                                 FMT_RGB_RANDOM_ENABLE |
8752                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8753                 else
8754                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8755                 break;
8756         case 10:
8757                 if (dither == RADEON_FMT_DITHER_ENABLE)
8758                         /* XXX sort out optimal dither settings */
8759                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8760                                 FMT_RGB_RANDOM_ENABLE |
8761                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8762                 else
8763                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8764                 break;
8765         default:
8766                 /* not needed */
8767                 break;
8768         }
8769
8770         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8771 }
8772
8773 /* display watermark setup */
8774 /**
8775  * dce8_line_buffer_adjust - Set up the line buffer
8776  *
8777  * @rdev: radeon_device pointer
8778  * @radeon_crtc: the selected display controller
8779  * @mode: the current display mode on the selected display
8780  * controller
8781  *
8782  * Setup up the line buffer allocation for
8783  * the selected display controller (CIK).
8784  * Returns the line buffer size in pixels.
8785  */
8786 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8787                                    struct radeon_crtc *radeon_crtc,
8788                                    struct drm_display_mode *mode)
8789 {
8790         u32 tmp, buffer_alloc, i;
8791         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8792         /*
8793          * Line Buffer Setup
8794          * There are 6 line buffers, one for each display controllers.
8795          * There are 3 partitions per LB. Select the number of partitions
8796          * to enable based on the display width.  For display widths larger
8797          * than 4096, you need use to use 2 display controllers and combine
8798          * them using the stereo blender.
8799          */
8800         if (radeon_crtc->base.enabled && mode) {
8801                 if (mode->crtc_hdisplay < 1920) {
8802                         tmp = 1;
8803                         buffer_alloc = 2;
8804                 } else if (mode->crtc_hdisplay < 2560) {
8805                         tmp = 2;
8806                         buffer_alloc = 2;
8807                 } else if (mode->crtc_hdisplay < 4096) {
8808                         tmp = 0;
8809                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8810                 } else {
8811                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8812                         tmp = 0;
8813                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8814                 }
8815         } else {
8816                 tmp = 1;
8817                 buffer_alloc = 0;
8818         }
8819
8820         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8821                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8822
8823         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8824                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8825         for (i = 0; i < rdev->usec_timeout; i++) {
8826                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8827                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8828                         break;
8829                 udelay(1);
8830         }
8831
8832         if (radeon_crtc->base.enabled && mode) {
8833                 switch (tmp) {
8834                 case 0:
8835                 default:
8836                         return 4096 * 2;
8837                 case 1:
8838                         return 1920 * 2;
8839                 case 2:
8840                         return 2560 * 2;
8841                 }
8842         }
8843
8844         /* controller not enabled, so no lb used */
8845         return 0;
8846 }
8847
8848 /**
8849  * cik_get_number_of_dram_channels - get the number of dram channels
8850  *
8851  * @rdev: radeon_device pointer
8852  *
8853  * Look up the number of video ram channels (CIK).
8854  * Used for display watermark bandwidth calculations
8855  * Returns the number of dram channels
8856  */
8857 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8858 {
8859         u32 tmp = RREG32(MC_SHARED_CHMAP);
8860
8861         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8862         case 0:
8863         default:
8864                 return 1;
8865         case 1:
8866                 return 2;
8867         case 2:
8868                 return 4;
8869         case 3:
8870                 return 8;
8871         case 4:
8872                 return 3;
8873         case 5:
8874                 return 6;
8875         case 6:
8876                 return 10;
8877         case 7:
8878                 return 12;
8879         case 8:
8880                 return 16;
8881         }
8882 }
8883
8884 struct dce8_wm_params {
8885         u32 dram_channels; /* number of dram channels */
8886         u32 yclk;          /* bandwidth per dram data pin in kHz */
8887         u32 sclk;          /* engine clock in kHz */
8888         u32 disp_clk;      /* display clock in kHz */
8889         u32 src_width;     /* viewport width */
8890         u32 active_time;   /* active display time in ns */
8891         u32 blank_time;    /* blank time in ns */
8892         bool interlaced;    /* mode is interlaced */
8893         fixed20_12 vsc;    /* vertical scale ratio */
8894         u32 num_heads;     /* number of active crtcs */
8895         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8896         u32 lb_size;       /* line buffer allocated to pipe */
8897         u32 vtaps;         /* vertical scaler taps */
8898 };
8899
8900 /**
8901  * dce8_dram_bandwidth - get the dram bandwidth
8902  *
8903  * @wm: watermark calculation data
8904  *
8905  * Calculate the raw dram bandwidth (CIK).
8906  * Used for display watermark bandwidth calculations
8907  * Returns the dram bandwidth in MBytes/s
8908  */
8909 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8910 {
8911         /* Calculate raw DRAM Bandwidth */
8912         fixed20_12 dram_efficiency; /* 0.7 */
8913         fixed20_12 yclk, dram_channels, bandwidth;
8914         fixed20_12 a;
8915
8916         a.full = dfixed_const(1000);
8917         yclk.full = dfixed_const(wm->yclk);
8918         yclk.full = dfixed_div(yclk, a);
8919         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8920         a.full = dfixed_const(10);
8921         dram_efficiency.full = dfixed_const(7);
8922         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8923         bandwidth.full = dfixed_mul(dram_channels, yclk);
8924         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8925
8926         return dfixed_trunc(bandwidth);
8927 }
8928
8929 /**
8930  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8931  *
8932  * @wm: watermark calculation data
8933  *
8934  * Calculate the dram bandwidth used for display (CIK).
8935  * Used for display watermark bandwidth calculations
8936  * Returns the dram bandwidth for display in MBytes/s
8937  */
8938 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8939 {
8940         /* Calculate DRAM Bandwidth and the part allocated to display. */
8941         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8942         fixed20_12 yclk, dram_channels, bandwidth;
8943         fixed20_12 a;
8944
8945         a.full = dfixed_const(1000);
8946         yclk.full = dfixed_const(wm->yclk);
8947         yclk.full = dfixed_div(yclk, a);
8948         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8949         a.full = dfixed_const(10);
8950         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8951         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8952         bandwidth.full = dfixed_mul(dram_channels, yclk);
8953         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8954
8955         return dfixed_trunc(bandwidth);
8956 }
8957
8958 /**
8959  * dce8_data_return_bandwidth - get the data return bandwidth
8960  *
8961  * @wm: watermark calculation data
8962  *
8963  * Calculate the data return bandwidth used for display (CIK).
8964  * Used for display watermark bandwidth calculations
8965  * Returns the data return bandwidth in MBytes/s
8966  */
8967 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8968 {
8969         /* Calculate the display Data return Bandwidth */
8970         fixed20_12 return_efficiency; /* 0.8 */
8971         fixed20_12 sclk, bandwidth;
8972         fixed20_12 a;
8973
8974         a.full = dfixed_const(1000);
8975         sclk.full = dfixed_const(wm->sclk);
8976         sclk.full = dfixed_div(sclk, a);
8977         a.full = dfixed_const(10);
8978         return_efficiency.full = dfixed_const(8);
8979         return_efficiency.full = dfixed_div(return_efficiency, a);
8980         a.full = dfixed_const(32);
8981         bandwidth.full = dfixed_mul(a, sclk);
8982         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8983
8984         return dfixed_trunc(bandwidth);
8985 }
8986
8987 /**
8988  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8989  *
8990  * @wm: watermark calculation data
8991  *
8992  * Calculate the dmif bandwidth used for display (CIK).
8993  * Used for display watermark bandwidth calculations
8994  * Returns the dmif bandwidth in MBytes/s
8995  */
8996 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8997 {
8998         /* Calculate the DMIF Request Bandwidth */
8999         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9000         fixed20_12 disp_clk, bandwidth;
9001         fixed20_12 a, b;
9002
9003         a.full = dfixed_const(1000);
9004         disp_clk.full = dfixed_const(wm->disp_clk);
9005         disp_clk.full = dfixed_div(disp_clk, a);
9006         a.full = dfixed_const(32);
9007         b.full = dfixed_mul(a, disp_clk);
9008
9009         a.full = dfixed_const(10);
9010         disp_clk_request_efficiency.full = dfixed_const(8);
9011         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9012
9013         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9014
9015         return dfixed_trunc(bandwidth);
9016 }
9017
9018 /**
9019  * dce8_available_bandwidth - get the min available bandwidth
9020  *
9021  * @wm: watermark calculation data
9022  *
9023  * Calculate the min available bandwidth used for display (CIK).
9024  * Used for display watermark bandwidth calculations
9025  * Returns the min available bandwidth in MBytes/s
9026  */
9027 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9028 {
9029         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9030         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9031         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9032         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9033
9034         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9035 }
9036
9037 /**
9038  * dce8_average_bandwidth - get the average available bandwidth
9039  *
9040  * @wm: watermark calculation data
9041  *
9042  * Calculate the average available bandwidth used for display (CIK).
9043  * Used for display watermark bandwidth calculations
9044  * Returns the average available bandwidth in MBytes/s
9045  */
9046 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9047 {
9048         /* Calculate the display mode Average Bandwidth
9049          * DisplayMode should contain the source and destination dimensions,
9050          * timing, etc.
9051          */
9052         fixed20_12 bpp;
9053         fixed20_12 line_time;
9054         fixed20_12 src_width;
9055         fixed20_12 bandwidth;
9056         fixed20_12 a;
9057
9058         a.full = dfixed_const(1000);
9059         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9060         line_time.full = dfixed_div(line_time, a);
9061         bpp.full = dfixed_const(wm->bytes_per_pixel);
9062         src_width.full = dfixed_const(wm->src_width);
9063         bandwidth.full = dfixed_mul(src_width, bpp);
9064         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9065         bandwidth.full = dfixed_div(bandwidth, line_time);
9066
9067         return dfixed_trunc(bandwidth);
9068 }
9069
9070 /**
9071  * dce8_latency_watermark - get the latency watermark
9072  *
9073  * @wm: watermark calculation data
9074  *
9075  * Calculate the latency watermark (CIK).
9076  * Used for display watermark bandwidth calculations
9077  * Returns the latency watermark in ns
9078  */
9079 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9080 {
9081         /* First calculate the latency in ns */
9082         u32 mc_latency = 2000; /* 2000 ns. */
9083         u32 available_bandwidth = dce8_available_bandwidth(wm);
9084         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9085         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9086         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9087         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9088                 (wm->num_heads * cursor_line_pair_return_time);
9089         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9090         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9091         u32 tmp, dmif_size = 12288;
9092         fixed20_12 a, b, c;
9093
9094         if (wm->num_heads == 0)
9095                 return 0;
9096
9097         a.full = dfixed_const(2);
9098         b.full = dfixed_const(1);
9099         if ((wm->vsc.full > a.full) ||
9100             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9101             (wm->vtaps >= 5) ||
9102             ((wm->vsc.full >= a.full) && wm->interlaced))
9103                 max_src_lines_per_dst_line = 4;
9104         else
9105                 max_src_lines_per_dst_line = 2;
9106
9107         a.full = dfixed_const(available_bandwidth);
9108         b.full = dfixed_const(wm->num_heads);
9109         a.full = dfixed_div(a, b);
9110         tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
9111         tmp = min(dfixed_trunc(a), tmp);
9112
9113         lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
9114
9115         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9116         b.full = dfixed_const(1000);
9117         c.full = dfixed_const(lb_fill_bw);
9118         b.full = dfixed_div(c, b);
9119         a.full = dfixed_div(a, b);
9120         line_fill_time = dfixed_trunc(a);
9121
9122         if (line_fill_time < wm->active_time)
9123                 return latency;
9124         else
9125                 return latency + (line_fill_time - wm->active_time);
9126
9127 }
9128
9129 /**
9130  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9131  * average and available dram bandwidth
9132  *
9133  * @wm: watermark calculation data
9134  *
9135  * Check if the display average bandwidth fits in the display
9136  * dram bandwidth (CIK).
9137  * Used for display watermark bandwidth calculations
9138  * Returns true if the display fits, false if not.
9139  */
9140 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9141 {
9142         if (dce8_average_bandwidth(wm) <=
9143             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9144                 return true;
9145         else
9146                 return false;
9147 }
9148
9149 /**
9150  * dce8_average_bandwidth_vs_available_bandwidth - check
9151  * average and available bandwidth
9152  *
9153  * @wm: watermark calculation data
9154  *
9155  * Check if the display average bandwidth fits in the display
9156  * available bandwidth (CIK).
9157  * Used for display watermark bandwidth calculations
9158  * Returns true if the display fits, false if not.
9159  */
9160 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9161 {
9162         if (dce8_average_bandwidth(wm) <=
9163             (dce8_available_bandwidth(wm) / wm->num_heads))
9164                 return true;
9165         else
9166                 return false;
9167 }
9168
9169 /**
9170  * dce8_check_latency_hiding - check latency hiding
9171  *
9172  * @wm: watermark calculation data
9173  *
9174  * Check latency hiding (CIK).
9175  * Used for display watermark bandwidth calculations
9176  * Returns true if the display fits, false if not.
9177  */
9178 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9179 {
9180         u32 lb_partitions = wm->lb_size / wm->src_width;
9181         u32 line_time = wm->active_time + wm->blank_time;
9182         u32 latency_tolerant_lines;
9183         u32 latency_hiding;
9184         fixed20_12 a;
9185
9186         a.full = dfixed_const(1);
9187         if (wm->vsc.full > a.full)
9188                 latency_tolerant_lines = 1;
9189         else {
9190                 if (lb_partitions <= (wm->vtaps + 1))
9191                         latency_tolerant_lines = 1;
9192                 else
9193                         latency_tolerant_lines = 2;
9194         }
9195
9196         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9197
9198         if (dce8_latency_watermark(wm) <= latency_hiding)
9199                 return true;
9200         else
9201                 return false;
9202 }
9203
9204 /**
9205  * dce8_program_watermarks - program display watermarks
9206  *
9207  * @rdev: radeon_device pointer
9208  * @radeon_crtc: the selected display controller
9209  * @lb_size: line buffer size
9210  * @num_heads: number of display controllers in use
9211  *
9212  * Calculate and program the display watermarks for the
9213  * selected display controller (CIK).
9214  */
9215 static void dce8_program_watermarks(struct radeon_device *rdev,
9216                                     struct radeon_crtc *radeon_crtc,
9217                                     u32 lb_size, u32 num_heads)
9218 {
9219         struct drm_display_mode *mode = &radeon_crtc->base.mode;
9220         struct dce8_wm_params wm_low, wm_high;
9221         u32 active_time;
9222         u32 line_time = 0;
9223         u32 latency_watermark_a = 0, latency_watermark_b = 0;
9224         u32 tmp, wm_mask;
9225
9226         if (radeon_crtc->base.enabled && num_heads && mode) {
9227                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
9228                                             (u32)mode->clock);
9229                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
9230                                           (u32)mode->clock);
9231                 line_time = min(line_time, (u32)65535);
9232
9233                 /* watermark for high clocks */
9234                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9235                     rdev->pm.dpm_enabled) {
9236                         wm_high.yclk =
9237                                 radeon_dpm_get_mclk(rdev, false) * 10;
9238                         wm_high.sclk =
9239                                 radeon_dpm_get_sclk(rdev, false) * 10;
9240                 } else {
9241                         wm_high.yclk = rdev->pm.current_mclk * 10;
9242                         wm_high.sclk = rdev->pm.current_sclk * 10;
9243                 }
9244
9245                 wm_high.disp_clk = mode->clock;
9246                 wm_high.src_width = mode->crtc_hdisplay;
9247                 wm_high.active_time = active_time;
9248                 wm_high.blank_time = line_time - wm_high.active_time;
9249                 wm_high.interlaced = false;
9250                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9251                         wm_high.interlaced = true;
9252                 wm_high.vsc = radeon_crtc->vsc;
9253                 wm_high.vtaps = 1;
9254                 if (radeon_crtc->rmx_type != RMX_OFF)
9255                         wm_high.vtaps = 2;
9256                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9257                 wm_high.lb_size = lb_size;
9258                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9259                 wm_high.num_heads = num_heads;
9260
9261                 /* set for high clocks */
9262                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9263
9264                 /* possibly force display priority to high */
9265                 /* should really do this at mode validation time... */
9266                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9267                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9268                     !dce8_check_latency_hiding(&wm_high) ||
9269                     (rdev->disp_priority == 2)) {
9270                         DRM_DEBUG_KMS("force priority to high\n");
9271                 }
9272
9273                 /* watermark for low clocks */
9274                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9275                     rdev->pm.dpm_enabled) {
9276                         wm_low.yclk =
9277                                 radeon_dpm_get_mclk(rdev, true) * 10;
9278                         wm_low.sclk =
9279                                 radeon_dpm_get_sclk(rdev, true) * 10;
9280                 } else {
9281                         wm_low.yclk = rdev->pm.current_mclk * 10;
9282                         wm_low.sclk = rdev->pm.current_sclk * 10;
9283                 }
9284
9285                 wm_low.disp_clk = mode->clock;
9286                 wm_low.src_width = mode->crtc_hdisplay;
9287                 wm_low.active_time = active_time;
9288                 wm_low.blank_time = line_time - wm_low.active_time;
9289                 wm_low.interlaced = false;
9290                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9291                         wm_low.interlaced = true;
9292                 wm_low.vsc = radeon_crtc->vsc;
9293                 wm_low.vtaps = 1;
9294                 if (radeon_crtc->rmx_type != RMX_OFF)
9295                         wm_low.vtaps = 2;
9296                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9297                 wm_low.lb_size = lb_size;
9298                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9299                 wm_low.num_heads = num_heads;
9300
9301                 /* set for low clocks */
9302                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9303
9304                 /* possibly force display priority to high */
9305                 /* should really do this at mode validation time... */
9306                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9307                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9308                     !dce8_check_latency_hiding(&wm_low) ||
9309                     (rdev->disp_priority == 2)) {
9310                         DRM_DEBUG_KMS("force priority to high\n");
9311                 }
9312
9313                 /* Save number of lines the linebuffer leads before the scanout */
9314                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
9315         }
9316
9317         /* select wm A */
9318         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9319         tmp = wm_mask;
9320         tmp &= ~LATENCY_WATERMARK_MASK(3);
9321         tmp |= LATENCY_WATERMARK_MASK(1);
9322         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9323         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9324                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9325                 LATENCY_HIGH_WATERMARK(line_time)));
9326         /* select wm B */
9327         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9328         tmp &= ~LATENCY_WATERMARK_MASK(3);
9329         tmp |= LATENCY_WATERMARK_MASK(2);
9330         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9331         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9332                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9333                 LATENCY_HIGH_WATERMARK(line_time)));
9334         /* restore original selection */
9335         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9336
9337         /* save values for DPM */
9338         radeon_crtc->line_time = line_time;
9339         radeon_crtc->wm_high = latency_watermark_a;
9340         radeon_crtc->wm_low = latency_watermark_b;
9341 }
9342
9343 /**
9344  * dce8_bandwidth_update - program display watermarks
9345  *
9346  * @rdev: radeon_device pointer
9347  *
9348  * Calculate and program the display watermarks and line
9349  * buffer allocation (CIK).
9350  */
9351 void dce8_bandwidth_update(struct radeon_device *rdev)
9352 {
9353         struct drm_display_mode *mode = NULL;
9354         u32 num_heads = 0, lb_size;
9355         int i;
9356
9357         if (!rdev->mode_info.mode_config_initialized)
9358                 return;
9359
9360         radeon_update_display_priority(rdev);
9361
9362         for (i = 0; i < rdev->num_crtc; i++) {
9363                 if (rdev->mode_info.crtcs[i]->base.enabled)
9364                         num_heads++;
9365         }
9366         for (i = 0; i < rdev->num_crtc; i++) {
9367                 mode = &rdev->mode_info.crtcs[i]->base.mode;
9368                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9369                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9370         }
9371 }
9372
9373 /**
9374  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9375  *
9376  * @rdev: radeon_device pointer
9377  *
9378  * Fetches a GPU clock counter snapshot (SI).
9379  * Returns the 64 bit clock counter snapshot.
9380  */
9381 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9382 {
9383         uint64_t clock;
9384
9385         mutex_lock(&rdev->gpu_clock_mutex);
9386         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9387         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9388                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9389         mutex_unlock(&rdev->gpu_clock_mutex);
9390         return clock;
9391 }
9392
9393 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9394                              u32 cntl_reg, u32 status_reg)
9395 {
9396         int r, i;
9397         struct atom_clock_dividers dividers;
9398         uint32_t tmp;
9399
9400         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9401                                            clock, false, &dividers);
9402         if (r)
9403                 return r;
9404
9405         tmp = RREG32_SMC(cntl_reg);
9406         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9407         tmp |= dividers.post_divider;
9408         WREG32_SMC(cntl_reg, tmp);
9409
9410         for (i = 0; i < 100; i++) {
9411                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9412                         break;
9413                 mdelay(10);
9414         }
9415         if (i == 100)
9416                 return -ETIMEDOUT;
9417
9418         return 0;
9419 }
9420
9421 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9422 {
9423         int r = 0;
9424
9425         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9426         if (r)
9427                 return r;
9428
9429         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9430         return r;
9431 }
9432
9433 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9434 {
9435         int r, i;
9436         struct atom_clock_dividers dividers;
9437         u32 tmp;
9438
9439         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9440                                            ecclk, false, &dividers);
9441         if (r)
9442                 return r;
9443
9444         for (i = 0; i < 100; i++) {
9445                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9446                         break;
9447                 mdelay(10);
9448         }
9449         if (i == 100)
9450                 return -ETIMEDOUT;
9451
9452         tmp = RREG32_SMC(CG_ECLK_CNTL);
9453         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9454         tmp |= dividers.post_divider;
9455         WREG32_SMC(CG_ECLK_CNTL, tmp);
9456
9457         for (i = 0; i < 100; i++) {
9458                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9459                         break;
9460                 mdelay(10);
9461         }
9462         if (i == 100)
9463                 return -ETIMEDOUT;
9464
9465         return 0;
9466 }
9467
9468 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9469 {
9470         struct pci_dev *root = rdev->pdev->bus->self;
9471         int bridge_pos, gpu_pos;
9472         u32 speed_cntl, mask, current_data_rate;
9473         int ret, i;
9474         u16 tmp16;
9475
9476 #if 0
9477         if (pci_is_root_bus(rdev->pdev->bus))
9478                 return;
9479 #endif
9480
9481         if (radeon_pcie_gen2 == 0)
9482                 return;
9483
9484         if (rdev->flags & RADEON_IS_IGP)
9485                 return;
9486
9487         if (!(rdev->flags & RADEON_IS_PCIE))
9488                 return;
9489
9490         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9491         if (ret != 0)
9492                 return;
9493
9494         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9495                 return;
9496
9497         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9498         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9499                 LC_CURRENT_DATA_RATE_SHIFT;
9500         if (mask & DRM_PCIE_SPEED_80) {
9501                 if (current_data_rate == 2) {
9502                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9503                         return;
9504                 }
9505                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9506         } else if (mask & DRM_PCIE_SPEED_50) {
9507                 if (current_data_rate == 1) {
9508                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9509                         return;
9510                 }
9511                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9512         }
9513
9514         bridge_pos = pci_pcie_cap(root);
9515         if (!bridge_pos)
9516                 return;
9517
9518         gpu_pos = pci_pcie_cap(rdev->pdev);
9519         if (!gpu_pos)
9520                 return;
9521
9522         if (mask & DRM_PCIE_SPEED_80) {
9523                 /* re-try equalization if gen3 is not already enabled */
9524                 if (current_data_rate != 2) {
9525                         u16 bridge_cfg, gpu_cfg;
9526                         u16 bridge_cfg2, gpu_cfg2;
9527                         u32 max_lw, current_lw, tmp;
9528
9529                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9530                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9531
9532                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9533                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9534
9535                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9536                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9537
9538                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9539                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9540                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9541
9542                         if (current_lw < max_lw) {
9543                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9544                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9545                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9546                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9547                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9548                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9549                                 }
9550                         }
9551
9552                         for (i = 0; i < 10; i++) {
9553                                 /* check status */
9554                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9555                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9556                                         break;
9557
9558                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9559                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9560
9561                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9562                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9563
9564                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9565                                 tmp |= LC_SET_QUIESCE;
9566                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9567
9568                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9569                                 tmp |= LC_REDO_EQ;
9570                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9571
9572                                 mdelay(100);
9573
9574                                 /* linkctl */
9575                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9576                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9577                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9578                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9579
9580                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9581                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9582                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9583                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9584
9585                                 /* linkctl2 */
9586                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9587                                 tmp16 &= ~((1 << 4) | (7 << 9));
9588                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9589                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9590
9591                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9592                                 tmp16 &= ~((1 << 4) | (7 << 9));
9593                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9594                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9595
9596                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9597                                 tmp &= ~LC_SET_QUIESCE;
9598                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9599                         }
9600                 }
9601         }
9602
9603         /* set the link speed */
9604         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9605         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9606         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9607
9608         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9609         tmp16 &= ~0xf;
9610         if (mask & DRM_PCIE_SPEED_80)
9611                 tmp16 |= 3; /* gen3 */
9612         else if (mask & DRM_PCIE_SPEED_50)
9613                 tmp16 |= 2; /* gen2 */
9614         else
9615                 tmp16 |= 1; /* gen1 */
9616         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9617
9618         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9619         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9620         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9621
9622         for (i = 0; i < rdev->usec_timeout; i++) {
9623                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9624                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9625                         break;
9626                 udelay(1);
9627         }
9628 }
9629
9630 static void cik_program_aspm(struct radeon_device *rdev)
9631 {
9632         u32 data, orig;
9633         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9634 #if 0
9635         bool disable_clkreq = false;
9636 #endif
9637
9638         if (radeon_aspm == 0)
9639                 return;
9640
9641         /* XXX double check IGPs */
9642         if (rdev->flags & RADEON_IS_IGP)
9643                 return;
9644
9645         if (!(rdev->flags & RADEON_IS_PCIE))
9646                 return;
9647
9648         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9649         data &= ~LC_XMIT_N_FTS_MASK;
9650         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9651         if (orig != data)
9652                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9653
9654         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9655         data |= LC_GO_TO_RECOVERY;
9656         if (orig != data)
9657                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9658
9659         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9660         data |= P_IGNORE_EDB_ERR;
9661         if (orig != data)
9662                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9663
9664         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9665         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9666         data |= LC_PMI_TO_L1_DIS;
9667         if (!disable_l0s)
9668                 data |= LC_L0S_INACTIVITY(7);
9669
9670         if (!disable_l1) {
9671                 data |= LC_L1_INACTIVITY(7);
9672                 data &= ~LC_PMI_TO_L1_DIS;
9673                 if (orig != data)
9674                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9675
9676                 if (!disable_plloff_in_l1) {
9677                         bool clk_req_support;
9678
9679                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9680                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9681                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9682                         if (orig != data)
9683                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9684
9685                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9686                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9687                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9688                         if (orig != data)
9689                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9690
9691                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9692                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9693                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9694                         if (orig != data)
9695                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9696
9697                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9698                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9699                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9700                         if (orig != data)
9701                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9702
9703                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9704                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9705                         data |= LC_DYN_LANES_PWR_STATE(3);
9706                         if (orig != data)
9707                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9708
9709 #ifdef zMN_TODO
9710                         if (!disable_clkreq &&
9711                             !pci_is_root_bus(rdev->pdev->bus)) {
9712                                 struct pci_dev *root = rdev->pdev->bus->self;
9713                                 u32 lnkcap;
9714
9715                                 clk_req_support = false;
9716                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9717                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9718                                         clk_req_support = true;
9719                         } else {
9720                                 clk_req_support = false;
9721                         }
9722 #else
9723                         clk_req_support = false;
9724 #endif
9725
9726                         if (clk_req_support) {
9727                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9728                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9729                                 if (orig != data)
9730                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9731
9732                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9733                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9734                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9735                                 if (orig != data)
9736                                         WREG32_SMC(THM_CLK_CNTL, data);
9737
9738                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9739                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9740                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9741                                 if (orig != data)
9742                                         WREG32_SMC(MISC_CLK_CTRL, data);
9743
9744                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9745                                 data &= ~BCLK_AS_XCLK;
9746                                 if (orig != data)
9747                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9748
9749                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9750                                 data &= ~FORCE_BIF_REFCLK_EN;
9751                                 if (orig != data)
9752                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9753
9754                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9755                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9756                                 data |= MPLL_CLKOUT_SEL(4);
9757                                 if (orig != data)
9758                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9759                         }
9760                 }
9761         } else {
9762                 if (orig != data)
9763                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9764         }
9765
9766         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9767         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9768         if (orig != data)
9769                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9770
9771         if (!disable_l0s) {
9772                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9773                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9774                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9775                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9776                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9777                                 data &= ~LC_L0S_INACTIVITY_MASK;
9778                                 if (orig != data)
9779                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9780                         }
9781                 }
9782         }
9783 }