sys/dev/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/module.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include "cikd.h"
  30 #include "atom.h"
  31 #include "cik_blit_shaders.h"
  32 #include "radeon_ucode.h"
  33 #include "clearstate_ci.h"
  34
  35 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  36 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  44 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  45 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  46 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  47 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  48 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  49 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  50 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  51 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  52 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  53 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  54 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  55 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  56 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  57 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  58 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  59 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  60 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  61 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  62 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  63 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  64 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  65 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
  66 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
  67 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
  68 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
  69 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
  70 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
  71
  72 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
  73 static void cik_rlc_stop(struct radeon_device *rdev);
  74 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  75 static void cik_program_aspm(struct radeon_device *rdev);
  76 static void cik_init_pg(struct radeon_device *rdev);
  77 static void cik_init_cg(struct radeon_device *rdev);
  78 static void cik_fini_pg(struct radeon_device *rdev);
  79 static void cik_fini_cg(struct radeon_device *rdev);
  80 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
  81                                           bool enable);
  82
  83 /* get temperature in millidegrees */
  84 int ci_get_temp(struct radeon_device *rdev)
  85 {
  86         u32 temp;
  87         int actual_temp = 0;
  88
  89         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
  90                 CTF_TEMP_SHIFT;
  91
  92         if (temp & 0x200)
  93                 actual_temp = 255;
  94         else
  95                 actual_temp = temp & 0x1ff;
  96
  97         actual_temp = actual_temp * 1000;
  98
  99         return actual_temp;
 100 }
 101
 102 /* get temperature in millidegrees */
 103 int kv_get_temp(struct radeon_device *rdev)
 104 {
 105         u32 temp;
 106         int actual_temp = 0;
 107
 108         temp = RREG32_SMC(0xC0300E0C);
 109
 110         if (temp)
 111                 actual_temp = (temp / 8) - 49;
 112         else
 113                 actual_temp = 0;
 114
 115         actual_temp = actual_temp * 1000;
 116
 117         return actual_temp;
 118 }
 119
 120 /*
 121  * Indirect registers accessor
 122  */
 123 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 124 {
 125         u32 r;
 126
 127         spin_lock(&rdev->pciep_idx_lock);
 128         WREG32(PCIE_INDEX, reg);
 129         (void)RREG32(PCIE_INDEX);
 130         r = RREG32(PCIE_DATA);
 131         spin_unlock(&rdev->pciep_idx_lock);
 132         return r;
 133 }
 134
 135 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 136 {
 137         spin_lock(&rdev->pciep_idx_lock);
 138         WREG32(PCIE_INDEX, reg);
 139         (void)RREG32(PCIE_INDEX);
 140         WREG32(PCIE_DATA, v);
 141         (void)RREG32(PCIE_DATA);
 142         spin_unlock(&rdev->pciep_idx_lock);
 143 }
 144
 145 static const u32 spectre_rlc_save_restore_register_list[] =
 146 {
 147         (0x0e00 << 16) | (0xc12c >> 2),
 148         0x00000000,
 149         (0x0e00 << 16) | (0xc140 >> 2),
 150         0x00000000,
 151         (0x0e00 << 16) | (0xc150 >> 2),
 152         0x00000000,
 153         (0x0e00 << 16) | (0xc15c >> 2),
 154         0x00000000,
 155         (0x0e00 << 16) | (0xc168 >> 2),
 156         0x00000000,
 157         (0x0e00 << 16) | (0xc170 >> 2),
 158         0x00000000,
 159         (0x0e00 << 16) | (0xc178 >> 2),
 160         0x00000000,
 161         (0x0e00 << 16) | (0xc204 >> 2),
 162         0x00000000,
 163         (0x0e00 << 16) | (0xc2b4 >> 2),
 164         0x00000000,
 165         (0x0e00 << 16) | (0xc2b8 >> 2),
 166         0x00000000,
 167         (0x0e00 << 16) | (0xc2bc >> 2),
 168         0x00000000,
 169         (0x0e00 << 16) | (0xc2c0 >> 2),
 170         0x00000000,
 171         (0x0e00 << 16) | (0x8228 >> 2),
 172         0x00000000,
 173         (0x0e00 << 16) | (0x829c >> 2),
 174         0x00000000,
 175         (0x0e00 << 16) | (0x869c >> 2),
 176         0x00000000,
 177         (0x0600 << 16) | (0x98f4 >> 2),
 178         0x00000000,
 179         (0x0e00 << 16) | (0x98f8 >> 2),
 180         0x00000000,
 181         (0x0e00 << 16) | (0x9900 >> 2),
 182         0x00000000,
 183         (0x0e00 << 16) | (0xc260 >> 2),
 184         0x00000000,
 185         (0x0e00 << 16) | (0x90e8 >> 2),
 186         0x00000000,
 187         (0x0e00 << 16) | (0x3c000 >> 2),
 188         0x00000000,
 189         (0x0e00 << 16) | (0x3c00c >> 2),
 190         0x00000000,
 191         (0x0e00 << 16) | (0x8c1c >> 2),
 192         0x00000000,
 193         (0x0e00 << 16) | (0x9700 >> 2),
 194         0x00000000,
 195         (0x0e00 << 16) | (0xcd20 >> 2),
 196         0x00000000,
 197         (0x4e00 << 16) | (0xcd20 >> 2),
 198         0x00000000,
 199         (0x5e00 << 16) | (0xcd20 >> 2),
 200         0x00000000,
 201         (0x6e00 << 16) | (0xcd20 >> 2),
 202         0x00000000,
 203         (0x7e00 << 16) | (0xcd20 >> 2),
 204         0x00000000,
 205         (0x8e00 << 16) | (0xcd20 >> 2),
 206         0x00000000,
 207         (0x9e00 << 16) | (0xcd20 >> 2),
 208         0x00000000,
 209         (0xae00 << 16) | (0xcd20 >> 2),
 210         0x00000000,
 211         (0xbe00 << 16) | (0xcd20 >> 2),
 212         0x00000000,
 213         (0x0e00 << 16) | (0x89bc >> 2),
 214         0x00000000,
 215         (0x0e00 << 16) | (0x8900 >> 2),
 216         0x00000000,
 217         0x3,
 218         (0x0e00 << 16) | (0xc130 >> 2),
 219         0x00000000,
 220         (0x0e00 << 16) | (0xc134 >> 2),
 221         0x00000000,
 222         (0x0e00 << 16) | (0xc1fc >> 2),
 223         0x00000000,
 224         (0x0e00 << 16) | (0xc208 >> 2),
 225         0x00000000,
 226         (0x0e00 << 16) | (0xc264 >> 2),
 227         0x00000000,
 228         (0x0e00 << 16) | (0xc268 >> 2),
 229         0x00000000,
 230         (0x0e00 << 16) | (0xc26c >> 2),
 231         0x00000000,
 232         (0x0e00 << 16) | (0xc270 >> 2),
 233         0x00000000,
 234         (0x0e00 << 16) | (0xc274 >> 2),
 235         0x00000000,
 236         (0x0e00 << 16) | (0xc278 >> 2),
 237         0x00000000,
 238         (0x0e00 << 16) | (0xc27c >> 2),
 239         0x00000000,
 240         (0x0e00 << 16) | (0xc280 >> 2),
 241         0x00000000,
 242         (0x0e00 << 16) | (0xc284 >> 2),
 243         0x00000000,
 244         (0x0e00 << 16) | (0xc288 >> 2),
 245         0x00000000,
 246         (0x0e00 << 16) | (0xc28c >> 2),
 247         0x00000000,
 248         (0x0e00 << 16) | (0xc290 >> 2),
 249         0x00000000,
 250         (0x0e00 << 16) | (0xc294 >> 2),
 251         0x00000000,
 252         (0x0e00 << 16) | (0xc298 >> 2),
 253         0x00000000,
 254         (0x0e00 << 16) | (0xc29c >> 2),
 255         0x00000000,
 256         (0x0e00 << 16) | (0xc2a0 >> 2),
 257         0x00000000,
 258         (0x0e00 << 16) | (0xc2a4 >> 2),
 259         0x00000000,
 260         (0x0e00 << 16) | (0xc2a8 >> 2),
 261         0x00000000,
 262         (0x0e00 << 16) | (0xc2ac  >> 2),
 263         0x00000000,
 264         (0x0e00 << 16) | (0xc2b0 >> 2),
 265         0x00000000,
 266         (0x0e00 << 16) | (0x301d0 >> 2),
 267         0x00000000,
 268         (0x0e00 << 16) | (0x30238 >> 2),
 269         0x00000000,
 270         (0x0e00 << 16) | (0x30250 >> 2),
 271         0x00000000,
 272         (0x0e00 << 16) | (0x30254 >> 2),
 273         0x00000000,
 274         (0x0e00 << 16) | (0x30258 >> 2),
 275         0x00000000,
 276         (0x0e00 << 16) | (0x3025c >> 2),
 277         0x00000000,
 278         (0x4e00 << 16) | (0xc900 >> 2),
 279         0x00000000,
 280         (0x5e00 << 16) | (0xc900 >> 2),
 281         0x00000000,
 282         (0x6e00 << 16) | (0xc900 >> 2),
 283         0x00000000,
 284         (0x7e00 << 16) | (0xc900 >> 2),
 285         0x00000000,
 286         (0x8e00 << 16) | (0xc900 >> 2),
 287         0x00000000,
 288         (0x9e00 << 16) | (0xc900 >> 2),
 289         0x00000000,
 290         (0xae00 << 16) | (0xc900 >> 2),
 291         0x00000000,
 292         (0xbe00 << 16) | (0xc900 >> 2),
 293         0x00000000,
 294         (0x4e00 << 16) | (0xc904 >> 2),
 295         0x00000000,
 296         (0x5e00 << 16) | (0xc904 >> 2),
 297         0x00000000,
 298         (0x6e00 << 16) | (0xc904 >> 2),
 299         0x00000000,
 300         (0x7e00 << 16) | (0xc904 >> 2),
 301         0x00000000,
 302         (0x8e00 << 16) | (0xc904 >> 2),
 303         0x00000000,
 304         (0x9e00 << 16) | (0xc904 >> 2),
 305         0x00000000,
 306         (0xae00 << 16) | (0xc904 >> 2),
 307         0x00000000,
 308         (0xbe00 << 16) | (0xc904 >> 2),
 309         0x00000000,
 310         (0x4e00 << 16) | (0xc908 >> 2),
 311         0x00000000,
 312         (0x5e00 << 16) | (0xc908 >> 2),
 313         0x00000000,
 314         (0x6e00 << 16) | (0xc908 >> 2),
 315         0x00000000,
 316         (0x7e00 << 16) | (0xc908 >> 2),
 317         0x00000000,
 318         (0x8e00 << 16) | (0xc908 >> 2),
 319         0x00000000,
 320         (0x9e00 << 16) | (0xc908 >> 2),
 321         0x00000000,
 322         (0xae00 << 16) | (0xc908 >> 2),
 323         0x00000000,
 324         (0xbe00 << 16) | (0xc908 >> 2),
 325         0x00000000,
 326         (0x4e00 << 16) | (0xc90c >> 2),
 327         0x00000000,
 328         (0x5e00 << 16) | (0xc90c >> 2),
 329         0x00000000,
 330         (0x6e00 << 16) | (0xc90c >> 2),
 331         0x00000000,
 332         (0x7e00 << 16) | (0xc90c >> 2),
 333         0x00000000,
 334         (0x8e00 << 16) | (0xc90c >> 2),
 335         0x00000000,
 336         (0x9e00 << 16) | (0xc90c >> 2),
 337         0x00000000,
 338         (0xae00 << 16) | (0xc90c >> 2),
 339         0x00000000,
 340         (0xbe00 << 16) | (0xc90c >> 2),
 341         0x00000000,
 342         (0x4e00 << 16) | (0xc910 >> 2),
 343         0x00000000,
 344         (0x5e00 << 16) | (0xc910 >> 2),
 345         0x00000000,
 346         (0x6e00 << 16) | (0xc910 >> 2),
 347         0x00000000,
 348         (0x7e00 << 16) | (0xc910 >> 2),
 349         0x00000000,
 350         (0x8e00 << 16) | (0xc910 >> 2),
 351         0x00000000,
 352         (0x9e00 << 16) | (0xc910 >> 2),
 353         0x00000000,
 354         (0xae00 << 16) | (0xc910 >> 2),
 355         0x00000000,
 356         (0xbe00 << 16) | (0xc910 >> 2),
 357         0x00000000,
 358         (0x0e00 << 16) | (0xc99c >> 2),
 359         0x00000000,
 360         (0x0e00 << 16) | (0x9834 >> 2),
 361         0x00000000,
 362         (0x0000 << 16) | (0x30f00 >> 2),
 363         0x00000000,
 364         (0x0001 << 16) | (0x30f00 >> 2),
 365         0x00000000,
 366         (0x0000 << 16) | (0x30f04 >> 2),
 367         0x00000000,
 368         (0x0001 << 16) | (0x30f04 >> 2),
 369         0x00000000,
 370         (0x0000 << 16) | (0x30f08 >> 2),
 371         0x00000000,
 372         (0x0001 << 16) | (0x30f08 >> 2),
 373         0x00000000,
 374         (0x0000 << 16) | (0x30f0c >> 2),
 375         0x00000000,
 376         (0x0001 << 16) | (0x30f0c >> 2),
 377         0x00000000,
 378         (0x0600 << 16) | (0x9b7c >> 2),
 379         0x00000000,
 380         (0x0e00 << 16) | (0x8a14 >> 2),
 381         0x00000000,
 382         (0x0e00 << 16) | (0x8a18 >> 2),
 383         0x00000000,
 384         (0x0600 << 16) | (0x30a00 >> 2),
 385         0x00000000,
 386         (0x0e00 << 16) | (0x8bf0 >> 2),
 387         0x00000000,
 388         (0x0e00 << 16) | (0x8bcc >> 2),
 389         0x00000000,
 390         (0x0e00 << 16) | (0x8b24 >> 2),
 391         0x00000000,
 392         (0x0e00 << 16) | (0x30a04 >> 2),
 393         0x00000000,
 394         (0x0600 << 16) | (0x30a10 >> 2),
 395         0x00000000,
 396         (0x0600 << 16) | (0x30a14 >> 2),
 397         0x00000000,
 398         (0x0600 << 16) | (0x30a18 >> 2),
 399         0x00000000,
 400         (0x0600 << 16) | (0x30a2c >> 2),
 401         0x00000000,
 402         (0x0e00 << 16) | (0xc700 >> 2),
 403         0x00000000,
 404         (0x0e00 << 16) | (0xc704 >> 2),
 405         0x00000000,
 406         (0x0e00 << 16) | (0xc708 >> 2),
 407         0x00000000,
 408         (0x0e00 << 16) | (0xc768 >> 2),
 409         0x00000000,
 410         (0x0400 << 16) | (0xc770 >> 2),
 411         0x00000000,
 412         (0x0400 << 16) | (0xc774 >> 2),
 413         0x00000000,
 414         (0x0400 << 16) | (0xc778 >> 2),
 415         0x00000000,
 416         (0x0400 << 16) | (0xc77c >> 2),
 417         0x00000000,
 418         (0x0400 << 16) | (0xc780 >> 2),
 419         0x00000000,
 420         (0x0400 << 16) | (0xc784 >> 2),
 421         0x00000000,
 422         (0x0400 << 16) | (0xc788 >> 2),
 423         0x00000000,
 424         (0x0400 << 16) | (0xc78c >> 2),
 425         0x00000000,
 426         (0x0400 << 16) | (0xc798 >> 2),
 427         0x00000000,
 428         (0x0400 << 16) | (0xc79c >> 2),
 429         0x00000000,
 430         (0x0400 << 16) | (0xc7a0 >> 2),
 431         0x00000000,
 432         (0x0400 << 16) | (0xc7a4 >> 2),
 433         0x00000000,
 434         (0x0400 << 16) | (0xc7a8 >> 2),
 435         0x00000000,
 436         (0x0400 << 16) | (0xc7ac >> 2),
 437         0x00000000,
 438         (0x0400 << 16) | (0xc7b0 >> 2),
 439         0x00000000,
 440         (0x0400 << 16) | (0xc7b4 >> 2),
 441         0x00000000,
 442         (0x0e00 << 16) | (0x9100 >> 2),
 443         0x00000000,
 444         (0x0e00 << 16) | (0x3c010 >> 2),
 445         0x00000000,
 446         (0x0e00 << 16) | (0x92a8 >> 2),
 447         0x00000000,
 448         (0x0e00 << 16) | (0x92ac >> 2),
 449         0x00000000,
 450         (0x0e00 << 16) | (0x92b4 >> 2),
 451         0x00000000,
 452         (0x0e00 << 16) | (0x92b8 >> 2),
 453         0x00000000,
 454         (0x0e00 << 16) | (0x92bc >> 2),
 455         0x00000000,
 456         (0x0e00 << 16) | (0x92c0 >> 2),
 457         0x00000000,
 458         (0x0e00 << 16) | (0x92c4 >> 2),
 459         0x00000000,
 460         (0x0e00 << 16) | (0x92c8 >> 2),
 461         0x00000000,
 462         (0x0e00 << 16) | (0x92cc >> 2),
 463         0x00000000,
 464         (0x0e00 << 16) | (0x92d0 >> 2),
 465         0x00000000,
 466         (0x0e00 << 16) | (0x8c00 >> 2),
 467         0x00000000,
 468         (0x0e00 << 16) | (0x8c04 >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0x8c20 >> 2),
 471         0x00000000,
 472         (0x0e00 << 16) | (0x8c38 >> 2),
 473         0x00000000,
 474         (0x0e00 << 16) | (0x8c3c >> 2),
 475         0x00000000,
 476         (0x0e00 << 16) | (0xae00 >> 2),
 477         0x00000000,
 478         (0x0e00 << 16) | (0x9604 >> 2),
 479         0x00000000,
 480         (0x0e00 << 16) | (0xac08 >> 2),
 481         0x00000000,
 482         (0x0e00 << 16) | (0xac0c >> 2),
 483         0x00000000,
 484         (0x0e00 << 16) | (0xac10 >> 2),
 485         0x00000000,
 486         (0x0e00 << 16) | (0xac14 >> 2),
 487         0x00000000,
 488         (0x0e00 << 16) | (0xac58 >> 2),
 489         0x00000000,
 490         (0x0e00 << 16) | (0xac68 >> 2),
 491         0x00000000,
 492         (0x0e00 << 16) | (0xac6c >> 2),
 493         0x00000000,
 494         (0x0e00 << 16) | (0xac70 >> 2),
 495         0x00000000,
 496         (0x0e00 << 16) | (0xac74 >> 2),
 497         0x00000000,
 498         (0x0e00 << 16) | (0xac78 >> 2),
 499         0x00000000,
 500         (0x0e00 << 16) | (0xac7c >> 2),
 501         0x00000000,
 502         (0x0e00 << 16) | (0xac80 >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0xac84 >> 2),
 505         0x00000000,
 506         (0x0e00 << 16) | (0xac88 >> 2),
 507         0x00000000,
 508         (0x0e00 << 16) | (0xac8c >> 2),
 509         0x00000000,
 510         (0x0e00 << 16) | (0x970c >> 2),
 511         0x00000000,
 512         (0x0e00 << 16) | (0x9714 >> 2),
 513         0x00000000,
 514         (0x0e00 << 16) | (0x9718 >> 2),
 515         0x00000000,
 516         (0x0e00 << 16) | (0x971c >> 2),
 517         0x00000000,
 518         (0x0e00 << 16) | (0x31068 >> 2),
 519         0x00000000,
 520         (0x4e00 << 16) | (0x31068 >> 2),
 521         0x00000000,
 522         (0x5e00 << 16) | (0x31068 >> 2),
 523         0x00000000,
 524         (0x6e00 << 16) | (0x31068 >> 2),
 525         0x00000000,
 526         (0x7e00 << 16) | (0x31068 >> 2),
 527         0x00000000,
 528         (0x8e00 << 16) | (0x31068 >> 2),
 529         0x00000000,
 530         (0x9e00 << 16) | (0x31068 >> 2),
 531         0x00000000,
 532         (0xae00 << 16) | (0x31068 >> 2),
 533         0x00000000,
 534         (0xbe00 << 16) | (0x31068 >> 2),
 535         0x00000000,
 536         (0x0e00 << 16) | (0xcd10 >> 2),
 537         0x00000000,
 538         (0x0e00 << 16) | (0xcd14 >> 2),
 539         0x00000000,
 540         (0x0e00 << 16) | (0x88b0 >> 2),
 541         0x00000000,
 542         (0x0e00 << 16) | (0x88b4 >> 2),
 543         0x00000000,
 544         (0x0e00 << 16) | (0x88b8 >> 2),
 545         0x00000000,
 546         (0x0e00 << 16) | (0x88bc >> 2),
 547         0x00000000,
 548         (0x0400 << 16) | (0x89c0 >> 2),
 549         0x00000000,
 550         (0x0e00 << 16) | (0x88c4 >> 2),
 551         0x00000000,
 552         (0x0e00 << 16) | (0x88c8 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0x88d0 >> 2),
 555         0x00000000,
 556         (0x0e00 << 16) | (0x88d4 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0x88d8 >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0x8980 >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0x30938 >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0x3093c >> 2),
 565         0x00000000,
 566         (0x0e00 << 16) | (0x30940 >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0x89a0 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0x30900 >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x30904 >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x89b4 >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x3c210 >> 2),
 577         0x00000000,
 578         (0x0e00 << 16) | (0x3c214 >> 2),
 579         0x00000000,
 580         (0x0e00 << 16) | (0x3c218 >> 2),
 581         0x00000000,
 582         (0x0e00 << 16) | (0x8904 >> 2),
 583         0x00000000,
 584         0x5,
 585         (0x0e00 << 16) | (0x8c28 >> 2),
 586         (0x0e00 << 16) | (0x8c2c >> 2),
 587         (0x0e00 << 16) | (0x8c30 >> 2),
 588         (0x0e00 << 16) | (0x8c34 >> 2),
 589         (0x0e00 << 16) | (0x9600 >> 2),
 590 };
 591
 592 static const u32 kalindi_rlc_save_restore_register_list[] =
 593 {
 594         (0x0e00 << 16) | (0xc12c >> 2),
 595         0x00000000,
 596         (0x0e00 << 16) | (0xc140 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0xc150 >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0xc15c >> 2),
 601         0x00000000,
 602         (0x0e00 << 16) | (0xc168 >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0xc170 >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0xc204 >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0xc2b4 >> 2),
 609         0x00000000,
 610         (0x0e00 << 16) | (0xc2b8 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0xc2bc >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0xc2c0 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0x8228 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0x829c >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0x869c >> 2),
 621         0x00000000,
 622         (0x0600 << 16) | (0x98f4 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0x98f8 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0x9900 >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0xc260 >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0x90e8 >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0x3c000 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0x3c00c >> 2),
 635         0x00000000,
 636         (0x0e00 << 16) | (0x8c1c >> 2),
 637         0x00000000,
 638         (0x0e00 << 16) | (0x9700 >> 2),
 639         0x00000000,
 640         (0x0e00 << 16) | (0xcd20 >> 2),
 641         0x00000000,
 642         (0x4e00 << 16) | (0xcd20 >> 2),
 643         0x00000000,
 644         (0x5e00 << 16) | (0xcd20 >> 2),
 645         0x00000000,
 646         (0x6e00 << 16) | (0xcd20 >> 2),
 647         0x00000000,
 648         (0x7e00 << 16) | (0xcd20 >> 2),
 649         0x00000000,
 650         (0x0e00 << 16) | (0x89bc >> 2),
 651         0x00000000,
 652         (0x0e00 << 16) | (0x8900 >> 2),
 653         0x00000000,
 654         0x3,
 655         (0x0e00 << 16) | (0xc130 >> 2),
 656         0x00000000,
 657         (0x0e00 << 16) | (0xc134 >> 2),
 658         0x00000000,
 659         (0x0e00 << 16) | (0xc1fc >> 2),
 660         0x00000000,
 661         (0x0e00 << 16) | (0xc208 >> 2),
 662         0x00000000,
 663         (0x0e00 << 16) | (0xc264 >> 2),
 664         0x00000000,
 665         (0x0e00 << 16) | (0xc268 >> 2),
 666         0x00000000,
 667         (0x0e00 << 16) | (0xc26c >> 2),
 668         0x00000000,
 669         (0x0e00 << 16) | (0xc270 >> 2),
 670         0x00000000,
 671         (0x0e00 << 16) | (0xc274 >> 2),
 672         0x00000000,
 673         (0x0e00 << 16) | (0xc28c >> 2),
 674         0x00000000,
 675         (0x0e00 << 16) | (0xc290 >> 2),
 676         0x00000000,
 677         (0x0e00 << 16) | (0xc294 >> 2),
 678         0x00000000,
 679         (0x0e00 << 16) | (0xc298 >> 2),
 680         0x00000000,
 681         (0x0e00 << 16) | (0xc2a0 >> 2),
 682         0x00000000,
 683         (0x0e00 << 16) | (0xc2a4 >> 2),
 684         0x00000000,
 685         (0x0e00 << 16) | (0xc2a8 >> 2),
 686         0x00000000,
 687         (0x0e00 << 16) | (0xc2ac >> 2),
 688         0x00000000,
 689         (0x0e00 << 16) | (0x301d0 >> 2),
 690         0x00000000,
 691         (0x0e00 << 16) | (0x30238 >> 2),
 692         0x00000000,
 693         (0x0e00 << 16) | (0x30250 >> 2),
 694         0x00000000,
 695         (0x0e00 << 16) | (0x30254 >> 2),
 696         0x00000000,
 697         (0x0e00 << 16) | (0x30258 >> 2),
 698         0x00000000,
 699         (0x0e00 << 16) | (0x3025c >> 2),
 700         0x00000000,
 701         (0x4e00 << 16) | (0xc900 >> 2),
 702         0x00000000,
 703         (0x5e00 << 16) | (0xc900 >> 2),
 704         0x00000000,
 705         (0x6e00 << 16) | (0xc900 >> 2),
 706         0x00000000,
 707         (0x7e00 << 16) | (0xc900 >> 2),
 708         0x00000000,
 709         (0x4e00 << 16) | (0xc904 >> 2),
 710         0x00000000,
 711         (0x5e00 << 16) | (0xc904 >> 2),
 712         0x00000000,
 713         (0x6e00 << 16) | (0xc904 >> 2),
 714         0x00000000,
 715         (0x7e00 << 16) | (0xc904 >> 2),
 716         0x00000000,
 717         (0x4e00 << 16) | (0xc908 >> 2),
 718         0x00000000,
 719         (0x5e00 << 16) | (0xc908 >> 2),
 720         0x00000000,
 721         (0x6e00 << 16) | (0xc908 >> 2),
 722         0x00000000,
 723         (0x7e00 << 16) | (0xc908 >> 2),
 724         0x00000000,
 725         (0x4e00 << 16) | (0xc90c >> 2),
 726         0x00000000,
 727         (0x5e00 << 16) | (0xc90c >> 2),
 728         0x00000000,
 729         (0x6e00 << 16) | (0xc90c >> 2),
 730         0x00000000,
 731         (0x7e00 << 16) | (0xc90c >> 2),
 732         0x00000000,
 733         (0x4e00 << 16) | (0xc910 >> 2),
 734         0x00000000,
 735         (0x5e00 << 16) | (0xc910 >> 2),
 736         0x00000000,
 737         (0x6e00 << 16) | (0xc910 >> 2),
 738         0x00000000,
 739         (0x7e00 << 16) | (0xc910 >> 2),
 740         0x00000000,
 741         (0x0e00 << 16) | (0xc99c >> 2),
 742         0x00000000,
 743         (0x0e00 << 16) | (0x9834 >> 2),
 744         0x00000000,
 745         (0x0000 << 16) | (0x30f00 >> 2),
 746         0x00000000,
 747         (0x0000 << 16) | (0x30f04 >> 2),
 748         0x00000000,
 749         (0x0000 << 16) | (0x30f08 >> 2),
 750         0x00000000,
 751         (0x0000 << 16) | (0x30f0c >> 2),
 752         0x00000000,
 753         (0x0600 << 16) | (0x9b7c >> 2),
 754         0x00000000,
 755         (0x0e00 << 16) | (0x8a14 >> 2),
 756         0x00000000,
 757         (0x0e00 << 16) | (0x8a18 >> 2),
 758         0x00000000,
 759         (0x0600 << 16) | (0x30a00 >> 2),
 760         0x00000000,
 761         (0x0e00 << 16) | (0x8bf0 >> 2),
 762         0x00000000,
 763         (0x0e00 << 16) | (0x8bcc >> 2),
 764         0x00000000,
 765         (0x0e00 << 16) | (0x8b24 >> 2),
 766         0x00000000,
 767         (0x0e00 << 16) | (0x30a04 >> 2),
 768         0x00000000,
 769         (0x0600 << 16) | (0x30a10 >> 2),
 770         0x00000000,
 771         (0x0600 << 16) | (0x30a14 >> 2),
 772         0x00000000,
 773         (0x0600 << 16) | (0x30a18 >> 2),
 774         0x00000000,
 775         (0x0600 << 16) | (0x30a2c >> 2),
 776         0x00000000,
 777         (0x0e00 << 16) | (0xc700 >> 2),
 778         0x00000000,
 779         (0x0e00 << 16) | (0xc704 >> 2),
 780         0x00000000,
 781         (0x0e00 << 16) | (0xc708 >> 2),
 782         0x00000000,
 783         (0x0e00 << 16) | (0xc768 >> 2),
 784         0x00000000,
 785         (0x0400 << 16) | (0xc770 >> 2),
 786         0x00000000,
 787         (0x0400 << 16) | (0xc774 >> 2),
 788         0x00000000,
 789         (0x0400 << 16) | (0xc798 >> 2),
 790         0x00000000,
 791         (0x0400 << 16) | (0xc79c >> 2),
 792         0x00000000,
 793         (0x0e00 << 16) | (0x9100 >> 2),
 794         0x00000000,
 795         (0x0e00 << 16) | (0x3c010 >> 2),
 796         0x00000000,
 797         (0x0e00 << 16) | (0x8c00 >> 2),
 798         0x00000000,
 799         (0x0e00 << 16) | (0x8c04 >> 2),
 800         0x00000000,
 801         (0x0e00 << 16) | (0x8c20 >> 2),
 802         0x00000000,
 803         (0x0e00 << 16) | (0x8c38 >> 2),
 804         0x00000000,
 805         (0x0e00 << 16) | (0x8c3c >> 2),
 806         0x00000000,
 807         (0x0e00 << 16) | (0xae00 >> 2),
 808         0x00000000,
 809         (0x0e00 << 16) | (0x9604 >> 2),
 810         0x00000000,
 811         (0x0e00 << 16) | (0xac08 >> 2),
 812         0x00000000,
 813         (0x0e00 << 16) | (0xac0c >> 2),
 814         0x00000000,
 815         (0x0e00 << 16) | (0xac10 >> 2),
 816         0x00000000,
 817         (0x0e00 << 16) | (0xac14 >> 2),
 818         0x00000000,
 819         (0x0e00 << 16) | (0xac58 >> 2),
 820         0x00000000,
 821         (0x0e00 << 16) | (0xac68 >> 2),
 822         0x00000000,
 823         (0x0e00 << 16) | (0xac6c >> 2),
 824         0x00000000,
 825         (0x0e00 << 16) | (0xac70 >> 2),
 826         0x00000000,
 827         (0x0e00 << 16) | (0xac74 >> 2),
 828         0x00000000,
 829         (0x0e00 << 16) | (0xac78 >> 2),
 830         0x00000000,
 831         (0x0e00 << 16) | (0xac7c >> 2),
 832         0x00000000,
 833         (0x0e00 << 16) | (0xac80 >> 2),
 834         0x00000000,
 835         (0x0e00 << 16) | (0xac84 >> 2),
 836         0x00000000,
 837         (0x0e00 << 16) | (0xac88 >> 2),
 838         0x00000000,
 839         (0x0e00 << 16) | (0xac8c >> 2),
 840         0x00000000,
 841         (0x0e00 << 16) | (0x970c >> 2),
 842         0x00000000,
 843         (0x0e00 << 16) | (0x9714 >> 2),
 844         0x00000000,
 845         (0x0e00 << 16) | (0x9718 >> 2),
 846         0x00000000,
 847         (0x0e00 << 16) | (0x971c >> 2),
 848         0x00000000,
 849         (0x0e00 << 16) | (0x31068 >> 2),
 850         0x00000000,
 851         (0x4e00 << 16) | (0x31068 >> 2),
 852         0x00000000,
 853         (0x5e00 << 16) | (0x31068 >> 2),
 854         0x00000000,
 855         (0x6e00 << 16) | (0x31068 >> 2),
 856         0x00000000,
 857         (0x7e00 << 16) | (0x31068 >> 2),
 858         0x00000000,
 859         (0x0e00 << 16) | (0xcd10 >> 2),
 860         0x00000000,
 861         (0x0e00 << 16) | (0xcd14 >> 2),
 862         0x00000000,
 863         (0x0e00 << 16) | (0x88b0 >> 2),
 864         0x00000000,
 865         (0x0e00 << 16) | (0x88b4 >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0x88b8 >> 2),
 868         0x00000000,
 869         (0x0e00 << 16) | (0x88bc >> 2),
 870         0x00000000,
 871         (0x0400 << 16) | (0x89c0 >> 2),
 872         0x00000000,
 873         (0x0e00 << 16) | (0x88c4 >> 2),
 874         0x00000000,
 875         (0x0e00 << 16) | (0x88c8 >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0x88d0 >> 2),
 878         0x00000000,
 879         (0x0e00 << 16) | (0x88d4 >> 2),
 880         0x00000000,
 881         (0x0e00 << 16) | (0x88d8 >> 2),
 882         0x00000000,
 883         (0x0e00 << 16) | (0x8980 >> 2),
 884         0x00000000,
 885         (0x0e00 << 16) | (0x30938 >> 2),
 886         0x00000000,
 887         (0x0e00 << 16) | (0x3093c >> 2),
 888         0x00000000,
 889         (0x0e00 << 16) | (0x30940 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0x89a0 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0x30900 >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0x30904 >> 2),
 896         0x00000000,
 897         (0x0e00 << 16) | (0x89b4 >> 2),
 898         0x00000000,
 899         (0x0e00 << 16) | (0x3e1fc >> 2),
 900         0x00000000,
 901         (0x0e00 << 16) | (0x3c210 >> 2),
 902         0x00000000,
 903         (0x0e00 << 16) | (0x3c214 >> 2),
 904         0x00000000,
 905         (0x0e00 << 16) | (0x3c218 >> 2),
 906         0x00000000,
 907         (0x0e00 << 16) | (0x8904 >> 2),
 908         0x00000000,
 909         0x5,
 910         (0x0e00 << 16) | (0x8c28 >> 2),
 911         (0x0e00 << 16) | (0x8c2c >> 2),
 912         (0x0e00 << 16) | (0x8c30 >> 2),
 913         (0x0e00 << 16) | (0x8c34 >> 2),
 914         (0x0e00 << 16) | (0x9600 >> 2),
 915 };
 916
 917 static const u32 bonaire_golden_spm_registers[] =
 918 {
 919         0x30800, 0xe0ffffff, 0xe0000000
 920 };
 921
 922 static const u32 bonaire_golden_common_registers[] =
 923 {
 924         0xc770, 0xffffffff, 0x00000800,
 925         0xc774, 0xffffffff, 0x00000800,
 926         0xc798, 0xffffffff, 0x00007fbf,
 927         0xc79c, 0xffffffff, 0x00007faf
 928 };
 929
 930 static const u32 bonaire_golden_registers[] =
 931 {
 932         0x3354, 0x00000333, 0x00000333,
 933         0x3350, 0x000c0fc0, 0x00040200,
 934         0x9a10, 0x00010000, 0x00058208,
 935         0x3c000, 0xffff1fff, 0x00140000,
 936         0x3c200, 0xfdfc0fff, 0x00000100,
 937         0x3c234, 0x40000000, 0x40000200,
 938         0x9830, 0xffffffff, 0x00000000,
 939         0x9834, 0xf00fffff, 0x00000400,
 940         0x9838, 0x0002021c, 0x00020200,
 941         0xc78, 0x00000080, 0x00000000,
 942         0x5bb0, 0x000000f0, 0x00000070,
 943         0x5bc0, 0xf0311fff, 0x80300000,
 944         0x98f8, 0x73773777, 0x12010001,
 945         0x350c, 0x00810000, 0x408af000,
 946         0x7030, 0x31000111, 0x00000011,
 947         0x2f48, 0x73773777, 0x12010001,
 948         0x220c, 0x00007fb6, 0x0021a1b1,
 949         0x2210, 0x00007fb6, 0x002021b1,
 950         0x2180, 0x00007fb6, 0x00002191,
 951         0x2218, 0x00007fb6, 0x002121b1,
 952         0x221c, 0x00007fb6, 0x002021b1,
 953         0x21dc, 0x00007fb6, 0x00002191,
 954         0x21e0, 0x00007fb6, 0x00002191,
 955         0x3628, 0x0000003f, 0x0000000a,
 956         0x362c, 0x0000003f, 0x0000000a,
 957         0x2ae4, 0x00073ffe, 0x000022a2,
 958         0x240c, 0x000007ff, 0x00000000,
 959         0x8a14, 0xf000003f, 0x00000007,
 960         0x8bf0, 0x00002001, 0x00000001,
 961         0x8b24, 0xffffffff, 0x00ffffff,
 962         0x30a04, 0x0000ff0f, 0x00000000,
 963         0x28a4c, 0x07ffffff, 0x06000000,
 964         0x4d8, 0x00000fff, 0x00000100,
 965         0x3e78, 0x00000001, 0x00000002,
 966         0x9100, 0x03000000, 0x0362c688,
 967         0x8c00, 0x000000ff, 0x00000001,
 968         0xe40, 0x00001fff, 0x00001fff,
 969         0x9060, 0x0000007f, 0x00000020,
 970         0x9508, 0x00010000, 0x00010000,
 971         0xac14, 0x000003ff, 0x000000f3,
 972         0xac0c, 0xffffffff, 0x00001032
 973 };
 974
 975 static const u32 bonaire_mgcg_cgcg_init[] =
 976 {
 977         0xc420, 0xffffffff, 0xfffffffc,
 978         0x30800, 0xffffffff, 0xe0000000,
 979         0x3c2a0, 0xffffffff, 0x00000100,
 980         0x3c208, 0xffffffff, 0x00000100,
 981         0x3c2c0, 0xffffffff, 0xc0000100,
 982         0x3c2c8, 0xffffffff, 0xc0000100,
 983         0x3c2c4, 0xffffffff, 0xc0000100,
 984         0x55e4, 0xffffffff, 0x00600100,
 985         0x3c280, 0xffffffff, 0x00000100,
 986         0x3c214, 0xffffffff, 0x06000100,
 987         0x3c220, 0xffffffff, 0x00000100,
 988         0x3c218, 0xffffffff, 0x06000100,
 989         0x3c204, 0xffffffff, 0x00000100,
 990         0x3c2e0, 0xffffffff, 0x00000100,
 991         0x3c224, 0xffffffff, 0x00000100,
 992         0x3c200, 0xffffffff, 0x00000100,
 993         0x3c230, 0xffffffff, 0x00000100,
 994         0x3c234, 0xffffffff, 0x00000100,
 995         0x3c250, 0xffffffff, 0x00000100,
 996         0x3c254, 0xffffffff, 0x00000100,
 997         0x3c258, 0xffffffff, 0x00000100,
 998         0x3c25c, 0xffffffff, 0x00000100,
 999         0x3c260, 0xffffffff, 0x00000100,
1000         0x3c27c, 0xffffffff, 0x00000100,
1001         0x3c278, 0xffffffff, 0x00000100,
1002         0x3c210, 0xffffffff, 0x06000100,
1003         0x3c290, 0xffffffff, 0x00000100,
1004         0x3c274, 0xffffffff, 0x00000100,
1005         0x3c2b4, 0xffffffff, 0x00000100,
1006         0x3c2b0, 0xffffffff, 0x00000100,
1007         0x3c270, 0xffffffff, 0x00000100,
1008         0x30800, 0xffffffff, 0xe0000000,
1009         0x3c020, 0xffffffff, 0x00010000,
1010         0x3c024, 0xffffffff, 0x00030002,
1011         0x3c028, 0xffffffff, 0x00040007,
1012         0x3c02c, 0xffffffff, 0x00060005,
1013         0x3c030, 0xffffffff, 0x00090008,
1014         0x3c034, 0xffffffff, 0x00010000,
1015         0x3c038, 0xffffffff, 0x00030002,
1016         0x3c03c, 0xffffffff, 0x00040007,
1017         0x3c040, 0xffffffff, 0x00060005,
1018         0x3c044, 0xffffffff, 0x00090008,
1019         0x3c048, 0xffffffff, 0x00010000,
1020         0x3c04c, 0xffffffff, 0x00030002,
1021         0x3c050, 0xffffffff, 0x00040007,
1022         0x3c054, 0xffffffff, 0x00060005,
1023         0x3c058, 0xffffffff, 0x00090008,
1024         0x3c05c, 0xffffffff, 0x00010000,
1025         0x3c060, 0xffffffff, 0x00030002,
1026         0x3c064, 0xffffffff, 0x00040007,
1027         0x3c068, 0xffffffff, 0x00060005,
1028         0x3c06c, 0xffffffff, 0x00090008,
1029         0x3c070, 0xffffffff, 0x00010000,
1030         0x3c074, 0xffffffff, 0x00030002,
1031         0x3c078, 0xffffffff, 0x00040007,
1032         0x3c07c, 0xffffffff, 0x00060005,
1033         0x3c080, 0xffffffff, 0x00090008,
1034         0x3c084, 0xffffffff, 0x00010000,
1035         0x3c088, 0xffffffff, 0x00030002,
1036         0x3c08c, 0xffffffff, 0x00040007,
1037         0x3c090, 0xffffffff, 0x00060005,
1038         0x3c094, 0xffffffff, 0x00090008,
1039         0x3c098, 0xffffffff, 0x00010000,
1040         0x3c09c, 0xffffffff, 0x00030002,
1041         0x3c0a0, 0xffffffff, 0x00040007,
1042         0x3c0a4, 0xffffffff, 0x00060005,
1043         0x3c0a8, 0xffffffff, 0x00090008,
1044         0x3c000, 0xffffffff, 0x96e00200,
1045         0x8708, 0xffffffff, 0x00900100,
1046         0xc424, 0xffffffff, 0x0020003f,
1047         0x38, 0xffffffff, 0x0140001c,
1048         0x3c, 0x000f0000, 0x000f0000,
1049         0x220, 0xffffffff, 0xC060000C,
1050         0x224, 0xc0000fff, 0x00000100,
1051         0xf90, 0xffffffff, 0x00000100,
1052         0xf98, 0x00000101, 0x00000000,
1053         0x20a8, 0xffffffff, 0x00000104,
1054         0x55e4, 0xff000fff, 0x00000100,
1055         0x30cc, 0xc0000fff, 0x00000104,
1056         0xc1e4, 0x00000001, 0x00000001,
1057         0xd00c, 0xff000ff0, 0x00000100,
1058         0xd80c, 0xff000ff0, 0x00000100
1059 };
1060
1061 static const u32 spectre_golden_spm_registers[] =
1062 {
1063         0x30800, 0xe0ffffff, 0xe0000000
1064 };
1065
1066 static const u32 spectre_golden_common_registers[] =
1067 {
1068         0xc770, 0xffffffff, 0x00000800,
1069         0xc774, 0xffffffff, 0x00000800,
1070         0xc798, 0xffffffff, 0x00007fbf,
1071         0xc79c, 0xffffffff, 0x00007faf
1072 };
1073
1074 static const u32 spectre_golden_registers[] =
1075 {
1076         0x3c000, 0xffff1fff, 0x96940200,
1077         0x3c00c, 0xffff0001, 0xff000000,
1078         0x3c200, 0xfffc0fff, 0x00000100,
1079         0x6ed8, 0x00010101, 0x00010000,
1080         0x9834, 0xf00fffff, 0x00000400,
1081         0x9838, 0xfffffffc, 0x00020200,
1082         0x5bb0, 0x000000f0, 0x00000070,
1083         0x5bc0, 0xf0311fff, 0x80300000,
1084         0x98f8, 0x73773777, 0x12010001,
1085         0x9b7c, 0x00ff0000, 0x00fc0000,
1086         0x2f48, 0x73773777, 0x12010001,
1087         0x8a14, 0xf000003f, 0x00000007,
1088         0x8b24, 0xffffffff, 0x00ffffff,
1089         0x28350, 0x3f3f3fff, 0x00000082,
1090         0x28354, 0x0000003f, 0x00000000,
1091         0x3e78, 0x00000001, 0x00000002,
1092         0x913c, 0xffff03df, 0x00000004,
1093         0xc768, 0x00000008, 0x00000008,
1094         0x8c00, 0x000008ff, 0x00000800,
1095         0x9508, 0x00010000, 0x00010000,
1096         0xac0c, 0xffffffff, 0x54763210,
1097         0x214f8, 0x01ff01ff, 0x00000002,
1098         0x21498, 0x007ff800, 0x00200000,
1099         0x2015c, 0xffffffff, 0x00000f40,
1100         0x30934, 0xffffffff, 0x00000001
1101 };
1102
1103 static const u32 spectre_mgcg_cgcg_init[] =
1104 {
1105         0xc420, 0xffffffff, 0xfffffffc,
1106         0x30800, 0xffffffff, 0xe0000000,
1107         0x3c2a0, 0xffffffff, 0x00000100,
1108         0x3c208, 0xffffffff, 0x00000100,
1109         0x3c2c0, 0xffffffff, 0x00000100,
1110         0x3c2c8, 0xffffffff, 0x00000100,
1111         0x3c2c4, 0xffffffff, 0x00000100,
1112         0x55e4, 0xffffffff, 0x00600100,
1113         0x3c280, 0xffffffff, 0x00000100,
1114         0x3c214, 0xffffffff, 0x06000100,
1115         0x3c220, 0xffffffff, 0x00000100,
1116         0x3c218, 0xffffffff, 0x06000100,
1117         0x3c204, 0xffffffff, 0x00000100,
1118         0x3c2e0, 0xffffffff, 0x00000100,
1119         0x3c224, 0xffffffff, 0x00000100,
1120         0x3c200, 0xffffffff, 0x00000100,
1121         0x3c230, 0xffffffff, 0x00000100,
1122         0x3c234, 0xffffffff, 0x00000100,
1123         0x3c250, 0xffffffff, 0x00000100,
1124         0x3c254, 0xffffffff, 0x00000100,
1125         0x3c258, 0xffffffff, 0x00000100,
1126         0x3c25c, 0xffffffff, 0x00000100,
1127         0x3c260, 0xffffffff, 0x00000100,
1128         0x3c27c, 0xffffffff, 0x00000100,
1129         0x3c278, 0xffffffff, 0x00000100,
1130         0x3c210, 0xffffffff, 0x06000100,
1131         0x3c290, 0xffffffff, 0x00000100,
1132         0x3c274, 0xffffffff, 0x00000100,
1133         0x3c2b4, 0xffffffff, 0x00000100,
1134         0x3c2b0, 0xffffffff, 0x00000100,
1135         0x3c270, 0xffffffff, 0x00000100,
1136         0x30800, 0xffffffff, 0xe0000000,
1137         0x3c020, 0xffffffff, 0x00010000,
1138         0x3c024, 0xffffffff, 0x00030002,
1139         0x3c028, 0xffffffff, 0x00040007,
1140         0x3c02c, 0xffffffff, 0x00060005,
1141         0x3c030, 0xffffffff, 0x00090008,
1142         0x3c034, 0xffffffff, 0x00010000,
1143         0x3c038, 0xffffffff, 0x00030002,
1144         0x3c03c, 0xffffffff, 0x00040007,
1145         0x3c040, 0xffffffff, 0x00060005,
1146         0x3c044, 0xffffffff, 0x00090008,
1147         0x3c048, 0xffffffff, 0x00010000,
1148         0x3c04c, 0xffffffff, 0x00030002,
1149         0x3c050, 0xffffffff, 0x00040007,
1150         0x3c054, 0xffffffff, 0x00060005,
1151         0x3c058, 0xffffffff, 0x00090008,
1152         0x3c05c, 0xffffffff, 0x00010000,
1153         0x3c060, 0xffffffff, 0x00030002,
1154         0x3c064, 0xffffffff, 0x00040007,
1155         0x3c068, 0xffffffff, 0x00060005,
1156         0x3c06c, 0xffffffff, 0x00090008,
1157         0x3c070, 0xffffffff, 0x00010000,
1158         0x3c074, 0xffffffff, 0x00030002,
1159         0x3c078, 0xffffffff, 0x00040007,
1160         0x3c07c, 0xffffffff, 0x00060005,
1161         0x3c080, 0xffffffff, 0x00090008,
1162         0x3c084, 0xffffffff, 0x00010000,
1163         0x3c088, 0xffffffff, 0x00030002,
1164         0x3c08c, 0xffffffff, 0x00040007,
1165         0x3c090, 0xffffffff, 0x00060005,
1166         0x3c094, 0xffffffff, 0x00090008,
1167         0x3c098, 0xffffffff, 0x00010000,
1168         0x3c09c, 0xffffffff, 0x00030002,
1169         0x3c0a0, 0xffffffff, 0x00040007,
1170         0x3c0a4, 0xffffffff, 0x00060005,
1171         0x3c0a8, 0xffffffff, 0x00090008,
1172         0x3c0ac, 0xffffffff, 0x00010000,
1173         0x3c0b0, 0xffffffff, 0x00030002,
1174         0x3c0b4, 0xffffffff, 0x00040007,
1175         0x3c0b8, 0xffffffff, 0x00060005,
1176         0x3c0bc, 0xffffffff, 0x00090008,
1177         0x3c000, 0xffffffff, 0x96e00200,
1178         0x8708, 0xffffffff, 0x00900100,
1179         0xc424, 0xffffffff, 0x0020003f,
1180         0x38, 0xffffffff, 0x0140001c,
1181         0x3c, 0x000f0000, 0x000f0000,
1182         0x220, 0xffffffff, 0xC060000C,
1183         0x224, 0xc0000fff, 0x00000100,
1184         0xf90, 0xffffffff, 0x00000100,
1185         0xf98, 0x00000101, 0x00000000,
1186         0x20a8, 0xffffffff, 0x00000104,
1187         0x55e4, 0xff000fff, 0x00000100,
1188         0x30cc, 0xc0000fff, 0x00000104,
1189         0xc1e4, 0x00000001, 0x00000001,
1190         0xd00c, 0xff000ff0, 0x00000100,
1191         0xd80c, 0xff000ff0, 0x00000100
1192 };
1193
1194 static const u32 kalindi_golden_spm_registers[] =
1195 {
1196         0x30800, 0xe0ffffff, 0xe0000000
1197 };
1198
1199 static const u32 kalindi_golden_common_registers[] =
1200 {
1201         0xc770, 0xffffffff, 0x00000800,
1202         0xc774, 0xffffffff, 0x00000800,
1203         0xc798, 0xffffffff, 0x00007fbf,
1204         0xc79c, 0xffffffff, 0x00007faf
1205 };
1206
1207 static const u32 kalindi_golden_registers[] =
1208 {
1209         0x3c000, 0xffffdfff, 0x6e944040,
1210         0x55e4, 0xff607fff, 0xfc000100,
1211         0x3c220, 0xff000fff, 0x00000100,
1212         0x3c224, 0xff000fff, 0x00000100,
1213         0x3c200, 0xfffc0fff, 0x00000100,
1214         0x6ed8, 0x00010101, 0x00010000,
1215         0x9830, 0xffffffff, 0x00000000,
1216         0x9834, 0xf00fffff, 0x00000400,
1217         0x5bb0, 0x000000f0, 0x00000070,
1218         0x5bc0, 0xf0311fff, 0x80300000,
1219         0x98f8, 0x73773777, 0x12010001,
1220         0x98fc, 0xffffffff, 0x00000010,
1221         0x9b7c, 0x00ff0000, 0x00fc0000,
1222         0x8030, 0x00001f0f, 0x0000100a,
1223         0x2f48, 0x73773777, 0x12010001,
1224         0x2408, 0x000fffff, 0x000c007f,
1225         0x8a14, 0xf000003f, 0x00000007,
1226         0x8b24, 0x3fff3fff, 0x00ffcfff,
1227         0x30a04, 0x0000ff0f, 0x00000000,
1228         0x28a4c, 0x07ffffff, 0x06000000,
1229         0x4d8, 0x00000fff, 0x00000100,
1230         0x3e78, 0x00000001, 0x00000002,
1231         0xc768, 0x00000008, 0x00000008,
1232         0x8c00, 0x000000ff, 0x00000003,
1233         0x214f8, 0x01ff01ff, 0x00000002,
1234         0x21498, 0x007ff800, 0x00200000,
1235         0x2015c, 0xffffffff, 0x00000f40,
1236         0x88c4, 0x001f3ae3, 0x00000082,
1237         0x88d4, 0x0000001f, 0x00000010,
1238         0x30934, 0xffffffff, 0x00000000
1239 };
1240
1241 static const u32 kalindi_mgcg_cgcg_init[] =
1242 {
1243         0xc420, 0xffffffff, 0xfffffffc,
1244         0x30800, 0xffffffff, 0xe0000000,
1245         0x3c2a0, 0xffffffff, 0x00000100,
1246         0x3c208, 0xffffffff, 0x00000100,
1247         0x3c2c0, 0xffffffff, 0x00000100,
1248         0x3c2c8, 0xffffffff, 0x00000100,
1249         0x3c2c4, 0xffffffff, 0x00000100,
1250         0x55e4, 0xffffffff, 0x00600100,
1251         0x3c280, 0xffffffff, 0x00000100,
1252         0x3c214, 0xffffffff, 0x06000100,
1253         0x3c220, 0xffffffff, 0x00000100,
1254         0x3c218, 0xffffffff, 0x06000100,
1255         0x3c204, 0xffffffff, 0x00000100,
1256         0x3c2e0, 0xffffffff, 0x00000100,
1257         0x3c224, 0xffffffff, 0x00000100,
1258         0x3c200, 0xffffffff, 0x00000100,
1259         0x3c230, 0xffffffff, 0x00000100,
1260         0x3c234, 0xffffffff, 0x00000100,
1261         0x3c250, 0xffffffff, 0x00000100,
1262         0x3c254, 0xffffffff, 0x00000100,
1263         0x3c258, 0xffffffff, 0x00000100,
1264         0x3c25c, 0xffffffff, 0x00000100,
1265         0x3c260, 0xffffffff, 0x00000100,
1266         0x3c27c, 0xffffffff, 0x00000100,
1267         0x3c278, 0xffffffff, 0x00000100,
1268         0x3c210, 0xffffffff, 0x06000100,
1269         0x3c290, 0xffffffff, 0x00000100,
1270         0x3c274, 0xffffffff, 0x00000100,
1271         0x3c2b4, 0xffffffff, 0x00000100,
1272         0x3c2b0, 0xffffffff, 0x00000100,
1273         0x3c270, 0xffffffff, 0x00000100,
1274         0x30800, 0xffffffff, 0xe0000000,
1275         0x3c020, 0xffffffff, 0x00010000,
1276         0x3c024, 0xffffffff, 0x00030002,
1277         0x3c028, 0xffffffff, 0x00040007,
1278         0x3c02c, 0xffffffff, 0x00060005,
1279         0x3c030, 0xffffffff, 0x00090008,
1280         0x3c034, 0xffffffff, 0x00010000,
1281         0x3c038, 0xffffffff, 0x00030002,
1282         0x3c03c, 0xffffffff, 0x00040007,
1283         0x3c040, 0xffffffff, 0x00060005,
1284         0x3c044, 0xffffffff, 0x00090008,
1285         0x3c000, 0xffffffff, 0x96e00200,
1286         0x8708, 0xffffffff, 0x00900100,
1287         0xc424, 0xffffffff, 0x0020003f,
1288         0x38, 0xffffffff, 0x0140001c,
1289         0x3c, 0x000f0000, 0x000f0000,
1290         0x220, 0xffffffff, 0xC060000C,
1291         0x224, 0xc0000fff, 0x00000100,
1292         0x20a8, 0xffffffff, 0x00000104,
1293         0x55e4, 0xff000fff, 0x00000100,
1294         0x30cc, 0xc0000fff, 0x00000104,
1295         0xc1e4, 0x00000001, 0x00000001,
1296         0xd00c, 0xff000ff0, 0x00000100,
1297         0xd80c, 0xff000ff0, 0x00000100
1298 };
1299
1300 static const u32 hawaii_golden_spm_registers[] =
1301 {
1302         0x30800, 0xe0ffffff, 0xe0000000
1303 };
1304
1305 static const u32 hawaii_golden_common_registers[] =
1306 {
1307         0x30800, 0xffffffff, 0xe0000000,
1308         0x28350, 0xffffffff, 0x3a00161a,
1309         0x28354, 0xffffffff, 0x0000002e,
1310         0x9a10, 0xffffffff, 0x00018208,
1311         0x98f8, 0xffffffff, 0x12011003
1312 };
1313
1314 static const u32 hawaii_golden_registers[] =
1315 {
1316         0x3354, 0x00000333, 0x00000333,
1317         0x9a10, 0x00010000, 0x00058208,
1318         0x9830, 0xffffffff, 0x00000000,
1319         0x9834, 0xf00fffff, 0x00000400,
1320         0x9838, 0x0002021c, 0x00020200,
1321         0xc78, 0x00000080, 0x00000000,
1322         0x5bb0, 0x000000f0, 0x00000070,
1323         0x5bc0, 0xf0311fff, 0x80300000,
1324         0x350c, 0x00810000, 0x408af000,
1325         0x7030, 0x31000111, 0x00000011,
1326         0x2f48, 0x73773777, 0x12010001,
1327         0x2120, 0x0000007f, 0x0000001b,
1328         0x21dc, 0x00007fb6, 0x00002191,
1329         0x3628, 0x0000003f, 0x0000000a,
1330         0x362c, 0x0000003f, 0x0000000a,
1331         0x2ae4, 0x00073ffe, 0x000022a2,
1332         0x240c, 0x000007ff, 0x00000000,
1333         0x8bf0, 0x00002001, 0x00000001,
1334         0x8b24, 0xffffffff, 0x00ffffff,
1335         0x30a04, 0x0000ff0f, 0x00000000,
1336         0x28a4c, 0x07ffffff, 0x06000000,
1337         0x3e78, 0x00000001, 0x00000002,
1338         0xc768, 0x00000008, 0x00000008,
1339         0xc770, 0x00000f00, 0x00000800,
1340         0xc774, 0x00000f00, 0x00000800,
1341         0xc798, 0x00ffffff, 0x00ff7fbf,
1342         0xc79c, 0x00ffffff, 0x00ff7faf,
1343         0x8c00, 0x000000ff, 0x00000800,
1344         0xe40, 0x00001fff, 0x00001fff,
1345         0x9060, 0x0000007f, 0x00000020,
1346         0x9508, 0x00010000, 0x00010000,
1347         0xae00, 0x00100000, 0x000ff07c,
1348         0xac14, 0x000003ff, 0x0000000f,
1349         0xac10, 0xffffffff, 0x7564fdec,
1350         0xac0c, 0xffffffff, 0x3120b9a8,
1351         0xac08, 0x20000000, 0x0f9c0000
1352 };
1353
1354 static const u32 hawaii_mgcg_cgcg_init[] =
1355 {
1356         0xc420, 0xffffffff, 0xfffffffd,
1357         0x30800, 0xffffffff, 0xe0000000,
1358         0x3c2a0, 0xffffffff, 0x00000100,
1359         0x3c208, 0xffffffff, 0x00000100,
1360         0x3c2c0, 0xffffffff, 0x00000100,
1361         0x3c2c8, 0xffffffff, 0x00000100,
1362         0x3c2c4, 0xffffffff, 0x00000100,
1363         0x55e4, 0xffffffff, 0x00200100,
1364         0x3c280, 0xffffffff, 0x00000100,
1365         0x3c214, 0xffffffff, 0x06000100,
1366         0x3c220, 0xffffffff, 0x00000100,
1367         0x3c218, 0xffffffff, 0x06000100,
1368         0x3c204, 0xffffffff, 0x00000100,
1369         0x3c2e0, 0xffffffff, 0x00000100,
1370         0x3c224, 0xffffffff, 0x00000100,
1371         0x3c200, 0xffffffff, 0x00000100,
1372         0x3c230, 0xffffffff, 0x00000100,
1373         0x3c234, 0xffffffff, 0x00000100,
1374         0x3c250, 0xffffffff, 0x00000100,
1375         0x3c254, 0xffffffff, 0x00000100,
1376         0x3c258, 0xffffffff, 0x00000100,
1377         0x3c25c, 0xffffffff, 0x00000100,
1378         0x3c260, 0xffffffff, 0x00000100,
1379         0x3c27c, 0xffffffff, 0x00000100,
1380         0x3c278, 0xffffffff, 0x00000100,
1381         0x3c210, 0xffffffff, 0x06000100,
1382         0x3c290, 0xffffffff, 0x00000100,
1383         0x3c274, 0xffffffff, 0x00000100,
1384         0x3c2b4, 0xffffffff, 0x00000100,
1385         0x3c2b0, 0xffffffff, 0x00000100,
1386         0x3c270, 0xffffffff, 0x00000100,
1387         0x30800, 0xffffffff, 0xe0000000,
1388         0x3c020, 0xffffffff, 0x00010000,
1389         0x3c024, 0xffffffff, 0x00030002,
1390         0x3c028, 0xffffffff, 0x00040007,
1391         0x3c02c, 0xffffffff, 0x00060005,
1392         0x3c030, 0xffffffff, 0x00090008,
1393         0x3c034, 0xffffffff, 0x00010000,
1394         0x3c038, 0xffffffff, 0x00030002,
1395         0x3c03c, 0xffffffff, 0x00040007,
1396         0x3c040, 0xffffffff, 0x00060005,
1397         0x3c044, 0xffffffff, 0x00090008,
1398         0x3c048, 0xffffffff, 0x00010000,
1399         0x3c04c, 0xffffffff, 0x00030002,
1400         0x3c050, 0xffffffff, 0x00040007,
1401         0x3c054, 0xffffffff, 0x00060005,
1402         0x3c058, 0xffffffff, 0x00090008,
1403         0x3c05c, 0xffffffff, 0x00010000,
1404         0x3c060, 0xffffffff, 0x00030002,
1405         0x3c064, 0xffffffff, 0x00040007,
1406         0x3c068, 0xffffffff, 0x00060005,
1407         0x3c06c, 0xffffffff, 0x00090008,
1408         0x3c070, 0xffffffff, 0x00010000,
1409         0x3c074, 0xffffffff, 0x00030002,
1410         0x3c078, 0xffffffff, 0x00040007,
1411         0x3c07c, 0xffffffff, 0x00060005,
1412         0x3c080, 0xffffffff, 0x00090008,
1413         0x3c084, 0xffffffff, 0x00010000,
1414         0x3c088, 0xffffffff, 0x00030002,
1415         0x3c08c, 0xffffffff, 0x00040007,
1416         0x3c090, 0xffffffff, 0x00060005,
1417         0x3c094, 0xffffffff, 0x00090008,
1418         0x3c098, 0xffffffff, 0x00010000,
1419         0x3c09c, 0xffffffff, 0x00030002,
1420         0x3c0a0, 0xffffffff, 0x00040007,
1421         0x3c0a4, 0xffffffff, 0x00060005,
1422         0x3c0a8, 0xffffffff, 0x00090008,
1423         0x3c0ac, 0xffffffff, 0x00010000,
1424         0x3c0b0, 0xffffffff, 0x00030002,
1425         0x3c0b4, 0xffffffff, 0x00040007,
1426         0x3c0b8, 0xffffffff, 0x00060005,
1427         0x3c0bc, 0xffffffff, 0x00090008,
1428         0x3c0c0, 0xffffffff, 0x00010000,
1429         0x3c0c4, 0xffffffff, 0x00030002,
1430         0x3c0c8, 0xffffffff, 0x00040007,
1431         0x3c0cc, 0xffffffff, 0x00060005,
1432         0x3c0d0, 0xffffffff, 0x00090008,
1433         0x3c0d4, 0xffffffff, 0x00010000,
1434         0x3c0d8, 0xffffffff, 0x00030002,
1435         0x3c0dc, 0xffffffff, 0x00040007,
1436         0x3c0e0, 0xffffffff, 0x00060005,
1437         0x3c0e4, 0xffffffff, 0x00090008,
1438         0x3c0e8, 0xffffffff, 0x00010000,
1439         0x3c0ec, 0xffffffff, 0x00030002,
1440         0x3c0f0, 0xffffffff, 0x00040007,
1441         0x3c0f4, 0xffffffff, 0x00060005,
1442         0x3c0f8, 0xffffffff, 0x00090008,
1443         0xc318, 0xffffffff, 0x00020200,
1444         0x3350, 0xffffffff, 0x00000200,
1445         0x15c0, 0xffffffff, 0x00000400,
1446         0x55e8, 0xffffffff, 0x00000000,
1447         0x2f50, 0xffffffff, 0x00000902,
1448         0x3c000, 0xffffffff, 0x96940200,
1449         0x8708, 0xffffffff, 0x00900100,
1450         0xc424, 0xffffffff, 0x0020003f,
1451         0x38, 0xffffffff, 0x0140001c,
1452         0x3c, 0x000f0000, 0x000f0000,
1453         0x220, 0xffffffff, 0xc060000c,
1454         0x224, 0xc0000fff, 0x00000100,
1455         0xf90, 0xffffffff, 0x00000100,
1456         0xf98, 0x00000101, 0x00000000,
1457         0x20a8, 0xffffffff, 0x00000104,
1458         0x55e4, 0xff000fff, 0x00000100,
1459         0x30cc, 0xc0000fff, 0x00000104,
1460         0xc1e4, 0x00000001, 0x00000001,
1461         0xd00c, 0xff000ff0, 0x00000100,
1462         0xd80c, 0xff000ff0, 0x00000100
1463 };
1464
1465 static const u32 godavari_golden_registers[] =
1466 {
1467         0x55e4, 0xff607fff, 0xfc000100,
1468         0x6ed8, 0x00010101, 0x00010000,
1469         0x9830, 0xffffffff, 0x00000000,
1470         0x98302, 0xf00fffff, 0x00000400,
1471         0x6130, 0xffffffff, 0x00010000,
1472         0x5bb0, 0x000000f0, 0x00000070,
1473         0x5bc0, 0xf0311fff, 0x80300000,
1474         0x98f8, 0x73773777, 0x12010001,
1475         0x98fc, 0xffffffff, 0x00000010,
1476         0x8030, 0x00001f0f, 0x0000100a,
1477         0x2f48, 0x73773777, 0x12010001,
1478         0x2408, 0x000fffff, 0x000c007f,
1479         0x8a14, 0xf000003f, 0x00000007,
1480         0x8b24, 0xffffffff, 0x00ff0fff,
1481         0x30a04, 0x0000ff0f, 0x00000000,
1482         0x28a4c, 0x07ffffff, 0x06000000,
1483         0x4d8, 0x00000fff, 0x00000100,
1484         0xd014, 0x00010000, 0x00810001,
1485         0xd814, 0x00010000, 0x00810001,
1486         0x3e78, 0x00000001, 0x00000002,
1487         0xc768, 0x00000008, 0x00000008,
1488         0xc770, 0x00000f00, 0x00000800,
1489         0xc774, 0x00000f00, 0x00000800,
1490         0xc798, 0x00ffffff, 0x00ff7fbf,
1491         0xc79c, 0x00ffffff, 0x00ff7faf,
1492         0x8c00, 0x000000ff, 0x00000001,
1493         0x214f8, 0x01ff01ff, 0x00000002,
1494         0x21498, 0x007ff800, 0x00200000,
1495         0x2015c, 0xffffffff, 0x00000f40,
1496         0x88c4, 0x001f3ae3, 0x00000082,
1497         0x88d4, 0x0000001f, 0x00000010,
1498         0x30934, 0xffffffff, 0x00000000
1499 };
1500
1501
1502 static void cik_init_golden_registers(struct radeon_device *rdev)
1503 {
1504         switch (rdev->family) {
1505         case CHIP_BONAIRE:
1506                 radeon_program_register_sequence(rdev,
1507                                                  bonaire_mgcg_cgcg_init,
1508                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1509                 radeon_program_register_sequence(rdev,
1510                                                  bonaire_golden_registers,
1511                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1512                 radeon_program_register_sequence(rdev,
1513                                                  bonaire_golden_common_registers,
1514                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1515                 radeon_program_register_sequence(rdev,
1516                                                  bonaire_golden_spm_registers,
1517                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1518                 break;
1519         case CHIP_KABINI:
1520                 radeon_program_register_sequence(rdev,
1521                                                  kalindi_mgcg_cgcg_init,
1522                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1523                 radeon_program_register_sequence(rdev,
1524                                                  kalindi_golden_registers,
1525                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1526                 radeon_program_register_sequence(rdev,
1527                                                  kalindi_golden_common_registers,
1528                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1529                 radeon_program_register_sequence(rdev,
1530                                                  kalindi_golden_spm_registers,
1531                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1532                 break;
1533         case CHIP_MULLINS:
1534                 radeon_program_register_sequence(rdev,
1535                                                  kalindi_mgcg_cgcg_init,
1536                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1537                 radeon_program_register_sequence(rdev,
1538                                                  godavari_golden_registers,
1539                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
1540                 radeon_program_register_sequence(rdev,
1541                                                  kalindi_golden_common_registers,
1542                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1543                 radeon_program_register_sequence(rdev,
1544                                                  kalindi_golden_spm_registers,
1545                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1546                 break;
1547         case CHIP_KAVERI:
1548                 radeon_program_register_sequence(rdev,
1549                                                  spectre_mgcg_cgcg_init,
1550                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1551                 radeon_program_register_sequence(rdev,
1552                                                  spectre_golden_registers,
1553                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1554                 radeon_program_register_sequence(rdev,
1555                                                  spectre_golden_common_registers,
1556                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1557                 radeon_program_register_sequence(rdev,
1558                                                  spectre_golden_spm_registers,
1559                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1560                 break;
1561         case CHIP_HAWAII:
1562                 radeon_program_register_sequence(rdev,
1563                                                  hawaii_mgcg_cgcg_init,
1564                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1565                 radeon_program_register_sequence(rdev,
1566                                                  hawaii_golden_registers,
1567                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1568                 radeon_program_register_sequence(rdev,
1569                                                  hawaii_golden_common_registers,
1570                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1571                 radeon_program_register_sequence(rdev,
1572                                                  hawaii_golden_spm_registers,
1573                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1574                 break;
1575         default:
1576                 break;
1577         }
1578 }
1579
1580 /**
1581  * cik_get_xclk - get the xclk
1582  *
1583  * @rdev: radeon_device pointer
1584  *
1585  * Returns the reference clock used by the gfx engine
1586  * (CIK).
1587  */
1588 u32 cik_get_xclk(struct radeon_device *rdev)
1589 {
1590         u32 reference_clock = rdev->clock.spll.reference_freq;
1591
1592         if (rdev->flags & RADEON_IS_IGP) {
1593                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1594                         return reference_clock / 2;
1595         } else {
1596                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1597                         return reference_clock / 4;
1598         }
1599         return reference_clock;
1600 }
1601
1602 /**
1603  * cik_mm_rdoorbell - read a doorbell dword
1604  *
1605  * @rdev: radeon_device pointer
1606  * @index: doorbell index
1607  *
1608  * Returns the value in the doorbell aperture at the
1609  * requested doorbell index (CIK).
1610  */
1611 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1612 {
1613         if (index < rdev->doorbell.num_doorbells) {
1614                 return readl(rdev->doorbell.ptr + index);
1615         } else {
1616                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1617                 return 0;
1618         }
1619 }
1620
1621 /**
1622  * cik_mm_wdoorbell - write a doorbell dword
1623  *
1624  * @rdev: radeon_device pointer
1625  * @index: doorbell index
1626  * @v: value to write
1627  *
1628  * Writes @v to the doorbell aperture at the
1629  * requested doorbell index (CIK).
1630  */
1631 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1632 {
1633         if (index < rdev->doorbell.num_doorbells) {
1634                 writel(v, rdev->doorbell.ptr + index);
1635         } else {
1636                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1637         }
1638 }
1639
1640 #define BONAIRE_IO_MC_REGS_SIZE 36
1641
1642 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1643 {
1644         {0x00000070, 0x04400000},
1645         {0x00000071, 0x80c01803},
1646         {0x00000072, 0x00004004},
1647         {0x00000073, 0x00000100},
1648         {0x00000074, 0x00ff0000},
1649         {0x00000075, 0x34000000},
1650         {0x00000076, 0x08000014},
1651         {0x00000077, 0x00cc08ec},
1652         {0x00000078, 0x00000400},
1653         {0x00000079, 0x00000000},
1654         {0x0000007a, 0x04090000},
1655         {0x0000007c, 0x00000000},
1656         {0x0000007e, 0x4408a8e8},
1657         {0x0000007f, 0x00000304},
1658         {0x00000080, 0x00000000},
1659         {0x00000082, 0x00000001},
1660         {0x00000083, 0x00000002},
1661         {0x00000084, 0xf3e4f400},
1662         {0x00000085, 0x052024e3},
1663         {0x00000087, 0x00000000},
1664         {0x00000088, 0x01000000},
1665         {0x0000008a, 0x1c0a0000},
1666         {0x0000008b, 0xff010000},
1667         {0x0000008d, 0xffffefff},
1668         {0x0000008e, 0xfff3efff},
1669         {0x0000008f, 0xfff3efbf},
1670         {0x00000092, 0xf7ffffff},
1671         {0x00000093, 0xffffff7f},
1672         {0x00000095, 0x00101101},
1673         {0x00000096, 0x00000fff},
1674         {0x00000097, 0x00116fff},
1675         {0x00000098, 0x60010000},
1676         {0x00000099, 0x10010000},
1677         {0x0000009a, 0x00006000},
1678         {0x0000009b, 0x00001000},
1679         {0x0000009f, 0x00b48000}
1680 };
1681
1682 #define HAWAII_IO_MC_REGS_SIZE 22
1683
1684 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1685 {
1686         {0x0000007d, 0x40000000},
1687         {0x0000007e, 0x40180304},
1688         {0x0000007f, 0x0000ff00},
1689         {0x00000081, 0x00000000},
1690         {0x00000083, 0x00000800},
1691         {0x00000086, 0x00000000},
1692         {0x00000087, 0x00000100},
1693         {0x00000088, 0x00020100},
1694         {0x00000089, 0x00000000},
1695         {0x0000008b, 0x00040000},
1696         {0x0000008c, 0x00000100},
1697         {0x0000008e, 0xff010000},
1698         {0x00000090, 0xffffefff},
1699         {0x00000091, 0xfff3efff},
1700         {0x00000092, 0xfff3efbf},
1701         {0x00000093, 0xf7ffffff},
1702         {0x00000094, 0xffffff7f},
1703         {0x00000095, 0x00000fff},
1704         {0x00000096, 0x00116fff},
1705         {0x00000097, 0x60010000},
1706         {0x00000098, 0x10010000},
1707         {0x0000009f, 0x00c79000}
1708 };
1709
1710
1711 /**
1712  * cik_srbm_select - select specific register instances
1713  *
1714  * @rdev: radeon_device pointer
1715  * @me: selected ME (micro engine)
1716  * @pipe: pipe
1717  * @queue: queue
1718  * @vmid: VMID
1719  *
1720  * Switches the currently active registers instances.  Some
1721  * registers are instanced per VMID, others are instanced per
1722  * me/pipe/queue combination.
1723  */
1724 static void cik_srbm_select(struct radeon_device *rdev,
1725                             u32 me, u32 pipe, u32 queue, u32 vmid)
1726 {
1727         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1728                              MEID(me & 0x3) |
1729                              VMID(vmid & 0xf) |
1730                              QUEUEID(queue & 0x7));
1731         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1732 }
1733
1734 /* ucode loading */
1735 /**
1736  * ci_mc_load_microcode - load MC ucode into the hw
1737  *
1738  * @rdev: radeon_device pointer
1739  *
1740  * Load the GDDR MC ucode into the hw (CIK).
1741  * Returns 0 on success, error on failure.
1742  */
1743 int ci_mc_load_microcode(struct radeon_device *rdev)
1744 {
1745         const __be32 *fw_data;
1746         u32 running, blackout = 0;
1747         u32 *io_mc_regs;
1748         int i, regs_size, ucode_size;
1749
1750         if (!rdev->mc_fw)
1751                 return -EINVAL;
1752
1753         ucode_size = rdev->mc_fw->datasize / 4;
1754
1755         switch (rdev->family) {
1756         case CHIP_BONAIRE:
1757                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1758                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1759                 break;
1760         case CHIP_HAWAII:
1761                 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1762                 regs_size = HAWAII_IO_MC_REGS_SIZE;
1763                 break;
1764         default:
1765                 return -EINVAL;
1766         }
1767
1768         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1769
1770         if (running == 0) {
1771                 if (running) {
1772                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1773                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1774                 }
1775
1776                 /* reset the engine and set to writable */
1777                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1778                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1779
1780                 /* load mc io regs */
1781                 for (i = 0; i < regs_size; i++) {
1782                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1783                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1784                 }
1785                 /* load the MC ucode */
1786                 fw_data = (const __be32 *)rdev->mc_fw->data;
1787                 for (i = 0; i < ucode_size; i++)
1788                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1789
1790                 /* put the engine back into the active state */
1791                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1792                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1793                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1794
1795                 /* wait for training to complete */
1796                 for (i = 0; i < rdev->usec_timeout; i++) {
1797                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1798                                 break;
1799                         udelay(1);
1800                 }
1801                 for (i = 0; i < rdev->usec_timeout; i++) {
1802                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1803                                 break;
1804                         udelay(1);
1805                 }
1806
1807                 if (running)
1808                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1809         }
1810
1811         return 0;
1812 }
1813
1814 /**
1815  * cik_init_microcode - load ucode images from disk
1816  *
1817  * @rdev: radeon_device pointer
1818  *
1819  * Use the firmware interface to load the ucode images into
1820  * the driver (not loaded into hw).
1821  * Returns 0 on success, error on failure.
1822  */
1823 static int cik_init_microcode(struct radeon_device *rdev)
1824 {
1825         const char *chip_name;
1826         size_t pfp_req_size, me_req_size, ce_req_size,
1827                 mec_req_size, rlc_req_size, mc_req_size = 0,
1828                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1829         char fw_name[30];
1830         int err;
1831
1832         DRM_DEBUG("\n");
1833
1834         switch (rdev->family) {
1835         case CHIP_BONAIRE:
1836                 chip_name = "BONAIRE";
1837                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1838                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1839                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1840                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1841                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1842                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1843                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1844                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1845                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1846                 break;
1847         case CHIP_HAWAII:
1848                 chip_name = "HAWAII";
1849                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1850                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1851                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1852                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1853                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1854                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1855                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1856                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1857                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1858                 break;
1859         case CHIP_KAVERI:
1860                 chip_name = "KAVERI";
1861                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1862                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1863                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1864                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1865                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1866                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1867                 break;
1868         case CHIP_KABINI:
1869                 chip_name = "KABINI";
1870                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1871                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1872                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1873                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1874                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1875                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1876                 break;
1877         case CHIP_MULLINS:
1878                 chip_name = "MULLINS";
1879                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1880                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1881                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1882                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1883                 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1884                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1885                 break;
1886         default: BUG();
1887         }
1888
1889         DRM_INFO("Loading %s Microcode\n", chip_name);
1890
1891         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
1892         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1893         if (err)
1894                 goto out;
1895         if (rdev->pfp_fw->datasize != pfp_req_size) {
1896                 printk(KERN_ERR
1897                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1898                        rdev->pfp_fw->datasize, fw_name);
1899                 err = -EINVAL;
1900                 goto out;
1901         }
1902
1903         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
1904         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1905         if (err)
1906                 goto out;
1907         if (rdev->me_fw->datasize != me_req_size) {
1908                 printk(KERN_ERR
1909                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1910                        rdev->me_fw->datasize, fw_name);
1911                 err = -EINVAL;
1912         }
1913
1914         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name);
1915         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1916         if (err)
1917                 goto out;
1918         if (rdev->ce_fw->datasize != ce_req_size) {
1919                 printk(KERN_ERR
1920                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1921                        rdev->ce_fw->datasize, fw_name);
1922                 err = -EINVAL;
1923         }
1924
1925         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mec", chip_name);
1926         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1927         if (err)
1928                 goto out;
1929         if (rdev->mec_fw->datasize != mec_req_size) {
1930                 printk(KERN_ERR
1931                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1932                        rdev->mec_fw->datasize, fw_name);
1933                 err = -EINVAL;
1934         }
1935
1936         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", chip_name);
1937         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1938         if (err)
1939                 goto out;
1940         if (rdev->rlc_fw->datasize != rlc_req_size) {
1941                 printk(KERN_ERR
1942                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1943                        rdev->rlc_fw->datasize, fw_name);
1944                 err = -EINVAL;
1945         }
1946
1947         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_sdma", chip_name);
1948         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1949         if (err)
1950                 goto out;
1951         if (rdev->sdma_fw->datasize != sdma_req_size) {
1952                 printk(KERN_ERR
1953                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1954                        rdev->sdma_fw->datasize, fw_name);
1955                 err = -EINVAL;
1956         }
1957
1958         /* No SMC, MC ucode on APUs */
1959         if (!(rdev->flags & RADEON_IS_IGP)) {
1960                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc2", chip_name);
1961                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1962                 if (err) {
1963                         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
1964                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1965                         if (err)
1966                                 goto out;
1967                 }
1968                 if ((rdev->mc_fw->datasize != mc_req_size) &&
1969                     (rdev->mc_fw->datasize != mc2_req_size)){
1970                         printk(KERN_ERR
1971                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1972                                rdev->mc_fw->datasize, fw_name);
1973                         err = -EINVAL;
1974                 }
1975                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->datasize);
1976
1977                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
1978                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1979                 if (err) {
1980                         printk(KERN_ERR
1981                                "smc: error loading firmware \"%s\"\n",
1982                                fw_name);
1983                         release_firmware(rdev->smc_fw);
1984                         rdev->smc_fw = NULL;
1985                         err = 0;
1986                 } else if (rdev->smc_fw->datasize != smc_req_size) {
1987                         printk(KERN_ERR
1988                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1989                                rdev->smc_fw->datasize, fw_name);
1990                         err = -EINVAL;
1991                 }
1992         }
1993
1994 out:
1995         if (err) {
1996                 if (err != -EINVAL)
1997                         printk(KERN_ERR
1998                                "cik_cp: Failed to load firmware \"%s\"\n",
1999                                fw_name);
2000                 release_firmware(rdev->pfp_fw);
2001                 rdev->pfp_fw = NULL;
2002                 release_firmware(rdev->me_fw);
2003                 rdev->me_fw = NULL;
2004                 release_firmware(rdev->ce_fw);
2005                 rdev->ce_fw = NULL;
2006                 release_firmware(rdev->mec_fw);
2007                 rdev->mec_fw = NULL;
2008                 release_firmware(rdev->rlc_fw);
2009                 rdev->rlc_fw = NULL;
2010                 release_firmware(rdev->sdma_fw);
2011                 rdev->sdma_fw = NULL;
2012                 release_firmware(rdev->mc_fw);
2013                 rdev->mc_fw = NULL;
2014                 release_firmware(rdev->smc_fw);
2015                 rdev->smc_fw = NULL;
2016         }
2017         return err;
2018 }
2019
2020 /*
2021  * Core functions
2022  */
2023 /**
2024  * cik_tiling_mode_table_init - init the hw tiling table
2025  *
2026  * @rdev: radeon_device pointer
2027  *
2028  * Starting with SI, the tiling setup is done globally in a
2029  * set of 32 tiling modes.  Rather than selecting each set of
2030  * parameters per surface as on older asics, we just select
2031  * which index in the tiling table we want to use, and the
2032  * surface uses those parameters (CIK).
2033  */
2034 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2035 {
2036         const u32 num_tile_mode_states = 32;
2037         const u32 num_secondary_tile_mode_states = 16;
2038         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2039         u32 num_pipe_configs;
2040         u32 num_rbs = rdev->config.cik.max_backends_per_se *
2041                 rdev->config.cik.max_shader_engines;
2042
2043         switch (rdev->config.cik.mem_row_size_in_kb) {
2044         case 1:
2045                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2046                 break;
2047         case 2:
2048         default:
2049                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2050                 break;
2051         case 4:
2052                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2053                 break;
2054         }
2055
2056         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2057         if (num_pipe_configs > 8)
2058                 num_pipe_configs = 16;
2059
2060         if (num_pipe_configs == 16) {
2061                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2062                         switch (reg_offset) {
2063                         case 0:
2064                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2065                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2066                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2067                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2068                                 break;
2069                         case 1:
2070                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2071                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2072                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2073                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2074                                 break;
2075                         case 2:
2076                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2078                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2080                                 break;
2081                         case 3:
2082                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2083                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2084                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2085                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2086                                 break;
2087                         case 4:
2088                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2089                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2090                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091                                                  TILE_SPLIT(split_equal_to_row_size));
2092                                 break;
2093                         case 5:
2094                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2096                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2097                                 break;
2098                         case 6:
2099                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2100                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2101                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2102                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2103                                 break;
2104                         case 7:
2105                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2106                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2107                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2108                                                  TILE_SPLIT(split_equal_to_row_size));
2109                                 break;
2110                         case 8:
2111                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2112                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2113                                 break;
2114                         case 9:
2115                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2116                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2117                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2118                                 break;
2119                         case 10:
2120                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2121                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2122                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2123                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2124                                 break;
2125                         case 11:
2126                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2127                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2128                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2129                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2130                                 break;
2131                         case 12:
2132                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2133                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2134                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2135                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2136                                 break;
2137                         case 13:
2138                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2139                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2140                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2141                                 break;
2142                         case 14:
2143                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2144                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2145                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2146                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2147                                 break;
2148                         case 16:
2149                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2150                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2151                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2152                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2153                                 break;
2154                         case 17:
2155                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2156                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2157                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2158                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2159                                 break;
2160                         case 27:
2161                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2162                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2163                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2164                                 break;
2165                         case 28:
2166                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2167                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2168                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2169                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2170                                 break;
2171                         case 29:
2172                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2173                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2174                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2175                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2176                                 break;
2177                         case 30:
2178                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2179                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2180                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2181                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2182                                 break;
2183                         default:
2184                                 gb_tile_moden = 0;
2185                                 break;
2186                         }
2187                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2188                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2189                 }
2190                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2191                         switch (reg_offset) {
2192                         case 0:
2193                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2194                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2195                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2196                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2197                                 break;
2198                         case 1:
2199                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2200                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2201                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2202                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2203                                 break;
2204                         case 2:
2205                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2206                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2207                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2208                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2209                                 break;
2210                         case 3:
2211                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2212                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2213                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2214                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2215                                 break;
2216                         case 4:
2217                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2218                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2219                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2220                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2221                                 break;
2222                         case 5:
2223                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2224                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2225                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2226                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2227                                 break;
2228                         case 6:
2229                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2230                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2231                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2232                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2233                                 break;
2234                         case 8:
2235                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2236                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2237                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2238                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2239                                 break;
2240                         case 9:
2241                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2242                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2243                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2244                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2245                                 break;
2246                         case 10:
2247                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2248                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2249                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2250                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2251                                 break;
2252                         case 11:
2253                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2254                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2255                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2256                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2257                                 break;
2258                         case 12:
2259                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2260                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2261                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2262                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2263                                 break;
2264                         case 13:
2265                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2266                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2267                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2268                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2269                                 break;
2270                         case 14:
2271                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2272                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2273                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2274                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2275                                 break;
2276                         default:
2277                                 gb_tile_moden = 0;
2278                                 break;
2279                         }
2280                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2281                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2282                 }
2283         } else if (num_pipe_configs == 8) {
2284                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2285                         switch (reg_offset) {
2286                         case 0:
2287                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2288                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2289                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2290                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2291                                 break;
2292                         case 1:
2293                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2294                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2295                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2296                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2297                                 break;
2298                         case 2:
2299                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2300                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2301                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2302                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2303                                 break;
2304                         case 3:
2305                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2306                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2307                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2308                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2309                                 break;
2310                         case 4:
2311                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2313                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2314                                                  TILE_SPLIT(split_equal_to_row_size));
2315                                 break;
2316                         case 5:
2317                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2318                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2320                                 break;
2321                         case 6:
2322                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2323                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2324                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2325                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2326                                 break;
2327                         case 7:
2328                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2330                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331                                                  TILE_SPLIT(split_equal_to_row_size));
2332                                 break;
2333                         case 8:
2334                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2335                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2336                                 break;
2337                         case 9:
2338                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2339                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2340                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2341                                 break;
2342                         case 10:
2343                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2344                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2345                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2346                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2347                                 break;
2348                         case 11:
2349                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2350                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2351                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2352                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2353                                 break;
2354                         case 12:
2355                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2356                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2357                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2358                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2359                                 break;
2360                         case 13:
2361                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2362                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2363                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2364                                 break;
2365                         case 14:
2366                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2367                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2368                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2369                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2370                                 break;
2371                         case 16:
2372                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2373                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2374                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2375                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2376                                 break;
2377                         case 17:
2378                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2379                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2380                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2381                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2382                                 break;
2383                         case 27:
2384                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2385                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2386                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2387                                 break;
2388                         case 28:
2389                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2391                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2392                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2393                                 break;
2394                         case 29:
2395                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2396                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2397                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2398                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2399                                 break;
2400                         case 30:
2401                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2402                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2403                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2404                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2405                                 break;
2406                         default:
2407                                 gb_tile_moden = 0;
2408                                 break;
2409                         }
2410                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2411                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2412                 }
2413                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2414                         switch (reg_offset) {
2415                         case 0:
2416                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2418                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2419                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2420                                 break;
2421                         case 1:
2422                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2424                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2425                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2426                                 break;
2427                         case 2:
2428                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2430                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2431                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2432                                 break;
2433                         case 3:
2434                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2437                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2438                                 break;
2439                         case 4:
2440                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2443                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2444                                 break;
2445                         case 5:
2446                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2447                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2448                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2449                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2450                                 break;
2451                         case 6:
2452                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2454                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2455                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2456                                 break;
2457                         case 8:
2458                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2460                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2461                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2462                                 break;
2463                         case 9:
2464                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2466                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2467                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2468                                 break;
2469                         case 10:
2470                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2471                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2472                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2473                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2474                                 break;
2475                         case 11:
2476                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2479                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2480                                 break;
2481                         case 12:
2482                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2483                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2484                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2485                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2486                                 break;
2487                         case 13:
2488                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2492                                 break;
2493                         case 14:
2494                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2495                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2496                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2497                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2498                                 break;
2499                         default:
2500                                 gb_tile_moden = 0;
2501                                 break;
2502                         }
2503                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2504                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2505                 }
2506         } else if (num_pipe_configs == 4) {
2507                 if (num_rbs == 4) {
2508                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2509                                 switch (reg_offset) {
2510                                 case 0:
2511                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2512                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2513                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2514                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2515                                         break;
2516                                 case 1:
2517                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2518                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2519                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2520                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2521                                         break;
2522                                 case 2:
2523                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2525                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2526                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2527                                         break;
2528                                 case 3:
2529                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2530                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2531                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2532                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2533                                         break;
2534                                 case 4:
2535                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2537                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2538                                                          TILE_SPLIT(split_equal_to_row_size));
2539                                         break;
2540                                 case 5:
2541                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2542                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2543                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2544                                         break;
2545                                 case 6:
2546                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2547                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2548                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2549                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2550                                         break;
2551                                 case 7:
2552                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2553                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2554                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2555                                                          TILE_SPLIT(split_equal_to_row_size));
2556                                         break;
2557                                 case 8:
2558                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2559                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2560                                         break;
2561                                 case 9:
2562                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2563                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2564                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2565                                         break;
2566                                 case 10:
2567                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2568                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2569                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2570                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2571                                         break;
2572                                 case 11:
2573                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2574                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2575                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2576                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2577                                         break;
2578                                 case 12:
2579                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2580                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2581                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2582                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2583                                         break;
2584                                 case 13:
2585                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2586                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2587                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2588                                         break;
2589                                 case 14:
2590                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2591                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2592                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2593                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2594                                         break;
2595                                 case 16:
2596                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2597                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2598                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2599                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2600                                         break;
2601                                 case 17:
2602                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2603                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2604                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2605                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2606                                         break;
2607                                 case 27:
2608                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2609                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2610                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2611                                         break;
2612                                 case 28:
2613                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2614                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2615                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2616                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2617                                         break;
2618                                 case 29:
2619                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2620                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2621                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2622                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2623                                         break;
2624                                 case 30:
2625                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2626                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2627                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2628                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2629                                         break;
2630                                 default:
2631                                         gb_tile_moden = 0;
2632                                         break;
2633                                 }
2634                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2635                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2636                         }
2637                 } else if (num_rbs < 4) {
2638                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2639                                 switch (reg_offset) {
2640                                 case 0:
2641                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2643                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2645                                         break;
2646                                 case 1:
2647                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2648                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2649                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2650                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2651                                         break;
2652                                 case 2:
2653                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2655                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2657                                         break;
2658                                 case 3:
2659                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2661                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2662                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2663                                         break;
2664                                 case 4:
2665                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2667                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2668                                                          TILE_SPLIT(split_equal_to_row_size));
2669                                         break;
2670                                 case 5:
2671                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2672                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2673                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2674                                         break;
2675                                 case 6:
2676                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2677                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2678                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2679                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2680                                         break;
2681                                 case 7:
2682                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2683                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2684                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2685                                                          TILE_SPLIT(split_equal_to_row_size));
2686                                         break;
2687                                 case 8:
2688                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2689                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2690                                         break;
2691                                 case 9:
2692                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2693                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2694                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2695                                         break;
2696                                 case 10:
2697                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2698                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2699                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2700                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2701                                         break;
2702                                 case 11:
2703                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2704                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2705                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2706                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2707                                         break;
2708                                 case 12:
2709                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2710                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2711                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2712                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2713                                         break;
2714                                 case 13:
2715                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2716                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2717                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2718                                         break;
2719                                 case 14:
2720                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2721                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2722                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2724                                         break;
2725                                 case 16:
2726                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2727                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2728                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2729                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2730                                         break;
2731                                 case 17:
2732                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2733                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2734                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2735                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2736                                         break;
2737                                 case 27:
2738                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2739                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2740                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2741                                         break;
2742                                 case 28:
2743                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2744                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2745                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2746                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2747                                         break;
2748                                 case 29:
2749                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2750                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2751                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2752                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2753                                         break;
2754                                 case 30:
2755                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2756                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2757                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2759                                         break;
2760                                 default:
2761                                         gb_tile_moden = 0;
2762                                         break;
2763                                 }
2764                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2765                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2766                         }
2767                 }
2768                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2769                         switch (reg_offset) {
2770                         case 0:
2771                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2773                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2774                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2775                                 break;
2776                         case 1:
2777                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2778                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2779                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2780                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2781                                 break;
2782                         case 2:
2783                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2785                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2786                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2787                                 break;
2788                         case 3:
2789                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2792                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2793                                 break;
2794                         case 4:
2795                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2798                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2799                                 break;
2800                         case 5:
2801                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2802                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2803                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2804                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2805                                 break;
2806                         case 6:
2807                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2808                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2809                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2810                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2811                                 break;
2812                         case 8:
2813                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2814                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2815                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2816                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2817                                 break;
2818                         case 9:
2819                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2820                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2821                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2822                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2823                                 break;
2824                         case 10:
2825                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2827                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2828                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2829                                 break;
2830                         case 11:
2831                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2832                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2833                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2834                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2835                                 break;
2836                         case 12:
2837                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2838                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2839                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2840                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2841                                 break;
2842                         case 13:
2843                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2845                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2846                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2847                                 break;
2848                         case 14:
2849                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2850                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2851                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2852                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2853                                 break;
2854                         default:
2855                                 gb_tile_moden = 0;
2856                                 break;
2857                         }
2858                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2859                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2860                 }
2861         } else if (num_pipe_configs == 2) {
2862                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2863                         switch (reg_offset) {
2864                         case 0:
2865                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2867                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2868                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2869                                 break;
2870                         case 1:
2871                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2872                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2873                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2874                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2875                                 break;
2876                         case 2:
2877                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2879                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2880                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2881                                 break;
2882                         case 3:
2883                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2884                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2885                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2886                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2887                                 break;
2888                         case 4:
2889                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2890                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2891                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2892                                                  TILE_SPLIT(split_equal_to_row_size));
2893                                 break;
2894                         case 5:
2895                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2896                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2897                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2898                                 break;
2899                         case 6:
2900                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2901                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2902                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2903                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2904                                 break;
2905                         case 7:
2906                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2907                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2908                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2909                                                  TILE_SPLIT(split_equal_to_row_size));
2910                                 break;
2911                         case 8:
2912                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2913                                                 PIPE_CONFIG(ADDR_SURF_P2);
2914                                 break;
2915                         case 9:
2916                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2917                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2918                                                  PIPE_CONFIG(ADDR_SURF_P2));
2919                                 break;
2920                         case 10:
2921                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2922                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2923                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2924                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2925                                 break;
2926                         case 11:
2927                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2928                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2929                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2930                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2931                                 break;
2932                         case 12:
2933                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2934                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2935                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2936                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2937                                 break;
2938                         case 13:
2939                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2940                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2941                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2942                                 break;
2943                         case 14:
2944                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2945                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2946                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2947                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2948                                 break;
2949                         case 16:
2950                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2951                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2952                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2953                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2954                                 break;
2955                         case 17:
2956                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2957                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2958                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2959                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2960                                 break;
2961                         case 27:
2962                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2963                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2964                                                  PIPE_CONFIG(ADDR_SURF_P2));
2965                                 break;
2966                         case 28:
2967                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2968                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2969                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2970                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2971                                 break;
2972                         case 29:
2973                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2974                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2975                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2976                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2977                                 break;
2978                         case 30:
2979                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2980                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2981                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2982                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2983                                 break;
2984                         default:
2985                                 gb_tile_moden = 0;
2986                                 break;
2987                         }
2988                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2989                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2990                 }
2991                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2992                         switch (reg_offset) {
2993                         case 0:
2994                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2995                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2996                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2997                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2998                                 break;
2999                         case 1:
3000                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3001                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3004                                 break;
3005                         case 2:
3006                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3007                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3008                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3010                                 break;
3011                         case 3:
3012                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3014                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3015                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3016                                 break;
3017                         case 4:
3018                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3020                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3021                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3022                                 break;
3023                         case 5:
3024                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3025                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3026                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3027                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3028                                 break;
3029                         case 6:
3030                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3031                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3032                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3033                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3034                                 break;
3035                         case 8:
3036                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3037                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3038                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3039                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3040                                 break;
3041                         case 9:
3042                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3043                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3044                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3045                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3046                                 break;
3047                         case 10:
3048                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3049                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3050                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3051                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3052                                 break;
3053                         case 11:
3054                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3055                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3056                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3057                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3058                                 break;
3059                         case 12:
3060                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3061                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3062                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3063                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3064                                 break;
3065                         case 13:
3066                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3067                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3068                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3069                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3070                                 break;
3071                         case 14:
3072                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3073                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3074                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3075                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3076                                 break;
3077                         default:
3078                                 gb_tile_moden = 0;
3079                                 break;
3080                         }
3081                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3082                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3083                 }
3084         } else
3085                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3086 }
3087
3088 /**
3089  * cik_select_se_sh - select which SE, SH to address
3090  *
3091  * @rdev: radeon_device pointer
3092  * @se_num: shader engine to address
3093  * @sh_num: sh block to address
3094  *
3095  * Select which SE, SH combinations to address. Certain
3096  * registers are instanced per SE or SH.  0xffffffff means
3097  * broadcast to all SEs or SHs (CIK).
3098  */
3099 static void cik_select_se_sh(struct radeon_device *rdev,
3100                              u32 se_num, u32 sh_num)
3101 {
3102         u32 data = INSTANCE_BROADCAST_WRITES;
3103
3104         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3105                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3106         else if (se_num == 0xffffffff)
3107                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3108         else if (sh_num == 0xffffffff)
3109                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3110         else
3111                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3112         WREG32(GRBM_GFX_INDEX, data);
3113 }
3114
3115 /**
3116  * cik_create_bitmask - create a bitmask
3117  *
3118  * @bit_width: length of the mask
3119  *
3120  * create a variable length bit mask (CIK).
3121  * Returns the bitmask.
3122  */
3123 static u32 cik_create_bitmask(u32 bit_width)
3124 {
3125         u32 i, mask = 0;
3126
3127         for (i = 0; i < bit_width; i++) {
3128                 mask <<= 1;
3129                 mask |= 1;
3130         }
3131         return mask;
3132 }
3133
3134 /**
3135  * cik_get_rb_disabled - computes the mask of disabled RBs
3136  *
3137  * @rdev: radeon_device pointer
3138  * @max_rb_num: max RBs (render backends) for the asic
3139  * @se_num: number of SEs (shader engines) for the asic
3140  * @sh_per_se: number of SH blocks per SE for the asic
3141  *
3142  * Calculates the bitmask of disabled RBs (CIK).
3143  * Returns the disabled RB bitmask.
3144  */
3145 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3146                               u32 max_rb_num_per_se,
3147                               u32 sh_per_se)
3148 {
3149         u32 data, mask;
3150
3151         data = RREG32(CC_RB_BACKEND_DISABLE);
3152         if (data & 1)
3153                 data &= BACKEND_DISABLE_MASK;
3154         else
3155                 data = 0;
3156         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3157
3158         data >>= BACKEND_DISABLE_SHIFT;
3159
3160         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3161
3162         return data & mask;
3163 }
3164
3165 /**
3166  * cik_setup_rb - setup the RBs on the asic
3167  *
3168  * @rdev: radeon_device pointer
3169  * @se_num: number of SEs (shader engines) for the asic
3170  * @sh_per_se: number of SH blocks per SE for the asic
3171  * @max_rb_num: max RBs (render backends) for the asic
3172  *
3173  * Configures per-SE/SH RB registers (CIK).
3174  */
3175 static void cik_setup_rb(struct radeon_device *rdev,
3176                          u32 se_num, u32 sh_per_se,
3177                          u32 max_rb_num_per_se)
3178 {
3179         int i, j;
3180         u32 data, mask;
3181         u32 disabled_rbs = 0;
3182         u32 enabled_rbs = 0;
3183
3184         for (i = 0; i < se_num; i++) {
3185                 for (j = 0; j < sh_per_se; j++) {
3186                         cik_select_se_sh(rdev, i, j);
3187                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3188                         if (rdev->family == CHIP_HAWAII)
3189                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3190                         else
3191                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3192                 }
3193         }
3194         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3195
3196         mask = 1;
3197         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3198                 if (!(disabled_rbs & mask))
3199                         enabled_rbs |= mask;
3200                 mask <<= 1;
3201         }
3202
3203         rdev->config.cik.backend_enable_mask = enabled_rbs;
3204
3205         for (i = 0; i < se_num; i++) {
3206                 cik_select_se_sh(rdev, i, 0xffffffff);
3207                 data = 0;
3208                 for (j = 0; j < sh_per_se; j++) {
3209                         switch (enabled_rbs & 3) {
3210                         case 0:
3211                                 if (j == 0)
3212                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3213                                 else
3214                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3215                                 break;
3216                         case 1:
3217                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3218                                 break;
3219                         case 2:
3220                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3221                                 break;
3222                         case 3:
3223                         default:
3224                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3225                                 break;
3226                         }
3227                         enabled_rbs >>= 2;
3228                 }
3229                 WREG32(PA_SC_RASTER_CONFIG, data);
3230         }
3231         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3232 }
3233
3234 /**
3235  * cik_gpu_init - setup the 3D engine
3236  *
3237  * @rdev: radeon_device pointer
3238  *
3239  * Configures the 3D engine and tiling configuration
3240  * registers so that the 3D engine is usable.
3241  */
3242 static void cik_gpu_init(struct radeon_device *rdev)
3243 {
3244         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3245         u32 mc_shared_chmap, mc_arb_ramcfg;
3246         u32 hdp_host_path_cntl;
3247         u32 tmp;
3248         int i, j;
3249
3250         switch (rdev->family) {
3251         case CHIP_BONAIRE:
3252                 rdev->config.cik.max_shader_engines = 2;
3253                 rdev->config.cik.max_tile_pipes = 4;
3254                 rdev->config.cik.max_cu_per_sh = 7;
3255                 rdev->config.cik.max_sh_per_se = 1;
3256                 rdev->config.cik.max_backends_per_se = 2;
3257                 rdev->config.cik.max_texture_channel_caches = 4;
3258                 rdev->config.cik.max_gprs = 256;
3259                 rdev->config.cik.max_gs_threads = 32;
3260                 rdev->config.cik.max_hw_contexts = 8;
3261
3262                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3263                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3264                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3265                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3266                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3267                 break;
3268         case CHIP_HAWAII:
3269                 rdev->config.cik.max_shader_engines = 4;
3270                 rdev->config.cik.max_tile_pipes = 16;
3271                 rdev->config.cik.max_cu_per_sh = 11;
3272                 rdev->config.cik.max_sh_per_se = 1;
3273                 rdev->config.cik.max_backends_per_se = 4;
3274                 rdev->config.cik.max_texture_channel_caches = 16;
3275                 rdev->config.cik.max_gprs = 256;
3276                 rdev->config.cik.max_gs_threads = 32;
3277                 rdev->config.cik.max_hw_contexts = 8;
3278
3279                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3280                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3281                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3282                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3283                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3284                 break;
3285         case CHIP_KAVERI:
3286                 rdev->config.cik.max_shader_engines = 1;
3287                 rdev->config.cik.max_tile_pipes = 4;
3288                 if ((rdev->pdev->device == 0x1304) ||
3289                     (rdev->pdev->device == 0x1305) ||
3290                     (rdev->pdev->device == 0x130C) ||
3291                     (rdev->pdev->device == 0x130F) ||
3292                     (rdev->pdev->device == 0x1310) ||
3293                     (rdev->pdev->device == 0x1311) ||
3294                     (rdev->pdev->device == 0x131C)) {
3295                         rdev->config.cik.max_cu_per_sh = 8;
3296                         rdev->config.cik.max_backends_per_se = 2;
3297                 } else if ((rdev->pdev->device == 0x1309) ||
3298                            (rdev->pdev->device == 0x130A) ||
3299                            (rdev->pdev->device == 0x130D) ||
3300                            (rdev->pdev->device == 0x1313) ||
3301                            (rdev->pdev->device == 0x131D)) {
3302                         rdev->config.cik.max_cu_per_sh = 6;
3303                         rdev->config.cik.max_backends_per_se = 2;
3304                 } else if ((rdev->pdev->device == 0x1306) ||
3305                            (rdev->pdev->device == 0x1307) ||
3306                            (rdev->pdev->device == 0x130B) ||
3307                            (rdev->pdev->device == 0x130E) ||
3308                            (rdev->pdev->device == 0x1315) ||
3309                            (rdev->pdev->device == 0x1318) ||
3310                            (rdev->pdev->device == 0x131B)) {
3311                         rdev->config.cik.max_cu_per_sh = 4;
3312                         rdev->config.cik.max_backends_per_se = 1;
3313                 } else {
3314                         rdev->config.cik.max_cu_per_sh = 3;
3315                         rdev->config.cik.max_backends_per_se = 1;
3316                 }
3317                 rdev->config.cik.max_sh_per_se = 1;
3318                 rdev->config.cik.max_texture_channel_caches = 4;
3319                 rdev->config.cik.max_gprs = 256;
3320                 rdev->config.cik.max_gs_threads = 16;
3321                 rdev->config.cik.max_hw_contexts = 8;
3322
3323                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3324                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3325                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3326                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3327                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3328                 break;
3329         case CHIP_KABINI:
3330         case CHIP_MULLINS:
3331         default:
3332                 rdev->config.cik.max_shader_engines = 1;
3333                 rdev->config.cik.max_tile_pipes = 2;
3334                 rdev->config.cik.max_cu_per_sh = 2;
3335                 rdev->config.cik.max_sh_per_se = 1;
3336                 rdev->config.cik.max_backends_per_se = 1;
3337                 rdev->config.cik.max_texture_channel_caches = 2;
3338                 rdev->config.cik.max_gprs = 256;
3339                 rdev->config.cik.max_gs_threads = 16;
3340                 rdev->config.cik.max_hw_contexts = 8;
3341
3342                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3343                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3344                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3345                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3346                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3347                 break;
3348         }
3349
3350         /* Initialize HDP */
3351         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3352                 WREG32((0x2c14 + j), 0x00000000);
3353                 WREG32((0x2c18 + j), 0x00000000);
3354                 WREG32((0x2c1c + j), 0x00000000);
3355                 WREG32((0x2c20 + j), 0x00000000);
3356                 WREG32((0x2c24 + j), 0x00000000);
3357         }
3358
3359         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3360
3361         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3362
3363         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3364         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3365
3366         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3367         rdev->config.cik.mem_max_burst_length_bytes = 256;
3368         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3369         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3370         if (rdev->config.cik.mem_row_size_in_kb > 4)
3371                 rdev->config.cik.mem_row_size_in_kb = 4;
3372         /* XXX use MC settings? */
3373         rdev->config.cik.shader_engine_tile_size = 32;
3374         rdev->config.cik.num_gpus = 1;
3375         rdev->config.cik.multi_gpu_tile_size = 64;
3376
3377         /* fix up row size */
3378         gb_addr_config &= ~ROW_SIZE_MASK;
3379         switch (rdev->config.cik.mem_row_size_in_kb) {
3380         case 1:
3381         default:
3382                 gb_addr_config |= ROW_SIZE(0);
3383                 break;
3384         case 2:
3385                 gb_addr_config |= ROW_SIZE(1);
3386                 break;
3387         case 4:
3388                 gb_addr_config |= ROW_SIZE(2);
3389                 break;
3390         }
3391
3392         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3393          * not have bank info, so create a custom tiling dword.
3394          * bits 3:0   num_pipes
3395          * bits 7:4   num_banks
3396          * bits 11:8  group_size
3397          * bits 15:12 row_size
3398          */
3399         rdev->config.cik.tile_config = 0;
3400         switch (rdev->config.cik.num_tile_pipes) {
3401         case 1:
3402                 rdev->config.cik.tile_config |= (0 << 0);
3403                 break;
3404         case 2:
3405                 rdev->config.cik.tile_config |= (1 << 0);
3406                 break;
3407         case 4:
3408                 rdev->config.cik.tile_config |= (2 << 0);
3409                 break;
3410         case 8:
3411         default:
3412                 /* XXX what about 12? */
3413                 rdev->config.cik.tile_config |= (3 << 0);
3414                 break;
3415         }
3416         rdev->config.cik.tile_config |=
3417                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3418         rdev->config.cik.tile_config |=
3419                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3420         rdev->config.cik.tile_config |=
3421                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3422
3423         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3424         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3425         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3426         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3427         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3428         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3429         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3430         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3431
3432         cik_tiling_mode_table_init(rdev);
3433
3434         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3435                      rdev->config.cik.max_sh_per_se,
3436                      rdev->config.cik.max_backends_per_se);
3437
3438         rdev->config.cik.active_cus = 0;
3439         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3440                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3441                         rdev->config.cik.active_cus +=
3442                                 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3443                 }
3444         }
3445
3446         /* set HW defaults for 3D engine */
3447         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3448
3449         WREG32(SX_DEBUG_1, 0x20);
3450
3451         WREG32(TA_CNTL_AUX, 0x00010000);
3452
3453         tmp = RREG32(SPI_CONFIG_CNTL);
3454         tmp |= 0x03000000;
3455         WREG32(SPI_CONFIG_CNTL, tmp);
3456
3457         WREG32(SQ_CONFIG, 1);
3458
3459         WREG32(DB_DEBUG, 0);
3460
3461         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3462         tmp |= 0x00000400;
3463         WREG32(DB_DEBUG2, tmp);
3464
3465         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3466         tmp |= 0x00020200;
3467         WREG32(DB_DEBUG3, tmp);
3468
3469         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3470         tmp |= 0x00018208;
3471         WREG32(CB_HW_CONTROL, tmp);
3472
3473         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3474
3475         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3476                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3477                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3478                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3479
3480         WREG32(VGT_NUM_INSTANCES, 1);
3481
3482         WREG32(CP_PERFMON_CNTL, 0);
3483
3484         WREG32(SQ_CONFIG, 0);
3485
3486         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3487                                           FORCE_EOV_MAX_REZ_CNT(255)));
3488
3489         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3490                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3491
3492         WREG32(VGT_GS_VERTEX_REUSE, 16);
3493         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3494
3495         tmp = RREG32(HDP_MISC_CNTL);
3496         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3497         WREG32(HDP_MISC_CNTL, tmp);
3498
3499         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3500         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3501
3502         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3503         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3504
3505         udelay(50);
3506 }
3507
3508 /*
3509  * GPU scratch registers helpers function.
3510  */
3511 /**
3512  * cik_scratch_init - setup driver info for CP scratch regs
3513  *
3514  * @rdev: radeon_device pointer
3515  *
3516  * Set up the number and offset of the CP scratch registers.
3517  * NOTE: use of CP scratch registers is a legacy inferface and
3518  * is not used by default on newer asics (r6xx+).  On newer asics,
3519  * memory buffers are used for fences rather than scratch regs.
3520  */
3521 static void cik_scratch_init(struct radeon_device *rdev)
3522 {
3523         int i;
3524
3525         rdev->scratch.num_reg = 7;
3526         rdev->scratch.reg_base = SCRATCH_REG0;
3527         for (i = 0; i < rdev->scratch.num_reg; i++) {
3528                 rdev->scratch.free[i] = true;
3529                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3530         }
3531 }
3532
3533 /**
3534  * cik_ring_test - basic gfx ring test
3535  *
3536  * @rdev: radeon_device pointer
3537  * @ring: radeon_ring structure holding ring information
3538  *
3539  * Allocate a scratch register and write to it using the gfx ring (CIK).
3540  * Provides a basic gfx ring test to verify that the ring is working.
3541  * Used by cik_cp_gfx_resume();
3542  * Returns 0 on success, error on failure.
3543  */
3544 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3545 {
3546         uint32_t scratch;
3547         uint32_t tmp = 0;
3548         unsigned i;
3549         int r;
3550
3551         r = radeon_scratch_get(rdev, &scratch);
3552         if (r) {
3553                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3554                 return r;
3555         }
3556         WREG32(scratch, 0xCAFEDEAD);
3557         r = radeon_ring_lock(rdev, ring, 3);
3558         if (r) {
3559                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3560                 radeon_scratch_free(rdev, scratch);
3561                 return r;
3562         }
3563         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3564         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3565         radeon_ring_write(ring, 0xDEADBEEF);
3566         radeon_ring_unlock_commit(rdev, ring, false);
3567
3568         for (i = 0; i < rdev->usec_timeout; i++) {
3569                 tmp = RREG32(scratch);
3570                 if (tmp == 0xDEADBEEF)
3571                         break;
3572                 DRM_UDELAY(1);
3573         }
3574         if (i < rdev->usec_timeout) {
3575                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3576         } else {
3577                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3578                           ring->idx, scratch, tmp);
3579                 r = -EINVAL;
3580         }
3581         radeon_scratch_free(rdev, scratch);
3582         return r;
3583 }
3584
3585 /**
3586  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3587  *
3588  * @rdev: radeon_device pointer
3589  * @ridx: radeon ring index
3590  *
3591  * Emits an hdp flush on the cp.
3592  */
3593 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3594                                        int ridx)
3595 {
3596         struct radeon_ring *ring = &rdev->ring[ridx];
3597         u32 ref_and_mask;
3598
3599         switch (ring->idx) {
3600         case CAYMAN_RING_TYPE_CP1_INDEX:
3601         case CAYMAN_RING_TYPE_CP2_INDEX:
3602         default:
3603                 switch (ring->me) {
3604                 case 0:
3605                         ref_and_mask = CP2 << ring->pipe;
3606                         break;
3607                 case 1:
3608                         ref_and_mask = CP6 << ring->pipe;
3609                         break;
3610                 default:
3611                         return;
3612                 }
3613                 break;
3614         case RADEON_RING_TYPE_GFX_INDEX:
3615                 ref_and_mask = CP0;
3616                 break;
3617         }
3618
3619         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3620         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3621                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3622                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3623         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3624         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3625         radeon_ring_write(ring, ref_and_mask);
3626         radeon_ring_write(ring, ref_and_mask);
3627         radeon_ring_write(ring, 0x20); /* poll interval */
3628 }
3629
3630 /**
3631  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3632  *
3633  * @rdev: radeon_device pointer
3634  * @fence: radeon fence object
3635  *
3636  * Emits a fence sequnce number on the gfx ring and flushes
3637  * GPU caches.
3638  */
3639 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3640                              struct radeon_fence *fence)
3641 {
3642         struct radeon_ring *ring = &rdev->ring[fence->ring];
3643         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3644
3645         /* EVENT_WRITE_EOP - flush caches, send int */
3646         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3647         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3648                                  EOP_TC_ACTION_EN |
3649                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3650                                  EVENT_INDEX(5)));
3651         radeon_ring_write(ring, addr & 0xfffffffc);
3652         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3653         radeon_ring_write(ring, fence->seq);
3654         radeon_ring_write(ring, 0);
3655 }
3656
3657 /**
3658  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3659  *
3660  * @rdev: radeon_device pointer
3661  * @fence: radeon fence object
3662  *
3663  * Emits a fence sequnce number on the compute ring and flushes
3664  * GPU caches.
3665  */
3666 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3667                                  struct radeon_fence *fence)
3668 {
3669         struct radeon_ring *ring = &rdev->ring[fence->ring];
3670         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3671
3672         /* RELEASE_MEM - flush caches, send int */
3673         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3674         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3675                                  EOP_TC_ACTION_EN |
3676                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3677                                  EVENT_INDEX(5)));
3678         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3679         radeon_ring_write(ring, addr & 0xfffffffc);
3680         radeon_ring_write(ring, upper_32_bits(addr));
3681         radeon_ring_write(ring, fence->seq);
3682         radeon_ring_write(ring, 0);
3683 }
3684
3685 /**
3686  * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3687  *
3688  * @rdev: radeon_device pointer
3689  * @ring: radeon ring buffer object
3690  * @semaphore: radeon semaphore object
3691  * @emit_wait: Is this a sempahore wait?
3692  *
3693  * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3694  * from running ahead of semaphore waits.
3695  */
3696 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3697                              struct radeon_ring *ring,
3698                              struct radeon_semaphore *semaphore,
3699                              bool emit_wait)
3700 {
3701         uint64_t addr = semaphore->gpu_addr;
3702         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3703
3704         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3705         radeon_ring_write(ring, lower_32_bits(addr));
3706         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3707
3708         if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3709                 /* Prevent the PFP from running ahead of the semaphore wait */
3710                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3711                 radeon_ring_write(ring, 0x0);
3712         }
3713
3714         return true;
3715 }
3716
3717 /**
3718  * cik_copy_cpdma - copy pages using the CP DMA engine
3719  *
3720  * @rdev: radeon_device pointer
3721  * @src_offset: src GPU address
3722  * @dst_offset: dst GPU address
3723  * @num_gpu_pages: number of GPU pages to xfer
3724  * @fence: radeon fence object
3725  *
3726  * Copy GPU paging using the CP DMA engine (CIK+).
3727  * Used by the radeon ttm implementation to move pages if
3728  * registered as the asic copy callback.
3729  */
3730 int cik_copy_cpdma(struct radeon_device *rdev,
3731                    uint64_t src_offset, uint64_t dst_offset,
3732                    unsigned num_gpu_pages,
3733                    struct radeon_fence **fence)
3734 {
3735         struct radeon_semaphore *sem = NULL;
3736         int ring_index = rdev->asic->copy.blit_ring_index;
3737         struct radeon_ring *ring = &rdev->ring[ring_index];
3738         u32 size_in_bytes, cur_size_in_bytes, control;
3739         int i, num_loops;
3740         int r = 0;
3741
3742         r = radeon_semaphore_create(rdev, &sem);
3743         if (r) {
3744                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3745                 return r;
3746         }
3747
3748         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3749         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3750         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3751         if (r) {
3752                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3753                 radeon_semaphore_free(rdev, &sem, NULL);
3754                 return r;
3755         }
3756
3757         radeon_semaphore_sync_to(sem, *fence);
3758         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3759
3760         for (i = 0; i < num_loops; i++) {
3761                 cur_size_in_bytes = size_in_bytes;
3762                 if (cur_size_in_bytes > 0x1fffff)
3763                         cur_size_in_bytes = 0x1fffff;
3764                 size_in_bytes -= cur_size_in_bytes;
3765                 control = 0;
3766                 if (size_in_bytes == 0)
3767                         control |= PACKET3_DMA_DATA_CP_SYNC;
3768                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3769                 radeon_ring_write(ring, control);
3770                 radeon_ring_write(ring, lower_32_bits(src_offset));
3771                 radeon_ring_write(ring, upper_32_bits(src_offset));
3772                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3773                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3774                 radeon_ring_write(ring, cur_size_in_bytes);
3775                 src_offset += cur_size_in_bytes;
3776                 dst_offset += cur_size_in_bytes;
3777         }
3778
3779         r = radeon_fence_emit(rdev, fence, ring->idx);
3780         if (r) {
3781                 radeon_ring_unlock_undo(rdev, ring);
3782                 radeon_semaphore_free(rdev, &sem, NULL);
3783                 return r;
3784         }
3785
3786         radeon_ring_unlock_commit(rdev, ring, false);
3787         radeon_semaphore_free(rdev, &sem, *fence);
3788
3789         return r;
3790 }
3791
3792 /*
3793  * IB stuff
3794  */
3795 /**
3796  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3797  *
3798  * @rdev: radeon_device pointer
3799  * @ib: radeon indirect buffer object
3800  *
3801  * Emits an DE (drawing engine) or CE (constant engine) IB
3802  * on the gfx ring.  IBs are usually generated by userspace
3803  * acceleration drivers and submitted to the kernel for
3804  * sheduling on the ring.  This function schedules the IB
3805  * on the gfx ring for execution by the GPU.
3806  */
3807 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3808 {
3809         struct radeon_ring *ring = &rdev->ring[ib->ring];
3810         u32 header, control = INDIRECT_BUFFER_VALID;
3811
3812         if (ib->is_const_ib) {
3813                 /* set switch buffer packet before const IB */
3814                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3815                 radeon_ring_write(ring, 0);
3816
3817                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3818         } else {
3819                 u32 next_rptr;
3820                 if (ring->rptr_save_reg) {
3821                         next_rptr = ring->wptr + 3 + 4;
3822                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3823                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3824                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3825                         radeon_ring_write(ring, next_rptr);
3826                 } else if (rdev->wb.enabled) {
3827                         next_rptr = ring->wptr + 5 + 4;
3828                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3829                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3830                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3831                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3832                         radeon_ring_write(ring, next_rptr);
3833                 }
3834
3835                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3836         }
3837
3838         control |= ib->length_dw |
3839                 (ib->vm ? (ib->vm->id << 24) : 0);
3840
3841         radeon_ring_write(ring, header);
3842         radeon_ring_write(ring,
3843 #ifdef __BIG_ENDIAN
3844                           (2 << 0) |
3845 #endif
3846                           (ib->gpu_addr & 0xFFFFFFFC));
3847         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3848         radeon_ring_write(ring, control);
3849 }
3850
3851 /**
3852  * cik_ib_test - basic gfx ring IB test
3853  *
3854  * @rdev: radeon_device pointer
3855  * @ring: radeon_ring structure holding ring information
3856  *
3857  * Allocate an IB and execute it on the gfx ring (CIK).
3858  * Provides a basic gfx ring test to verify that IBs are working.
3859  * Returns 0 on success, error on failure.
3860  */
3861 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3862 {
3863         struct radeon_ib ib;
3864         uint32_t scratch;
3865         uint32_t tmp = 0;
3866         unsigned i;
3867         int r;
3868
3869         r = radeon_scratch_get(rdev, &scratch);
3870         if (r) {
3871                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3872                 return r;
3873         }
3874         WREG32(scratch, 0xCAFEDEAD);
3875         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3876         if (r) {
3877                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3878                 radeon_scratch_free(rdev, scratch);
3879                 return r;
3880         }
3881         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3882         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3883         ib.ptr[2] = 0xDEADBEEF;
3884         ib.length_dw = 3;
3885         r = radeon_ib_schedule(rdev, &ib, NULL, false);
3886         if (r) {
3887                 radeon_scratch_free(rdev, scratch);
3888                 radeon_ib_free(rdev, &ib);
3889                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3890                 return r;
3891         }
3892         r = radeon_fence_wait(ib.fence, false);
3893         if (r) {
3894                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3895                 radeon_scratch_free(rdev, scratch);
3896                 radeon_ib_free(rdev, &ib);
3897                 return r;
3898         }
3899         for (i = 0; i < rdev->usec_timeout; i++) {
3900                 tmp = RREG32(scratch);
3901                 if (tmp == 0xDEADBEEF)
3902                         break;
3903                 DRM_UDELAY(1);
3904         }
3905         if (i < rdev->usec_timeout) {
3906                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3907         } else {
3908                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3909                           scratch, tmp);
3910                 r = -EINVAL;
3911         }
3912         radeon_scratch_free(rdev, scratch);
3913         radeon_ib_free(rdev, &ib);
3914         return r;
3915 }
3916
3917 /*
3918  * CP.
3919  * On CIK, gfx and compute now have independant command processors.
3920  *
3921  * GFX
3922  * Gfx consists of a single ring and can process both gfx jobs and
3923  * compute jobs.  The gfx CP consists of three microengines (ME):
3924  * PFP - Pre-Fetch Parser
3925  * ME - Micro Engine
3926  * CE - Constant Engine
3927  * The PFP and ME make up what is considered the Drawing Engine (DE).
3928  * The CE is an asynchronous engine used for updating buffer desciptors
3929  * used by the DE so that they can be loaded into cache in parallel
3930  * while the DE is processing state update packets.
3931  *
3932  * Compute
3933  * The compute CP consists of two microengines (ME):
3934  * MEC1 - Compute MicroEngine 1
3935  * MEC2 - Compute MicroEngine 2
3936  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3937  * The queues are exposed to userspace and are programmed directly
3938  * by the compute runtime.
3939  */
3940 /**
3941  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3942  *
3943  * @rdev: radeon_device pointer
3944  * @enable: enable or disable the MEs
3945  *
3946  * Halts or unhalts the gfx MEs.
3947  */
3948 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3949 {
3950         if (enable)
3951                 WREG32(CP_ME_CNTL, 0);
3952         else {
3953                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3954                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3955                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3956                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3957         }
3958         udelay(50);
3959 }
3960
3961 /**
3962  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3963  *
3964  * @rdev: radeon_device pointer
3965  *
3966  * Loads the gfx PFP, ME, and CE ucode.
3967  * Returns 0 for success, -EINVAL if the ucode is not available.
3968  */
3969 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3970 {
3971         const __be32 *fw_data;
3972         int i;
3973
3974         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3975                 return -EINVAL;
3976
3977         cik_cp_gfx_enable(rdev, false);
3978
3979         /* PFP */
3980         fw_data = (const __be32 *)rdev->pfp_fw->data;
3981         WREG32(CP_PFP_UCODE_ADDR, 0);
3982         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3983                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3984         WREG32(CP_PFP_UCODE_ADDR, 0);
3985
3986         /* CE */
3987         fw_data = (const __be32 *)rdev->ce_fw->data;
3988         WREG32(CP_CE_UCODE_ADDR, 0);
3989         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3990                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3991         WREG32(CP_CE_UCODE_ADDR, 0);
3992
3993         /* ME */
3994         fw_data = (const __be32 *)rdev->me_fw->data;
3995         WREG32(CP_ME_RAM_WADDR, 0);
3996         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3997                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3998         WREG32(CP_ME_RAM_WADDR, 0);
3999
4000         WREG32(CP_PFP_UCODE_ADDR, 0);
4001         WREG32(CP_CE_UCODE_ADDR, 0);
4002         WREG32(CP_ME_RAM_WADDR, 0);
4003         WREG32(CP_ME_RAM_RADDR, 0);
4004         return 0;
4005 }
4006
4007 /**
4008  * cik_cp_gfx_start - start the gfx ring
4009  *
4010  * @rdev: radeon_device pointer
4011  *
4012  * Enables the ring and loads the clear state context and other
4013  * packets required to init the ring.
4014  * Returns 0 for success, error for failure.
4015  */
4016 static int cik_cp_gfx_start(struct radeon_device *rdev)
4017 {
4018         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4019         int r, i;
4020
4021         /* init the CP */
4022         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4023         WREG32(CP_ENDIAN_SWAP, 0);
4024         WREG32(CP_DEVICE_ID, 1);
4025
4026         cik_cp_gfx_enable(rdev, true);
4027
4028         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4029         if (r) {
4030                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4031                 return r;
4032         }
4033
4034         /* init the CE partitions.  CE only used for gfx on CIK */
4035         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4036         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4037         radeon_ring_write(ring, 0x8000);
4038         radeon_ring_write(ring, 0x8000);
4039
4040         /* setup clear context state */
4041         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4042         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4043
4044         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4045         radeon_ring_write(ring, 0x80000000);
4046         radeon_ring_write(ring, 0x80000000);
4047
4048         for (i = 0; i < cik_default_size; i++)
4049                 radeon_ring_write(ring, cik_default_state[i]);
4050
4051         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4052         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4053
4054         /* set clear context state */
4055         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4056         radeon_ring_write(ring, 0);
4057
4058         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4059         radeon_ring_write(ring, 0x00000316);
4060         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4061         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4062
4063         radeon_ring_unlock_commit(rdev, ring, false);
4064
4065         return 0;
4066 }
4067
4068 /**
4069  * cik_cp_gfx_fini - stop the gfx ring
4070  *
4071  * @rdev: radeon_device pointer
4072  *
4073  * Stop the gfx ring and tear down the driver ring
4074  * info.
4075  */
4076 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4077 {
4078         cik_cp_gfx_enable(rdev, false);
4079         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4080 }
4081
4082 /**
4083  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4084  *
4085  * @rdev: radeon_device pointer
4086  *
4087  * Program the location and size of the gfx ring buffer
4088  * and test it to make sure it's working.
4089  * Returns 0 for success, error for failure.
4090  */
4091 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4092 {
4093         struct radeon_ring *ring;
4094         u32 tmp;
4095         u32 rb_bufsz;
4096         u64 rb_addr;
4097         int r;
4098
4099         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4100         if (rdev->family != CHIP_HAWAII)
4101                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4102
4103         /* Set the write pointer delay */
4104         WREG32(CP_RB_WPTR_DELAY, 0);
4105
4106         /* set the RB to use vmid 0 */
4107         WREG32(CP_RB_VMID, 0);
4108
4109         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4110
4111         /* ring 0 - compute and gfx */
4112         /* Set ring buffer size */
4113         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4114         rb_bufsz = order_base_2(ring->ring_size / 8);
4115         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4116 #ifdef __BIG_ENDIAN
4117         tmp |= BUF_SWAP_32BIT;
4118 #endif
4119         WREG32(CP_RB0_CNTL, tmp);
4120
4121         /* Initialize the ring buffer's read and write pointers */
4122         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4123         ring->wptr = 0;
4124         WREG32(CP_RB0_WPTR, ring->wptr);
4125
4126         /* set the wb address wether it's enabled or not */
4127         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4128         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4129
4130         /* scratch register shadowing is no longer supported */
4131         WREG32(SCRATCH_UMSK, 0);
4132
4133         if (!rdev->wb.enabled)
4134                 tmp |= RB_NO_UPDATE;
4135
4136         mdelay(1);
4137         WREG32(CP_RB0_CNTL, tmp);
4138
4139         rb_addr = ring->gpu_addr >> 8;
4140         WREG32(CP_RB0_BASE, rb_addr);
4141         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4142
4143         /* start the ring */
4144         cik_cp_gfx_start(rdev);
4145         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4146         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4147         if (r) {
4148                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4149                 return r;
4150         }
4151
4152         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4153                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4154
4155         return 0;
4156 }
4157
4158 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4159                      struct radeon_ring *ring)
4160 {
4161         u32 rptr;
4162
4163         if (rdev->wb.enabled)
4164                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4165         else
4166                 rptr = RREG32(CP_RB0_RPTR);
4167
4168         return rptr;
4169 }
4170
4171 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4172                      struct radeon_ring *ring)
4173 {
4174         u32 wptr;
4175
4176         wptr = RREG32(CP_RB0_WPTR);
4177
4178         return wptr;
4179 }
4180
4181 void cik_gfx_set_wptr(struct radeon_device *rdev,
4182                       struct radeon_ring *ring)
4183 {
4184         WREG32(CP_RB0_WPTR, ring->wptr);
4185         (void)RREG32(CP_RB0_WPTR);
4186 }
4187
4188 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4189                          struct radeon_ring *ring)
4190 {
4191         u32 rptr;
4192
4193         if (rdev->wb.enabled) {
4194                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4195         } else {
4196                 spin_lock(&rdev->srbm_mutex);
4197                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4198                 rptr = RREG32(CP_HQD_PQ_RPTR);
4199                 cik_srbm_select(rdev, 0, 0, 0, 0);
4200                 spin_unlock(&rdev->srbm_mutex);
4201         }
4202
4203         return rptr;
4204 }
4205
4206 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4207                          struct radeon_ring *ring)
4208 {
4209         u32 wptr;
4210
4211         if (rdev->wb.enabled) {
4212                 /* XXX check if swapping is necessary on BE */
4213                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4214         } else {
4215                 spin_lock(&rdev->srbm_mutex);
4216                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4217                 wptr = RREG32(CP_HQD_PQ_WPTR);
4218                 cik_srbm_select(rdev, 0, 0, 0, 0);
4219                 spin_unlock(&rdev->srbm_mutex);
4220         }
4221
4222         return wptr;
4223 }
4224
4225 void cik_compute_set_wptr(struct radeon_device *rdev,
4226                           struct radeon_ring *ring)
4227 {
4228         /* XXX check if swapping is necessary on BE */
4229         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4230         WDOORBELL32(ring->doorbell_index, ring->wptr);
4231 }
4232
4233 /**
4234  * cik_cp_compute_enable - enable/disable the compute CP MEs
4235  *
4236  * @rdev: radeon_device pointer
4237  * @enable: enable or disable the MEs
4238  *
4239  * Halts or unhalts the compute MEs.
4240  */
4241 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4242 {
4243         if (enable)
4244                 WREG32(CP_MEC_CNTL, 0);
4245         else {
4246                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4247                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4248                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4249         }
4250         udelay(50);
4251 }
4252
4253 /**
4254  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4255  *
4256  * @rdev: radeon_device pointer
4257  *
4258  * Loads the compute MEC1&2 ucode.
4259  * Returns 0 for success, -EINVAL if the ucode is not available.
4260  */
4261 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4262 {
4263         const __be32 *fw_data;
4264         int i;
4265
4266         if (!rdev->mec_fw)
4267                 return -EINVAL;
4268
4269         cik_cp_compute_enable(rdev, false);
4270
4271         /* MEC1 */
4272         fw_data = (const __be32 *)rdev->mec_fw->data;
4273         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4274         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4275                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4276         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4277
4278         if (rdev->family == CHIP_KAVERI) {
4279                 /* MEC2 */
4280                 fw_data = (const __be32 *)rdev->mec_fw->data;
4281                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4282                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4283                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4284                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4285         }
4286
4287         return 0;
4288 }
4289
4290 /**
4291  * cik_cp_compute_start - start the compute queues
4292  *
4293  * @rdev: radeon_device pointer
4294  *
4295  * Enable the compute queues.
4296  * Returns 0 for success, error for failure.
4297  */
4298 static int cik_cp_compute_start(struct radeon_device *rdev)
4299 {
4300         cik_cp_compute_enable(rdev, true);
4301
4302         return 0;
4303 }
4304
4305 /**
4306  * cik_cp_compute_fini - stop the compute queues
4307  *
4308  * @rdev: radeon_device pointer
4309  *
4310  * Stop the compute queues and tear down the driver queue
4311  * info.
4312  */
4313 static void cik_cp_compute_fini(struct radeon_device *rdev)
4314 {
4315         int i, idx, r;
4316
4317         cik_cp_compute_enable(rdev, false);
4318
4319         for (i = 0; i < 2; i++) {
4320                 if (i == 0)
4321                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4322                 else
4323                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4324
4325                 if (rdev->ring[idx].mqd_obj) {
4326                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4327                         if (unlikely(r != 0))
4328                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4329
4330                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4331                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4332
4333                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4334                         rdev->ring[idx].mqd_obj = NULL;
4335                 }
4336         }
4337 }
4338
4339 static void cik_mec_fini(struct radeon_device *rdev)
4340 {
4341         int r;
4342
4343         if (rdev->mec.hpd_eop_obj) {
4344                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4345                 if (unlikely(r != 0))
4346                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4347                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4348                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4349
4350                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4351                 rdev->mec.hpd_eop_obj = NULL;
4352         }
4353 }
4354
4355 #define MEC_HPD_SIZE 2048
4356
4357 static int cik_mec_init(struct radeon_device *rdev)
4358 {
4359         int r;
4360         u32 *hpd;
4361
4362         /*
4363          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4364          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4365          */
4366         if (rdev->family == CHIP_KAVERI)
4367                 rdev->mec.num_mec = 2;
4368         else
4369                 rdev->mec.num_mec = 1;
4370         rdev->mec.num_pipe = 4;
4371         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4372
4373         if (rdev->mec.hpd_eop_obj == NULL) {
4374                 r = radeon_bo_create(rdev,
4375                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4376                                      PAGE_SIZE, true,
4377                                      RADEON_GEM_DOMAIN_GTT, 0, NULL,
4378                                      &rdev->mec.hpd_eop_obj);
4379                 if (r) {
4380                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4381                         return r;
4382                 }
4383         }
4384
4385         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4386         if (unlikely(r != 0)) {
4387                 cik_mec_fini(rdev);
4388                 return r;
4389         }
4390         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4391                           &rdev->mec.hpd_eop_gpu_addr);
4392         if (r) {
4393                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4394                 cik_mec_fini(rdev);
4395                 return r;
4396         }
4397         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4398         if (r) {
4399                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4400                 cik_mec_fini(rdev);
4401                 return r;
4402         }
4403
4404         /* clear memory.  Not sure if this is required or not */
4405         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4406
4407         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4408         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4409
4410         return 0;
4411 }
4412
4413 struct hqd_registers
4414 {
4415         u32 cp_mqd_base_addr;
4416         u32 cp_mqd_base_addr_hi;
4417         u32 cp_hqd_active;
4418         u32 cp_hqd_vmid;
4419         u32 cp_hqd_persistent_state;
4420         u32 cp_hqd_pipe_priority;
4421         u32 cp_hqd_queue_priority;
4422         u32 cp_hqd_quantum;
4423         u32 cp_hqd_pq_base;
4424         u32 cp_hqd_pq_base_hi;
4425         u32 cp_hqd_pq_rptr;
4426         u32 cp_hqd_pq_rptr_report_addr;
4427         u32 cp_hqd_pq_rptr_report_addr_hi;
4428         u32 cp_hqd_pq_wptr_poll_addr;
4429         u32 cp_hqd_pq_wptr_poll_addr_hi;
4430         u32 cp_hqd_pq_doorbell_control;
4431         u32 cp_hqd_pq_wptr;
4432         u32 cp_hqd_pq_control;
4433         u32 cp_hqd_ib_base_addr;
4434         u32 cp_hqd_ib_base_addr_hi;
4435         u32 cp_hqd_ib_rptr;
4436         u32 cp_hqd_ib_control;
4437         u32 cp_hqd_iq_timer;
4438         u32 cp_hqd_iq_rptr;
4439         u32 cp_hqd_dequeue_request;
4440         u32 cp_hqd_dma_offload;
4441         u32 cp_hqd_sema_cmd;
4442         u32 cp_hqd_msg_type;
4443         u32 cp_hqd_atomic0_preop_lo;
4444         u32 cp_hqd_atomic0_preop_hi;
4445         u32 cp_hqd_atomic1_preop_lo;
4446         u32 cp_hqd_atomic1_preop_hi;
4447         u32 cp_hqd_hq_scheduler0;
4448         u32 cp_hqd_hq_scheduler1;
4449         u32 cp_mqd_control;
4450 };
4451
4452 struct bonaire_mqd
4453 {
4454         u32 header;
4455         u32 dispatch_initiator;
4456         u32 dimensions[3];
4457         u32 start_idx[3];
4458         u32 num_threads[3];
4459         u32 pipeline_stat_enable;
4460         u32 perf_counter_enable;
4461         u32 pgm[2];
4462         u32 tba[2];
4463         u32 tma[2];
4464         u32 pgm_rsrc[2];
4465         u32 vmid;
4466         u32 resource_limits;
4467         u32 static_thread_mgmt01[2];
4468         u32 tmp_ring_size;
4469         u32 static_thread_mgmt23[2];
4470         u32 restart[3];
4471         u32 thread_trace_enable;
4472         u32 reserved1;
4473         u32 user_data[16];
4474         u32 vgtcs_invoke_count[2];
4475         struct hqd_registers queue_state;
4476         u32 dequeue_cntr;
4477         u32 interrupt_queue[64];
4478 };
4479
4480 /**
4481  * cik_cp_compute_resume - setup the compute queue registers
4482  *
4483  * @rdev: radeon_device pointer
4484  *
4485  * Program the compute queues and test them to make sure they
4486  * are working.
4487  * Returns 0 for success, error for failure.
4488  */
4489 static int cik_cp_compute_resume(struct radeon_device *rdev)
4490 {
4491         int r, i, j, idx;
4492         u32 tmp;
4493         bool use_doorbell = true;
4494         u64 hqd_gpu_addr;
4495         u64 mqd_gpu_addr;
4496         u64 eop_gpu_addr;
4497         u64 wb_gpu_addr;
4498         u32 *buf;
4499         struct bonaire_mqd *mqd;
4500
4501         r = cik_cp_compute_start(rdev);
4502         if (r)
4503                 return r;
4504
4505         /* fix up chicken bits */
4506         tmp = RREG32(CP_CPF_DEBUG);
4507         tmp |= (1 << 23);
4508         WREG32(CP_CPF_DEBUG, tmp);
4509
4510         /* init the pipes */
4511         spin_lock(&rdev->srbm_mutex);
4512         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4513                 int me = (i < 4) ? 1 : 2;
4514                 int pipe = (i < 4) ? i : (i - 4);
4515
4516                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4517
4518                 cik_srbm_select(rdev, me, pipe, 0, 0);
4519
4520                 /* write the EOP addr */
4521                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4522                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4523
4524                 /* set the VMID assigned */
4525                 WREG32(CP_HPD_EOP_VMID, 0);
4526
4527                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4528                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4529                 tmp &= ~EOP_SIZE_MASK;
4530                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4531                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4532         }
4533         cik_srbm_select(rdev, 0, 0, 0, 0);
4534         spin_unlock(&rdev->srbm_mutex);
4535
4536         /* init the queues.  Just two for now. */
4537         for (i = 0; i < 2; i++) {
4538                 if (i == 0)
4539                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4540                 else
4541                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4542
4543                 if (rdev->ring[idx].mqd_obj == NULL) {
4544                         r = radeon_bo_create(rdev,
4545                                              sizeof(struct bonaire_mqd),
4546                                              PAGE_SIZE, true,
4547                                              RADEON_GEM_DOMAIN_GTT, 0, NULL,
4548                                              &rdev->ring[idx].mqd_obj);
4549                         if (r) {
4550                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4551                                 return r;
4552                         }
4553                 }
4554
4555                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4556                 if (unlikely(r != 0)) {
4557                         cik_cp_compute_fini(rdev);
4558                         return r;
4559                 }
4560                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4561                                   &mqd_gpu_addr);
4562                 if (r) {
4563                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4564                         cik_cp_compute_fini(rdev);
4565                         return r;
4566                 }
4567                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4568                 if (r) {
4569                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4570                         cik_cp_compute_fini(rdev);
4571                         return r;
4572                 }
4573
4574                 /* init the mqd struct */
4575                 memset(buf, 0, sizeof(struct bonaire_mqd));
4576
4577                 mqd = (struct bonaire_mqd *)buf;
4578                 mqd->header = 0xC0310800;
4579                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4580                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4581                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4582                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4583
4584                 spin_lock(&rdev->srbm_mutex);
4585                 cik_srbm_select(rdev, rdev->ring[idx].me,
4586                                 rdev->ring[idx].pipe,
4587                                 rdev->ring[idx].queue, 0);
4588
4589                 /* disable wptr polling */
4590                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4591                 tmp &= ~WPTR_POLL_EN;
4592                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4593
4594                 /* enable doorbell? */
4595                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4596                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4597                 if (use_doorbell)
4598                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4599                 else
4600                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4601                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4602                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4603
4604                 /* disable the queue if it's active */
4605                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4606                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4607                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4608                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4609                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4610                         for (j = 0; j < rdev->usec_timeout; j++) {
4611                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4612                                         break;
4613                                 udelay(1);
4614                         }
4615                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4616                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4617                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4618                 }
4619
4620                 /* set the pointer to the MQD */
4621                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4622                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4623                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4624                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4625                 /* set MQD vmid to 0 */
4626                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4627                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4628                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4629
4630                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4631                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4632                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4633                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4634                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4635                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4636
4637                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4638                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4639                 mqd->queue_state.cp_hqd_pq_control &=
4640                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4641
4642                 mqd->queue_state.cp_hqd_pq_control |=
4643                         order_base_2(rdev->ring[idx].ring_size / 8);
4644                 mqd->queue_state.cp_hqd_pq_control |=
4645                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4646 #ifdef __BIG_ENDIAN
4647                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4648 #endif
4649                 mqd->queue_state.cp_hqd_pq_control &=
4650                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4651                 mqd->queue_state.cp_hqd_pq_control |=
4652                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4653                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4654
4655                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4656                 if (i == 0)
4657                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4658                 else
4659                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4660                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4661                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4662                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4663                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4664                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4665
4666                 /* set the wb address wether it's enabled or not */
4667                 if (i == 0)
4668                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4669                 else
4670                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4671                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4672                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4673                         upper_32_bits(wb_gpu_addr) & 0xffff;
4674                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4675                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4676                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4677                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4678
4679                 /* enable the doorbell if requested */
4680                 if (use_doorbell) {
4681                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4682                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4683                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4684                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4685                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4686                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4687                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4688                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4689
4690                 } else {
4691                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4692                 }
4693                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4694                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4695
4696                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4697                 rdev->ring[idx].wptr = 0;
4698                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4699                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4700                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4701
4702                 /* set the vmid for the queue */
4703                 mqd->queue_state.cp_hqd_vmid = 0;
4704                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4705
4706                 /* activate the queue */
4707                 mqd->queue_state.cp_hqd_active = 1;
4708                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4709
4710                 cik_srbm_select(rdev, 0, 0, 0, 0);
4711                 spin_unlock(&rdev->srbm_mutex);
4712
4713                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4714                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4715
4716                 rdev->ring[idx].ready = true;
4717                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4718                 if (r)
4719                         rdev->ring[idx].ready = false;
4720         }
4721
4722         return 0;
4723 }
4724
4725 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4726 {
4727         cik_cp_gfx_enable(rdev, enable);
4728         cik_cp_compute_enable(rdev, enable);
4729 }
4730
4731 static int cik_cp_load_microcode(struct radeon_device *rdev)
4732 {
4733         int r;
4734
4735         r = cik_cp_gfx_load_microcode(rdev);
4736         if (r)
4737                 return r;
4738         r = cik_cp_compute_load_microcode(rdev);
4739         if (r)
4740                 return r;
4741
4742         return 0;
4743 }
4744
4745 static void cik_cp_fini(struct radeon_device *rdev)
4746 {
4747         cik_cp_gfx_fini(rdev);
4748         cik_cp_compute_fini(rdev);
4749 }
4750
4751 static int cik_cp_resume(struct radeon_device *rdev)
4752 {
4753         int r;
4754
4755         cik_enable_gui_idle_interrupt(rdev, false);
4756
4757         r = cik_cp_load_microcode(rdev);
4758         if (r)
4759                 return r;
4760
4761         r = cik_cp_gfx_resume(rdev);
4762         if (r)
4763                 return r;
4764         r = cik_cp_compute_resume(rdev);
4765         if (r)
4766                 return r;
4767
4768         cik_enable_gui_idle_interrupt(rdev, true);
4769
4770         return 0;
4771 }
4772
4773 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4774 {
4775         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4776                 RREG32(GRBM_STATUS));
4777         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4778                 RREG32(GRBM_STATUS2));
4779         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4780                 RREG32(GRBM_STATUS_SE0));
4781         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4782                 RREG32(GRBM_STATUS_SE1));
4783         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4784                 RREG32(GRBM_STATUS_SE2));
4785         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4786                 RREG32(GRBM_STATUS_SE3));
4787         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4788                 RREG32(SRBM_STATUS));
4789         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4790                 RREG32(SRBM_STATUS2));
4791         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4792                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4793         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4794                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4795         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4796         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4797                  RREG32(CP_STALLED_STAT1));
4798         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4799                  RREG32(CP_STALLED_STAT2));
4800         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4801                  RREG32(CP_STALLED_STAT3));
4802         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4803                  RREG32(CP_CPF_BUSY_STAT));
4804         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4805                  RREG32(CP_CPF_STALLED_STAT1));
4806         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4807         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4808         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4809                  RREG32(CP_CPC_STALLED_STAT1));
4810         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4811 }
4812
4813 /**
4814  * cik_gpu_check_soft_reset - check which blocks are busy
4815  *
4816  * @rdev: radeon_device pointer
4817  *
4818  * Check which blocks are busy and return the relevant reset
4819  * mask to be used by cik_gpu_soft_reset().
4820  * Returns a mask of the blocks to be reset.
4821  */
4822 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4823 {
4824         u32 reset_mask = 0;
4825         u32 tmp;
4826
4827         /* GRBM_STATUS */
4828         tmp = RREG32(GRBM_STATUS);
4829         if (tmp & (PA_BUSY | SC_BUSY |
4830                    BCI_BUSY | SX_BUSY |
4831                    TA_BUSY | VGT_BUSY |
4832                    DB_BUSY | CB_BUSY |
4833                    GDS_BUSY | SPI_BUSY |
4834                    IA_BUSY | IA_BUSY_NO_DMA))
4835                 reset_mask |= RADEON_RESET_GFX;
4836
4837         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4838                 reset_mask |= RADEON_RESET_CP;
4839
4840         /* GRBM_STATUS2 */
4841         tmp = RREG32(GRBM_STATUS2);
4842         if (tmp & RLC_BUSY)
4843                 reset_mask |= RADEON_RESET_RLC;
4844
4845         /* SDMA0_STATUS_REG */
4846         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4847         if (!(tmp & SDMA_IDLE))
4848                 reset_mask |= RADEON_RESET_DMA;
4849
4850         /* SDMA1_STATUS_REG */
4851         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4852         if (!(tmp & SDMA_IDLE))
4853                 reset_mask |= RADEON_RESET_DMA1;
4854
4855         /* SRBM_STATUS2 */
4856         tmp = RREG32(SRBM_STATUS2);
4857         if (tmp & SDMA_BUSY)
4858                 reset_mask |= RADEON_RESET_DMA;
4859
4860         if (tmp & SDMA1_BUSY)
4861                 reset_mask |= RADEON_RESET_DMA1;
4862
4863         /* SRBM_STATUS */
4864         tmp = RREG32(SRBM_STATUS);
4865
4866         if (tmp & IH_BUSY)
4867                 reset_mask |= RADEON_RESET_IH;
4868
4869         if (tmp & SEM_BUSY)
4870                 reset_mask |= RADEON_RESET_SEM;
4871
4872         if (tmp & GRBM_RQ_PENDING)
4873                 reset_mask |= RADEON_RESET_GRBM;
4874
4875         if (tmp & VMC_BUSY)
4876                 reset_mask |= RADEON_RESET_VMC;
4877
4878         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4879                    MCC_BUSY | MCD_BUSY))
4880                 reset_mask |= RADEON_RESET_MC;
4881
4882         if (evergreen_is_display_hung(rdev))
4883                 reset_mask |= RADEON_RESET_DISPLAY;
4884
4885         /* Skip MC reset as it's mostly likely not hung, just busy */
4886         if (reset_mask & RADEON_RESET_MC) {
4887                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4888                 reset_mask &= ~RADEON_RESET_MC;
4889         }
4890
4891         return reset_mask;
4892 }
4893
4894 /**
4895  * cik_gpu_soft_reset - soft reset GPU
4896  *
4897  * @rdev: radeon_device pointer
4898  * @reset_mask: mask of which blocks to reset
4899  *
4900  * Soft reset the blocks specified in @reset_mask.
4901  */
4902 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4903 {
4904         struct evergreen_mc_save save;
4905         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4906         u32 tmp;
4907
4908         if (reset_mask == 0)
4909                 return;
4910
4911         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4912
4913         cik_print_gpu_status_regs(rdev);
4914         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4915                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4916         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4917                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4918
4919         /* disable CG/PG */
4920         cik_fini_pg(rdev);
4921         cik_fini_cg(rdev);
4922
4923         /* stop the rlc */
4924         cik_rlc_stop(rdev);
4925
4926         /* Disable GFX parsing/prefetching */
4927         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4928
4929         /* Disable MEC parsing/prefetching */
4930         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4931
4932         if (reset_mask & RADEON_RESET_DMA) {
4933                 /* sdma0 */
4934                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4935                 tmp |= SDMA_HALT;
4936                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4937         }
4938         if (reset_mask & RADEON_RESET_DMA1) {
4939                 /* sdma1 */
4940                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4941                 tmp |= SDMA_HALT;
4942                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4943         }
4944
4945         evergreen_mc_stop(rdev, &save);
4946         if (evergreen_mc_wait_for_idle(rdev)) {
4947                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4948         }
4949
4950         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4951                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4952
4953         if (reset_mask & RADEON_RESET_CP) {
4954                 grbm_soft_reset |= SOFT_RESET_CP;
4955
4956                 srbm_soft_reset |= SOFT_RESET_GRBM;
4957         }
4958
4959         if (reset_mask & RADEON_RESET_DMA)
4960                 srbm_soft_reset |= SOFT_RESET_SDMA;
4961
4962         if (reset_mask & RADEON_RESET_DMA1)
4963                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4964
4965         if (reset_mask & RADEON_RESET_DISPLAY)
4966                 srbm_soft_reset |= SOFT_RESET_DC;
4967
4968         if (reset_mask & RADEON_RESET_RLC)
4969                 grbm_soft_reset |= SOFT_RESET_RLC;
4970
4971         if (reset_mask & RADEON_RESET_SEM)
4972                 srbm_soft_reset |= SOFT_RESET_SEM;
4973
4974         if (reset_mask & RADEON_RESET_IH)
4975                 srbm_soft_reset |= SOFT_RESET_IH;
4976
4977         if (reset_mask & RADEON_RESET_GRBM)
4978                 srbm_soft_reset |= SOFT_RESET_GRBM;
4979
4980         if (reset_mask & RADEON_RESET_VMC)
4981                 srbm_soft_reset |= SOFT_RESET_VMC;
4982
4983         if (!(rdev->flags & RADEON_IS_IGP)) {
4984                 if (reset_mask & RADEON_RESET_MC)
4985                         srbm_soft_reset |= SOFT_RESET_MC;
4986         }
4987
4988         if (grbm_soft_reset) {
4989                 tmp = RREG32(GRBM_SOFT_RESET);
4990                 tmp |= grbm_soft_reset;
4991                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4992                 WREG32(GRBM_SOFT_RESET, tmp);
4993                 tmp = RREG32(GRBM_SOFT_RESET);
4994
4995                 udelay(50);
4996
4997                 tmp &= ~grbm_soft_reset;
4998                 WREG32(GRBM_SOFT_RESET, tmp);
4999                 tmp = RREG32(GRBM_SOFT_RESET);
5000         }
5001
5002         if (srbm_soft_reset) {
5003                 tmp = RREG32(SRBM_SOFT_RESET);
5004                 tmp |= srbm_soft_reset;
5005                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5006                 WREG32(SRBM_SOFT_RESET, tmp);
5007                 tmp = RREG32(SRBM_SOFT_RESET);
5008
5009                 udelay(50);
5010
5011                 tmp &= ~srbm_soft_reset;
5012                 WREG32(SRBM_SOFT_RESET, tmp);
5013                 tmp = RREG32(SRBM_SOFT_RESET);
5014         }
5015
5016         /* Wait a little for things to settle down */
5017         udelay(50);
5018
5019         evergreen_mc_resume(rdev, &save);
5020         udelay(50);
5021
5022         cik_print_gpu_status_regs(rdev);
5023 }
5024
5025 struct kv_reset_save_regs {
5026         u32 gmcon_reng_execute;
5027         u32 gmcon_misc;
5028         u32 gmcon_misc3;
5029 };
5030
5031 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5032                                    struct kv_reset_save_regs *save)
5033 {
5034         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5035         save->gmcon_misc = RREG32(GMCON_MISC);
5036         save->gmcon_misc3 = RREG32(GMCON_MISC3);
5037
5038         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5039         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5040                                                 STCTRL_STUTTER_EN));
5041 }
5042
5043 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5044                                       struct kv_reset_save_regs *save)
5045 {
5046         int i;
5047
5048         WREG32(GMCON_PGFSM_WRITE, 0);
5049         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5050
5051         for (i = 0; i < 5; i++)
5052                 WREG32(GMCON_PGFSM_WRITE, 0);
5053
5054         WREG32(GMCON_PGFSM_WRITE, 0);
5055         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5056
5057         for (i = 0; i < 5; i++)
5058                 WREG32(GMCON_PGFSM_WRITE, 0);
5059
5060         WREG32(GMCON_PGFSM_WRITE, 0x210000);
5061         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5062
5063         for (i = 0; i < 5; i++)
5064                 WREG32(GMCON_PGFSM_WRITE, 0);
5065
5066         WREG32(GMCON_PGFSM_WRITE, 0x21003);
5067         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5068
5069         for (i = 0; i < 5; i++)
5070                 WREG32(GMCON_PGFSM_WRITE, 0);
5071
5072         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5073         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5074
5075         for (i = 0; i < 5; i++)
5076                 WREG32(GMCON_PGFSM_WRITE, 0);
5077
5078         WREG32(GMCON_PGFSM_WRITE, 0);
5079         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5080
5081         for (i = 0; i < 5; i++)
5082                 WREG32(GMCON_PGFSM_WRITE, 0);
5083
5084         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5085         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5086
5087         for (i = 0; i < 5; i++)
5088                 WREG32(GMCON_PGFSM_WRITE, 0);
5089
5090         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5091         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5092
5093         for (i = 0; i < 5; i++)
5094                 WREG32(GMCON_PGFSM_WRITE, 0);
5095
5096         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5097         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5098
5099         for (i = 0; i < 5; i++)
5100                 WREG32(GMCON_PGFSM_WRITE, 0);
5101
5102         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5103         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5104
5105         for (i = 0; i < 5; i++)
5106                 WREG32(GMCON_PGFSM_WRITE, 0);
5107
5108         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5109         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5110
5111         WREG32(GMCON_MISC3, save->gmcon_misc3);
5112         WREG32(GMCON_MISC, save->gmcon_misc);
5113         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5114 }
5115
5116 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5117 {
5118         struct evergreen_mc_save save;
5119         struct kv_reset_save_regs kv_save = { 0 };
5120         u32 tmp, i;
5121
5122         dev_info(rdev->dev, "GPU pci config reset\n");
5123
5124         /* disable dpm? */
5125
5126         /* disable cg/pg */
5127         cik_fini_pg(rdev);
5128         cik_fini_cg(rdev);
5129
5130         /* Disable GFX parsing/prefetching */
5131         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5132
5133         /* Disable MEC parsing/prefetching */
5134         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5135
5136         /* sdma0 */
5137         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5138         tmp |= SDMA_HALT;
5139         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5140         /* sdma1 */
5141         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5142         tmp |= SDMA_HALT;
5143         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5144         /* XXX other engines? */
5145
5146         /* halt the rlc, disable cp internal ints */
5147         cik_rlc_stop(rdev);
5148
5149         udelay(50);
5150
5151         /* disable mem access */
5152         evergreen_mc_stop(rdev, &save);
5153         if (evergreen_mc_wait_for_idle(rdev)) {
5154                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5155         }
5156
5157         if (rdev->flags & RADEON_IS_IGP)
5158                 kv_save_regs_for_reset(rdev, &kv_save);
5159
5160         /* disable BM */
5161         pci_disable_busmaster(rdev->pdev->dev);
5162         /* reset */
5163         radeon_pci_config_reset(rdev);
5164
5165         udelay(100);
5166
5167         /* wait for asic to come out of reset */
5168         for (i = 0; i < rdev->usec_timeout; i++) {
5169                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5170                         break;
5171                 udelay(1);
5172         }
5173
5174         /* does asic init need to be run first??? */
5175         if (rdev->flags & RADEON_IS_IGP)
5176                 kv_restore_regs_for_reset(rdev, &kv_save);
5177 }
5178
5179 /**
5180  * cik_asic_reset - soft reset GPU
5181  *
5182  * @rdev: radeon_device pointer
5183  *
5184  * Look up which blocks are hung and attempt
5185  * to reset them.
5186  * Returns 0 for success.
5187  */
5188 int cik_asic_reset(struct radeon_device *rdev)
5189 {
5190         u32 reset_mask;
5191
5192         reset_mask = cik_gpu_check_soft_reset(rdev);
5193
5194         if (reset_mask)
5195                 r600_set_bios_scratch_engine_hung(rdev, true);
5196
5197         /* try soft reset */
5198         cik_gpu_soft_reset(rdev, reset_mask);
5199
5200         reset_mask = cik_gpu_check_soft_reset(rdev);
5201
5202         /* try pci config reset */
5203         if (reset_mask && radeon_hard_reset)
5204                 cik_gpu_pci_config_reset(rdev);
5205
5206         reset_mask = cik_gpu_check_soft_reset(rdev);
5207
5208         if (!reset_mask)
5209                 r600_set_bios_scratch_engine_hung(rdev, false);
5210
5211         return 0;
5212 }
5213
5214 /**
5215  * cik_gfx_is_lockup - check if the 3D engine is locked up
5216  *
5217  * @rdev: radeon_device pointer
5218  * @ring: radeon_ring structure holding ring information
5219  *
5220  * Check if the 3D engine is locked up (CIK).
5221  * Returns true if the engine is locked, false if not.
5222  */
5223 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5224 {
5225         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5226
5227         if (!(reset_mask & (RADEON_RESET_GFX |
5228                             RADEON_RESET_COMPUTE |
5229                             RADEON_RESET_CP))) {
5230                 radeon_ring_lockup_update(rdev, ring);
5231                 return false;
5232         }
5233         return radeon_ring_test_lockup(rdev, ring);
5234 }
5235
5236 /* MC */
5237 /**
5238  * cik_mc_program - program the GPU memory controller
5239  *
5240  * @rdev: radeon_device pointer
5241  *
5242  * Set the location of vram, gart, and AGP in the GPU's
5243  * physical address space (CIK).
5244  */
5245 static void cik_mc_program(struct radeon_device *rdev)
5246 {
5247         struct evergreen_mc_save save;
5248         u32 tmp;
5249         int i, j;
5250
5251         /* Initialize HDP */
5252         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5253                 WREG32((0x2c14 + j), 0x00000000);
5254                 WREG32((0x2c18 + j), 0x00000000);
5255                 WREG32((0x2c1c + j), 0x00000000);
5256                 WREG32((0x2c20 + j), 0x00000000);
5257                 WREG32((0x2c24 + j), 0x00000000);
5258         }
5259         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5260
5261         evergreen_mc_stop(rdev, &save);
5262         if (radeon_mc_wait_for_idle(rdev)) {
5263                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5264         }
5265         /* Lockout access through VGA aperture*/
5266         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5267         /* Update configuration */
5268         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5269                rdev->mc.vram_start >> 12);
5270         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5271                rdev->mc.vram_end >> 12);
5272         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5273                rdev->vram_scratch.gpu_addr >> 12);
5274         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5275         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5276         WREG32(MC_VM_FB_LOCATION, tmp);
5277         /* XXX double check these! */
5278         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5279         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5280         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5281         WREG32(MC_VM_AGP_BASE, 0);
5282         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5283         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5284         if (radeon_mc_wait_for_idle(rdev)) {
5285                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5286         }
5287         evergreen_mc_resume(rdev, &save);
5288         /* we need to own VRAM, so turn off the VGA renderer here
5289          * to stop it overwriting our objects */
5290         rv515_vga_render_disable(rdev);
5291 }
5292
5293 /**
5294  * cik_mc_init - initialize the memory controller driver params
5295  *
5296  * @rdev: radeon_device pointer
5297  *
5298  * Look up the amount of vram, vram width, and decide how to place
5299  * vram and gart within the GPU's physical address space (CIK).
5300  * Returns 0 for success.
5301  */
5302 static int cik_mc_init(struct radeon_device *rdev)
5303 {
5304         u32 tmp;
5305         int chansize, numchan;
5306
5307         /* Get VRAM informations */
5308         rdev->mc.vram_is_ddr = true;
5309         tmp = RREG32(MC_ARB_RAMCFG);
5310         if (tmp & CHANSIZE_MASK) {
5311                 chansize = 64;
5312         } else {
5313                 chansize = 32;
5314         }
5315         tmp = RREG32(MC_SHARED_CHMAP);
5316         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5317         case 0:
5318         default:
5319                 numchan = 1;
5320                 break;
5321         case 1:
5322                 numchan = 2;
5323                 break;
5324         case 2:
5325                 numchan = 4;
5326                 break;
5327         case 3:
5328                 numchan = 8;
5329                 break;
5330         case 4:
5331                 numchan = 3;
5332                 break;
5333         case 5:
5334                 numchan = 6;
5335                 break;
5336         case 6:
5337                 numchan = 10;
5338                 break;
5339         case 7:
5340                 numchan = 12;
5341                 break;
5342         case 8:
5343                 numchan = 16;
5344                 break;
5345         }
5346         rdev->mc.vram_width = numchan * chansize;
5347         /* Could aper size report 0 ? */
5348         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5349         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5350         /* size in MB on si */
5351         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5352         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5353         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5354         si_vram_gtt_location(rdev, &rdev->mc);
5355         radeon_update_bandwidth_info(rdev);
5356
5357         return 0;
5358 }
5359
5360 /*
5361  * GART
5362  * VMID 0 is the physical GPU addresses as used by the kernel.
5363  * VMIDs 1-15 are used for userspace clients and are handled
5364  * by the radeon vm/hsa code.
5365  */
5366 /**
5367  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5368  *
5369  * @rdev: radeon_device pointer
5370  *
5371  * Flush the TLB for the VMID 0 page table (CIK).
5372  */
5373 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5374 {
5375         /* flush hdp cache */
5376         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5377
5378         /* bits 0-15 are the VM contexts0-15 */
5379         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5380 }
5381
5382 /**
5383  * cik_pcie_gart_enable - gart enable
5384  *
5385  * @rdev: radeon_device pointer
5386  *
5387  * This sets up the TLBs, programs the page tables for VMID0,
5388  * sets up the hw for VMIDs 1-15 which are allocated on
5389  * demand, and sets up the global locations for the LDS, GDS,
5390  * and GPUVM for FSA64 clients (CIK).
5391  * Returns 0 for success, errors for failure.
5392  */
5393 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5394 {
5395         int r, i;
5396
5397         if (rdev->gart.robj == NULL) {
5398                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5399                 return -EINVAL;
5400         }
5401         r = radeon_gart_table_vram_pin(rdev);
5402         if (r)
5403                 return r;
5404         /* Setup TLB control */
5405         WREG32(MC_VM_MX_L1_TLB_CNTL,
5406                (0xA << 7) |
5407                ENABLE_L1_TLB |
5408                ENABLE_L1_FRAGMENT_PROCESSING |
5409                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5410                ENABLE_ADVANCED_DRIVER_MODEL |
5411                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5412         /* Setup L2 cache */
5413         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5414                ENABLE_L2_FRAGMENT_PROCESSING |
5415                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5416                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5417                EFFECTIVE_L2_QUEUE_SIZE(7) |
5418                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5419         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5420         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5421                BANK_SELECT(4) |
5422                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5423         /* setup context0 */
5424         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5425         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5426         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5427         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5428                         (u32)(rdev->dummy_page.addr >> 12));
5429         WREG32(VM_CONTEXT0_CNTL2, 0);
5430         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5431                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5432
5433         WREG32(0x15D4, 0);
5434         WREG32(0x15D8, 0);
5435         WREG32(0x15DC, 0);
5436
5437         /* restore context1-15 */
5438         /* set vm size, must be a multiple of 4 */
5439         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5440         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5441         for (i = 1; i < 16; i++) {
5442                 if (i < 8)
5443                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5444                                rdev->vm_manager.saved_table_addr[i]);
5445                 else
5446                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5447                                rdev->vm_manager.saved_table_addr[i]);
5448         }
5449
5450         /* enable context1-15 */
5451         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5452                (u32)(rdev->dummy_page.addr >> 12));
5453         WREG32(VM_CONTEXT1_CNTL2, 4);
5454         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5455                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5456                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5457                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5458                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5459                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5460                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5461                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5462                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5463                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5464                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5465                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5466                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5467                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5468
5469         if (rdev->family == CHIP_KAVERI) {
5470                 u32 tmp = RREG32(CHUB_CONTROL);
5471                 tmp &= ~BYPASS_VM;
5472                 WREG32(CHUB_CONTROL, tmp);
5473         }
5474
5475         /* XXX SH_MEM regs */
5476         /* where to put LDS, scratch, GPUVM in FSA64 space */
5477         spin_lock(&rdev->srbm_mutex);
5478         for (i = 0; i < 16; i++) {
5479                 cik_srbm_select(rdev, 0, 0, 0, i);
5480                 /* CP and shaders */
5481                 WREG32(SH_MEM_CONFIG, 0);
5482                 WREG32(SH_MEM_APE1_BASE, 1);
5483                 WREG32(SH_MEM_APE1_LIMIT, 0);
5484                 WREG32(SH_MEM_BASES, 0);
5485                 /* SDMA GFX */
5486                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5487                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5488                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5489                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5490                 /* XXX SDMA RLC - todo */
5491         }
5492         cik_srbm_select(rdev, 0, 0, 0, 0);
5493         spin_unlock(&rdev->srbm_mutex);
5494
5495         cik_pcie_gart_tlb_flush(rdev);
5496         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5497                  (unsigned)(rdev->mc.gtt_size >> 20),
5498                  (unsigned long long)rdev->gart.table_addr);
5499         rdev->gart.ready = true;
5500         return 0;
5501 }
5502
5503 /**
5504  * cik_pcie_gart_disable - gart disable
5505  *
5506  * @rdev: radeon_device pointer
5507  *
5508  * This disables all VM page table (CIK).
5509  */
5510 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5511 {
5512         unsigned i;
5513
5514         for (i = 1; i < 16; ++i) {
5515                 uint32_t reg;
5516                 if (i < 8)
5517                         reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5518                 else
5519                         reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5520                 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5521         }
5522
5523         /* Disable all tables */
5524         WREG32(VM_CONTEXT0_CNTL, 0);
5525         WREG32(VM_CONTEXT1_CNTL, 0);
5526         /* Setup TLB control */
5527         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5528                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5529         /* Setup L2 cache */
5530         WREG32(VM_L2_CNTL,
5531                ENABLE_L2_FRAGMENT_PROCESSING |
5532                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5533                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5534                EFFECTIVE_L2_QUEUE_SIZE(7) |
5535                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5536         WREG32(VM_L2_CNTL2, 0);
5537         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5538                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5539         radeon_gart_table_vram_unpin(rdev);
5540 }
5541
5542 /**
5543  * cik_pcie_gart_fini - vm fini callback
5544  *
5545  * @rdev: radeon_device pointer
5546  *
5547  * Tears down the driver GART/VM setup (CIK).
5548  */
5549 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5550 {
5551         cik_pcie_gart_disable(rdev);
5552         radeon_gart_table_vram_free(rdev);
5553         radeon_gart_fini(rdev);
5554 }
5555
5556 /* vm parser */
5557 /**
5558  * cik_ib_parse - vm ib_parse callback
5559  *
5560  * @rdev: radeon_device pointer
5561  * @ib: indirect buffer pointer
5562  *
5563  * CIK uses hw IB checking so this is a nop (CIK).
5564  */
5565 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5566 {
5567         return 0;
5568 }
5569
5570 /*
5571  * vm
5572  * VMID 0 is the physical GPU addresses as used by the kernel.
5573  * VMIDs 1-15 are used for userspace clients and are handled
5574  * by the radeon vm/hsa code.
5575  */
5576 /**
5577  * cik_vm_init - cik vm init callback
5578  *
5579  * @rdev: radeon_device pointer
5580  *
5581  * Inits cik specific vm parameters (number of VMs, base of vram for
5582  * VMIDs 1-15) (CIK).
5583  * Returns 0 for success.
5584  */
5585 int cik_vm_init(struct radeon_device *rdev)
5586 {
5587         /* number of VMs */
5588         rdev->vm_manager.nvm = 16;
5589         /* base offset of vram pages */
5590         if (rdev->flags & RADEON_IS_IGP) {
5591                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5592                 tmp <<= 22;
5593                 rdev->vm_manager.vram_base_offset = tmp;
5594         } else
5595                 rdev->vm_manager.vram_base_offset = 0;
5596
5597         return 0;
5598 }
5599
5600 /**
5601  * cik_vm_fini - cik vm fini callback
5602  *
5603  * @rdev: radeon_device pointer
5604  *
5605  * Tear down any asic specific VM setup (CIK).
5606  */
5607 void cik_vm_fini(struct radeon_device *rdev)
5608 {
5609 }
5610
5611 /**
5612  * cik_vm_decode_fault - print human readable fault info
5613  *
5614  * @rdev: radeon_device pointer
5615  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5616  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5617  *
5618  * Print human readable fault information (CIK).
5619  */
5620 static void cik_vm_decode_fault(struct radeon_device *rdev,
5621                                 u32 status, u32 addr, u32 mc_client)
5622 {
5623         u32 mc_id;
5624         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5625         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5626         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5627                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5628
5629         if (rdev->family == CHIP_HAWAII)
5630                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5631         else
5632                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5633
5634         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5635                protections, vmid, addr,
5636                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5637                block, mc_client, mc_id);
5638 }
5639
5640 /**
5641  * cik_vm_flush - cik vm flush using the CP
5642  *
5643  * @rdev: radeon_device pointer
5644  *
5645  * Update the page table base and flush the VM TLB
5646  * using the CP (CIK).
5647  */
5648 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5649 {
5650         struct radeon_ring *ring = &rdev->ring[ridx];
5651         int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5652
5653         if (vm == NULL)
5654                 return;
5655
5656         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5657         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5658                                  WRITE_DATA_DST_SEL(0)));
5659         if (vm->id < 8) {
5660                 radeon_ring_write(ring,
5661                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5662         } else {
5663                 radeon_ring_write(ring,
5664                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5665         }
5666         radeon_ring_write(ring, 0);
5667         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5668
5669         /* update SH_MEM_* regs */
5670         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5671         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5672                                  WRITE_DATA_DST_SEL(0)));
5673         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5674         radeon_ring_write(ring, 0);
5675         radeon_ring_write(ring, VMID(vm->id));
5676
5677         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5678         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5679                                  WRITE_DATA_DST_SEL(0)));
5680         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5681         radeon_ring_write(ring, 0);
5682
5683         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5684         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5685         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5686         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5687
5688         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5689         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5690                                  WRITE_DATA_DST_SEL(0)));
5691         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5692         radeon_ring_write(ring, 0);
5693         radeon_ring_write(ring, VMID(0));
5694
5695         /* HDP flush */
5696         cik_hdp_flush_cp_ring_emit(rdev, ridx);
5697
5698         /* bits 0-15 are the VM contexts0-15 */
5699         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5700         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5701                                  WRITE_DATA_DST_SEL(0)));
5702         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5703         radeon_ring_write(ring, 0);
5704         radeon_ring_write(ring, 1 << vm->id);
5705
5706         /* compute doesn't have PFP */
5707         if (usepfp) {
5708                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5709                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5710                 radeon_ring_write(ring, 0x0);
5711         }
5712 }
5713
5714 /*
5715  * RLC
5716  * The RLC is a multi-purpose microengine that handles a
5717  * variety of functions, the most important of which is
5718  * the interrupt controller.
5719  */
5720 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5721                                           bool enable)
5722 {
5723         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5724
5725         if (enable)
5726                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5727         else
5728                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5729         WREG32(CP_INT_CNTL_RING0, tmp);
5730 }
5731
5732 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5733 {
5734         u32 tmp;
5735
5736         tmp = RREG32(RLC_LB_CNTL);
5737         if (enable)
5738                 tmp |= LOAD_BALANCE_ENABLE;
5739         else
5740                 tmp &= ~LOAD_BALANCE_ENABLE;
5741         WREG32(RLC_LB_CNTL, tmp);
5742 }
5743
5744 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5745 {
5746         u32 i, j, k;
5747         u32 mask;
5748
5749         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5750                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5751                         cik_select_se_sh(rdev, i, j);
5752                         for (k = 0; k < rdev->usec_timeout; k++) {
5753                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5754                                         break;
5755                                 udelay(1);
5756                         }
5757                 }
5758         }
5759         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5760
5761         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5762         for (k = 0; k < rdev->usec_timeout; k++) {
5763                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5764                         break;
5765                 udelay(1);
5766         }
5767 }
5768
5769 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5770 {
5771         u32 tmp;
5772
5773         tmp = RREG32(RLC_CNTL);
5774         if (tmp != rlc)
5775                 WREG32(RLC_CNTL, rlc);
5776 }
5777
5778 static u32 cik_halt_rlc(struct radeon_device *rdev)
5779 {
5780         u32 data, orig;
5781
5782         orig = data = RREG32(RLC_CNTL);
5783
5784         if (data & RLC_ENABLE) {
5785                 u32 i;
5786
5787                 data &= ~RLC_ENABLE;
5788                 WREG32(RLC_CNTL, data);
5789
5790                 for (i = 0; i < rdev->usec_timeout; i++) {
5791                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5792                                 break;
5793                         udelay(1);
5794                 }
5795
5796                 cik_wait_for_rlc_serdes(rdev);
5797         }
5798
5799         return orig;
5800 }
5801
5802 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5803 {
5804         u32 tmp, i, mask;
5805
5806         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5807         WREG32(RLC_GPR_REG2, tmp);
5808
5809         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5810         for (i = 0; i < rdev->usec_timeout; i++) {
5811                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5812                         break;
5813                 udelay(1);
5814         }
5815
5816         for (i = 0; i < rdev->usec_timeout; i++) {
5817                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5818                         break;
5819                 udelay(1);
5820         }
5821 }
5822
5823 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5824 {
5825         u32 tmp;
5826
5827         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5828         WREG32(RLC_GPR_REG2, tmp);
5829 }
5830
5831 /**
5832  * cik_rlc_stop - stop the RLC ME
5833  *
5834  * @rdev: radeon_device pointer
5835  *
5836  * Halt the RLC ME (MicroEngine) (CIK).
5837  */
5838 static void cik_rlc_stop(struct radeon_device *rdev)
5839 {
5840         WREG32(RLC_CNTL, 0);
5841
5842         cik_enable_gui_idle_interrupt(rdev, false);
5843
5844         cik_wait_for_rlc_serdes(rdev);
5845 }
5846
5847 /**
5848  * cik_rlc_start - start the RLC ME
5849  *
5850  * @rdev: radeon_device pointer
5851  *
5852  * Unhalt the RLC ME (MicroEngine) (CIK).
5853  */
5854 static void cik_rlc_start(struct radeon_device *rdev)
5855 {
5856         WREG32(RLC_CNTL, RLC_ENABLE);
5857
5858         cik_enable_gui_idle_interrupt(rdev, true);
5859
5860         udelay(50);
5861 }
5862
5863 /**
5864  * cik_rlc_resume - setup the RLC hw
5865  *
5866  * @rdev: radeon_device pointer
5867  *
5868  * Initialize the RLC registers, load the ucode,
5869  * and start the RLC (CIK).
5870  * Returns 0 for success, -EINVAL if the ucode is not available.
5871  */
5872 static int cik_rlc_resume(struct radeon_device *rdev)
5873 {
5874         u32 i, size, tmp;
5875         const __be32 *fw_data;
5876
5877         if (!rdev->rlc_fw)
5878                 return -EINVAL;
5879
5880         switch (rdev->family) {
5881         case CHIP_BONAIRE:
5882         case CHIP_HAWAII:
5883         default:
5884                 size = BONAIRE_RLC_UCODE_SIZE;
5885                 break;
5886         case CHIP_KAVERI:
5887                 size = KV_RLC_UCODE_SIZE;
5888                 break;
5889         case CHIP_KABINI:
5890                 size = KB_RLC_UCODE_SIZE;
5891                 break;
5892         case CHIP_MULLINS:
5893                 size = ML_RLC_UCODE_SIZE;
5894                 break;
5895         }
5896
5897         cik_rlc_stop(rdev);
5898
5899         /* disable CG */
5900         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5901         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5902
5903         si_rlc_reset(rdev);
5904
5905         cik_init_pg(rdev);
5906
5907         cik_init_cg(rdev);
5908
5909         WREG32(RLC_LB_CNTR_INIT, 0);
5910         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5911
5912         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5913         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5914         WREG32(RLC_LB_PARAMS, 0x00600408);
5915         WREG32(RLC_LB_CNTL, 0x80000004);
5916
5917         WREG32(RLC_MC_CNTL, 0);
5918         WREG32(RLC_UCODE_CNTL, 0);
5919
5920         fw_data = (const __be32 *)rdev->rlc_fw->data;
5921                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5922         for (i = 0; i < size; i++)
5923                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5924         WREG32(RLC_GPM_UCODE_ADDR, 0);
5925
5926         /* XXX - find out what chips support lbpw */
5927         cik_enable_lbpw(rdev, false);
5928
5929         if (rdev->family == CHIP_BONAIRE)
5930                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5931
5932         cik_rlc_start(rdev);
5933
5934         return 0;
5935 }
5936
5937 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5938 {
5939         u32 data, orig, tmp, tmp2;
5940
5941         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5942
5943         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5944                 cik_enable_gui_idle_interrupt(rdev, true);
5945
5946                 tmp = cik_halt_rlc(rdev);
5947
5948                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5949                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5950                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5951                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5952                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5953
5954                 cik_update_rlc(rdev, tmp);
5955
5956                 data |= CGCG_EN | CGLS_EN;
5957         } else {
5958                 cik_enable_gui_idle_interrupt(rdev, false);
5959
5960                 RREG32(CB_CGTT_SCLK_CTRL);
5961                 RREG32(CB_CGTT_SCLK_CTRL);
5962                 RREG32(CB_CGTT_SCLK_CTRL);
5963                 RREG32(CB_CGTT_SCLK_CTRL);
5964
5965                 data &= ~(CGCG_EN | CGLS_EN);
5966         }
5967
5968         if (orig != data)
5969                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5970
5971 }
5972
5973 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5974 {
5975         u32 data, orig, tmp = 0;
5976
5977         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5978                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5979                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5980                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5981                                 data |= CP_MEM_LS_EN;
5982                                 if (orig != data)
5983                                         WREG32(CP_MEM_SLP_CNTL, data);
5984                         }
5985                 }
5986
5987                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5988                 data &= 0xfffffffd;
5989                 if (orig != data)
5990                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5991
5992                 tmp = cik_halt_rlc(rdev);
5993
5994                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5995                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5996                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5997                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5998                 WREG32(RLC_SERDES_WR_CTRL, data);
5999
6000                 cik_update_rlc(rdev, tmp);
6001
6002                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6003                         orig = data = RREG32(CGTS_SM_CTRL_REG);
6004                         data &= ~SM_MODE_MASK;
6005                         data |= SM_MODE(0x2);
6006                         data |= SM_MODE_ENABLE;
6007                         data &= ~CGTS_OVERRIDE;
6008                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6009                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6010                                 data &= ~CGTS_LS_OVERRIDE;
6011                         data &= ~ON_MONITOR_ADD_MASK;
6012                         data |= ON_MONITOR_ADD_EN;
6013                         data |= ON_MONITOR_ADD(0x96);
6014                         if (orig != data)
6015                                 WREG32(CGTS_SM_CTRL_REG, data);
6016                 }
6017         } else {
6018                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6019                 data |= 0x00000002;
6020                 if (orig != data)
6021                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6022
6023                 data = RREG32(RLC_MEM_SLP_CNTL);
6024                 if (data & RLC_MEM_LS_EN) {
6025                         data &= ~RLC_MEM_LS_EN;
6026                         WREG32(RLC_MEM_SLP_CNTL, data);
6027                 }
6028
6029                 data = RREG32(CP_MEM_SLP_CNTL);
6030                 if (data & CP_MEM_LS_EN) {
6031                         data &= ~CP_MEM_LS_EN;
6032                         WREG32(CP_MEM_SLP_CNTL, data);
6033                 }
6034
6035                 orig = data = RREG32(CGTS_SM_CTRL_REG);
6036                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6037                 if (orig != data)
6038                         WREG32(CGTS_SM_CTRL_REG, data);
6039
6040                 tmp = cik_halt_rlc(rdev);
6041
6042                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6043                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6044                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6045                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6046                 WREG32(RLC_SERDES_WR_CTRL, data);
6047
6048                 cik_update_rlc(rdev, tmp);
6049         }
6050 }
6051
6052 static const u32 mc_cg_registers[] =
6053 {
6054         MC_HUB_MISC_HUB_CG,
6055         MC_HUB_MISC_SIP_CG,
6056         MC_HUB_MISC_VM_CG,
6057         MC_XPB_CLK_GAT,
6058         ATC_MISC_CG,
6059         MC_CITF_MISC_WR_CG,
6060         MC_CITF_MISC_RD_CG,
6061         MC_CITF_MISC_VM_CG,
6062         VM_L2_CG,
6063 };
6064
6065 static void cik_enable_mc_ls(struct radeon_device *rdev,
6066                              bool enable)
6067 {
6068         int i;
6069         u32 orig, data;
6070
6071         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6072                 orig = data = RREG32(mc_cg_registers[i]);
6073                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6074                         data |= MC_LS_ENABLE;
6075                 else
6076                         data &= ~MC_LS_ENABLE;
6077                 if (data != orig)
6078                         WREG32(mc_cg_registers[i], data);
6079         }
6080 }
6081
6082 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6083                                bool enable)
6084 {
6085         int i;
6086         u32 orig, data;
6087
6088         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6089                 orig = data = RREG32(mc_cg_registers[i]);
6090                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6091                         data |= MC_CG_ENABLE;
6092                 else
6093                         data &= ~MC_CG_ENABLE;
6094                 if (data != orig)
6095                         WREG32(mc_cg_registers[i], data);
6096         }
6097 }
6098
6099 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6100                                  bool enable)
6101 {
6102         u32 orig, data;
6103
6104         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6105                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6106                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6107         } else {
6108                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6109                 data |= 0xff000000;
6110                 if (data != orig)
6111                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6112
6113                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6114                 data |= 0xff000000;
6115                 if (data != orig)
6116                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6117         }
6118 }
6119
6120 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6121                                  bool enable)
6122 {
6123         u32 orig, data;
6124
6125         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6126                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6127                 data |= 0x100;
6128                 if (orig != data)
6129                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6130
6131                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6132                 data |= 0x100;
6133                 if (orig != data)
6134                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6135         } else {
6136                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6137                 data &= ~0x100;
6138                 if (orig != data)
6139                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6140
6141                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6142                 data &= ~0x100;
6143                 if (orig != data)
6144                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6145         }
6146 }
6147
6148 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6149                                 bool enable)
6150 {
6151         u32 orig, data;
6152
6153         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6154                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6155                 data = 0xfff;
6156                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6157
6158                 orig = data = RREG32(UVD_CGC_CTRL);
6159                 data |= DCM;
6160                 if (orig != data)
6161                         WREG32(UVD_CGC_CTRL, data);
6162         } else {
6163                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6164                 data &= ~0xfff;
6165                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6166
6167                 orig = data = RREG32(UVD_CGC_CTRL);
6168                 data &= ~DCM;
6169                 if (orig != data)
6170                         WREG32(UVD_CGC_CTRL, data);
6171         }
6172 }
6173
6174 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6175                                bool enable)
6176 {
6177         u32 orig, data;
6178
6179         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6180
6181         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6182                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6183                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6184         else
6185                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6186                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6187
6188         if (orig != data)
6189                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6190 }
6191
6192 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6193                                 bool enable)
6194 {
6195         u32 orig, data;
6196
6197         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6198
6199         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6200                 data &= ~CLOCK_GATING_DIS;
6201         else
6202                 data |= CLOCK_GATING_DIS;
6203
6204         if (orig != data)
6205                 WREG32(HDP_HOST_PATH_CNTL, data);
6206 }
6207
6208 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6209                               bool enable)
6210 {
6211         u32 orig, data;
6212
6213         orig = data = RREG32(HDP_MEM_POWER_LS);
6214
6215         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6216                 data |= HDP_LS_ENABLE;
6217         else
6218                 data &= ~HDP_LS_ENABLE;
6219
6220         if (orig != data)
6221                 WREG32(HDP_MEM_POWER_LS, data);
6222 }
6223
6224 void cik_update_cg(struct radeon_device *rdev,
6225                    u32 block, bool enable)
6226 {
6227
6228         if (block & RADEON_CG_BLOCK_GFX) {
6229                 cik_enable_gui_idle_interrupt(rdev, false);
6230                 /* order matters! */
6231                 if (enable) {
6232                         cik_enable_mgcg(rdev, true);
6233                         cik_enable_cgcg(rdev, true);
6234                 } else {
6235                         cik_enable_cgcg(rdev, false);
6236                         cik_enable_mgcg(rdev, false);
6237                 }
6238                 cik_enable_gui_idle_interrupt(rdev, true);
6239         }
6240
6241         if (block & RADEON_CG_BLOCK_MC) {
6242                 if (!(rdev->flags & RADEON_IS_IGP)) {
6243                         cik_enable_mc_mgcg(rdev, enable);
6244                         cik_enable_mc_ls(rdev, enable);
6245                 }
6246         }
6247
6248         if (block & RADEON_CG_BLOCK_SDMA) {
6249                 cik_enable_sdma_mgcg(rdev, enable);
6250                 cik_enable_sdma_mgls(rdev, enable);
6251         }
6252
6253         if (block & RADEON_CG_BLOCK_BIF) {
6254                 cik_enable_bif_mgls(rdev, enable);
6255         }
6256
6257         if (block & RADEON_CG_BLOCK_UVD) {
6258                 if (rdev->has_uvd)
6259                         cik_enable_uvd_mgcg(rdev, enable);
6260         }
6261
6262         if (block & RADEON_CG_BLOCK_HDP) {
6263                 cik_enable_hdp_mgcg(rdev, enable);
6264                 cik_enable_hdp_ls(rdev, enable);
6265         }
6266
6267         if (block & RADEON_CG_BLOCK_VCE) {
6268                 vce_v2_0_enable_mgcg(rdev, enable);
6269         }
6270 }
6271
6272 static void cik_init_cg(struct radeon_device *rdev)
6273 {
6274
6275         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6276
6277         if (rdev->has_uvd)
6278                 si_init_uvd_internal_cg(rdev);
6279
6280         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6281                              RADEON_CG_BLOCK_SDMA |
6282                              RADEON_CG_BLOCK_BIF |
6283                              RADEON_CG_BLOCK_UVD |
6284                              RADEON_CG_BLOCK_HDP), true);
6285 }
6286
6287 static void cik_fini_cg(struct radeon_device *rdev)
6288 {
6289         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6290                              RADEON_CG_BLOCK_SDMA |
6291                              RADEON_CG_BLOCK_BIF |
6292                              RADEON_CG_BLOCK_UVD |
6293                              RADEON_CG_BLOCK_HDP), false);
6294
6295         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6296 }
6297
6298 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6299                                           bool enable)
6300 {
6301         u32 data, orig;
6302
6303         orig = data = RREG32(RLC_PG_CNTL);
6304         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6305                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6306         else
6307                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6308         if (orig != data)
6309                 WREG32(RLC_PG_CNTL, data);
6310 }
6311
6312 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6313                                           bool enable)
6314 {
6315         u32 data, orig;
6316
6317         orig = data = RREG32(RLC_PG_CNTL);
6318         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6319                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6320         else
6321                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6322         if (orig != data)
6323                 WREG32(RLC_PG_CNTL, data);
6324 }
6325
6326 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6327 {
6328         u32 data, orig;
6329
6330         orig = data = RREG32(RLC_PG_CNTL);
6331         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6332                 data &= ~DISABLE_CP_PG;
6333         else
6334                 data |= DISABLE_CP_PG;
6335         if (orig != data)
6336                 WREG32(RLC_PG_CNTL, data);
6337 }
6338
6339 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6340 {
6341         u32 data, orig;
6342
6343         orig = data = RREG32(RLC_PG_CNTL);
6344         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6345                 data &= ~DISABLE_GDS_PG;
6346         else
6347                 data |= DISABLE_GDS_PG;
6348         if (orig != data)
6349                 WREG32(RLC_PG_CNTL, data);
6350 }
6351
6352 #define CP_ME_TABLE_SIZE    96
6353 #define CP_ME_TABLE_OFFSET  2048
6354 #define CP_MEC_TABLE_OFFSET 4096
6355
6356 void cik_init_cp_pg_table(struct radeon_device *rdev)
6357 {
6358         const __be32 *fw_data;
6359         volatile u32 *dst_ptr;
6360         int me, i, max_me = 4;
6361         u32 bo_offset = 0;
6362         u32 table_offset;
6363
6364         if (rdev->family == CHIP_KAVERI)
6365                 max_me = 5;
6366
6367         if (rdev->rlc.cp_table_ptr == NULL)
6368                 return;
6369
6370         /* write the cp table buffer */
6371         dst_ptr = rdev->rlc.cp_table_ptr;
6372         for (me = 0; me < max_me; me++) {
6373                 if (me == 0) {
6374                         fw_data = (const __be32 *)rdev->ce_fw->data;
6375                         table_offset = CP_ME_TABLE_OFFSET;
6376                 } else if (me == 1) {
6377                         fw_data = (const __be32 *)rdev->pfp_fw->data;
6378                         table_offset = CP_ME_TABLE_OFFSET;
6379                 } else if (me == 2) {
6380                         fw_data = (const __be32 *)rdev->me_fw->data;
6381                         table_offset = CP_ME_TABLE_OFFSET;
6382                 } else {
6383                         fw_data = (const __be32 *)rdev->mec_fw->data;
6384                         table_offset = CP_MEC_TABLE_OFFSET;
6385                 }
6386
6387                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6388                         dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6389                 }
6390                 bo_offset += CP_ME_TABLE_SIZE;
6391         }
6392 }
6393
6394 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6395                                 bool enable)
6396 {
6397         u32 data, orig;
6398
6399         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6400                 orig = data = RREG32(RLC_PG_CNTL);
6401                 data |= GFX_PG_ENABLE;
6402                 if (orig != data)
6403                         WREG32(RLC_PG_CNTL, data);
6404
6405                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6406                 data |= AUTO_PG_EN;
6407                 if (orig != data)
6408                         WREG32(RLC_AUTO_PG_CTRL, data);
6409         } else {
6410                 orig = data = RREG32(RLC_PG_CNTL);
6411                 data &= ~GFX_PG_ENABLE;
6412                 if (orig != data)
6413                         WREG32(RLC_PG_CNTL, data);
6414
6415                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6416                 data &= ~AUTO_PG_EN;
6417                 if (orig != data)
6418                         WREG32(RLC_AUTO_PG_CTRL, data);
6419
6420                 data = RREG32(DB_RENDER_CONTROL);
6421         }
6422 }
6423
6424 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6425 {
6426         u32 mask = 0, tmp, tmp1;
6427         int i;
6428
6429         cik_select_se_sh(rdev, se, sh);
6430         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6431         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6432         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6433
6434         tmp &= 0xffff0000;
6435
6436         tmp |= tmp1;
6437         tmp >>= 16;
6438
6439         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6440                 mask <<= 1;
6441                 mask |= 1;
6442         }
6443
6444         return (~tmp) & mask;
6445 }
6446
6447 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6448 {
6449         u32 i, j, k, active_cu_number = 0;
6450         u32 mask, counter, cu_bitmap;
6451         u32 tmp = 0;
6452
6453         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6454                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6455                         mask = 1;
6456                         cu_bitmap = 0;
6457                         counter = 0;
6458                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6459                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6460                                         if (counter < 2)
6461                                                 cu_bitmap |= mask;
6462                                         counter ++;
6463                                 }
6464                                 mask <<= 1;
6465                         }
6466
6467                         active_cu_number += counter;
6468                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6469                 }
6470         }
6471
6472         WREG32(RLC_PG_AO_CU_MASK, tmp);
6473
6474         tmp = RREG32(RLC_MAX_PG_CU);
6475         tmp &= ~MAX_PU_CU_MASK;
6476         tmp |= MAX_PU_CU(active_cu_number);
6477         WREG32(RLC_MAX_PG_CU, tmp);
6478 }
6479
6480 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6481                                        bool enable)
6482 {
6483         u32 data, orig;
6484
6485         orig = data = RREG32(RLC_PG_CNTL);
6486         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6487                 data |= STATIC_PER_CU_PG_ENABLE;
6488         else
6489                 data &= ~STATIC_PER_CU_PG_ENABLE;
6490         if (orig != data)
6491                 WREG32(RLC_PG_CNTL, data);
6492 }
6493
6494 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6495                                         bool enable)
6496 {
6497         u32 data, orig;
6498
6499         orig = data = RREG32(RLC_PG_CNTL);
6500         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6501                 data |= DYN_PER_CU_PG_ENABLE;
6502         else
6503                 data &= ~DYN_PER_CU_PG_ENABLE;
6504         if (orig != data)
6505                 WREG32(RLC_PG_CNTL, data);
6506 }
6507
6508 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6509 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6510
6511 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6512 {
6513         u32 data, orig;
6514         u32 i;
6515
6516         if (rdev->rlc.cs_data) {
6517                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6518                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6519                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6520                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6521         } else {
6522                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6523                 for (i = 0; i < 3; i++)
6524                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6525         }
6526         if (rdev->rlc.reg_list) {
6527                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6528                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6529                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6530         }
6531
6532         orig = data = RREG32(RLC_PG_CNTL);
6533         data |= GFX_PG_SRC;
6534         if (orig != data)
6535                 WREG32(RLC_PG_CNTL, data);
6536
6537         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6538         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6539
6540         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6541         data &= ~IDLE_POLL_COUNT_MASK;
6542         data |= IDLE_POLL_COUNT(0x60);
6543         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6544
6545         data = 0x10101010;
6546         WREG32(RLC_PG_DELAY, data);
6547
6548         data = RREG32(RLC_PG_DELAY_2);
6549         data &= ~0xff;
6550         data |= 0x3;
6551         WREG32(RLC_PG_DELAY_2, data);
6552
6553         data = RREG32(RLC_AUTO_PG_CTRL);
6554         data &= ~GRBM_REG_SGIT_MASK;
6555         data |= GRBM_REG_SGIT(0x700);
6556         WREG32(RLC_AUTO_PG_CTRL, data);
6557
6558 }
6559
6560 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6561 {
6562         cik_enable_gfx_cgpg(rdev, enable);
6563         cik_enable_gfx_static_mgpg(rdev, enable);
6564         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6565 }
6566
6567 u32 cik_get_csb_size(struct radeon_device *rdev)
6568 {
6569         u32 count = 0;
6570         const struct cs_section_def *sect = NULL;
6571         const struct cs_extent_def *ext = NULL;
6572
6573         if (rdev->rlc.cs_data == NULL)
6574                 return 0;
6575
6576         /* begin clear state */
6577         count += 2;
6578         /* context control state */
6579         count += 3;
6580
6581         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6582                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6583                         if (sect->id == SECT_CONTEXT)
6584                                 count += 2 + ext->reg_count;
6585                         else
6586                                 return 0;
6587                 }
6588         }
6589         /* pa_sc_raster_config/pa_sc_raster_config1 */
6590         count += 4;
6591         /* end clear state */
6592         count += 2;
6593         /* clear state */
6594         count += 2;
6595
6596         return count;
6597 }
6598
6599 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6600 {
6601         u32 count = 0, i;
6602         const struct cs_section_def *sect = NULL;
6603         const struct cs_extent_def *ext = NULL;
6604
6605         if (rdev->rlc.cs_data == NULL)
6606                 return;
6607         if (buffer == NULL)
6608                 return;
6609
6610         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6611         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6612
6613         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6614         buffer[count++] = cpu_to_le32(0x80000000);
6615         buffer[count++] = cpu_to_le32(0x80000000);
6616
6617         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6618                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6619                         if (sect->id == SECT_CONTEXT) {
6620                                 buffer[count++] =
6621                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6622                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6623                                 for (i = 0; i < ext->reg_count; i++)
6624                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6625                         } else {
6626                                 return;
6627                         }
6628                 }
6629         }
6630
6631         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6632         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6633         switch (rdev->family) {
6634         case CHIP_BONAIRE:
6635                 buffer[count++] = cpu_to_le32(0x16000012);
6636                 buffer[count++] = cpu_to_le32(0x00000000);
6637                 break;
6638         case CHIP_KAVERI:
6639                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6640                 buffer[count++] = cpu_to_le32(0x00000000);
6641                 break;
6642         case CHIP_KABINI:
6643         case CHIP_MULLINS:
6644                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6645                 buffer[count++] = cpu_to_le32(0x00000000);
6646                 break;
6647         case CHIP_HAWAII:
6648                 buffer[count++] = cpu_to_le32(0x3a00161a);
6649                 buffer[count++] = cpu_to_le32(0x0000002e);
6650                 break;
6651         default:
6652                 buffer[count++] = cpu_to_le32(0x00000000);
6653                 buffer[count++] = cpu_to_le32(0x00000000);
6654                 break;
6655         }
6656
6657         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6658         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6659
6660         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6661         buffer[count++] = cpu_to_le32(0);
6662 }
6663
6664 static void cik_init_pg(struct radeon_device *rdev)
6665 {
6666         if (rdev->pg_flags) {
6667                 cik_enable_sck_slowdown_on_pu(rdev, true);
6668                 cik_enable_sck_slowdown_on_pd(rdev, true);
6669                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6670                         cik_init_gfx_cgpg(rdev);
6671                         cik_enable_cp_pg(rdev, true);
6672                         cik_enable_gds_pg(rdev, true);
6673                 }
6674                 cik_init_ao_cu_mask(rdev);
6675                 cik_update_gfx_pg(rdev, true);
6676         }
6677 }
6678
6679 static void cik_fini_pg(struct radeon_device *rdev)
6680 {
6681         if (rdev->pg_flags) {
6682                 cik_update_gfx_pg(rdev, false);
6683                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6684                         cik_enable_cp_pg(rdev, false);
6685                         cik_enable_gds_pg(rdev, false);
6686                 }
6687         }
6688 }
6689
6690 /*
6691  * Interrupts
6692  * Starting with r6xx, interrupts are handled via a ring buffer.
6693  * Ring buffers are areas of GPU accessible memory that the GPU
6694  * writes interrupt vectors into and the host reads vectors out of.
6695  * There is a rptr (read pointer) that determines where the
6696  * host is currently reading, and a wptr (write pointer)
6697  * which determines where the GPU has written.  When the
6698  * pointers are equal, the ring is idle.  When the GPU
6699  * writes vectors to the ring buffer, it increments the
6700  * wptr.  When there is an interrupt, the host then starts
6701  * fetching commands and processing them until the pointers are
6702  * equal again at which point it updates the rptr.
6703  */
6704
6705 /**
6706  * cik_enable_interrupts - Enable the interrupt ring buffer
6707  *
6708  * @rdev: radeon_device pointer
6709  *
6710  * Enable the interrupt ring buffer (CIK).
6711  */
6712 static void cik_enable_interrupts(struct radeon_device *rdev)
6713 {
6714         u32 ih_cntl = RREG32(IH_CNTL);
6715         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6716
6717         ih_cntl |= ENABLE_INTR;
6718         ih_rb_cntl |= IH_RB_ENABLE;
6719         WREG32(IH_CNTL, ih_cntl);
6720         WREG32(IH_RB_CNTL, ih_rb_cntl);
6721         rdev->ih.enabled = true;
6722 }
6723
6724 /**
6725  * cik_disable_interrupts - Disable the interrupt ring buffer
6726  *
6727  * @rdev: radeon_device pointer
6728  *
6729  * Disable the interrupt ring buffer (CIK).
6730  */
6731 static void cik_disable_interrupts(struct radeon_device *rdev)
6732 {
6733         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6734         u32 ih_cntl = RREG32(IH_CNTL);
6735
6736         ih_rb_cntl &= ~IH_RB_ENABLE;
6737         ih_cntl &= ~ENABLE_INTR;
6738         WREG32(IH_RB_CNTL, ih_rb_cntl);
6739         WREG32(IH_CNTL, ih_cntl);
6740         /* set rptr, wptr to 0 */
6741         WREG32(IH_RB_RPTR, 0);
6742         WREG32(IH_RB_WPTR, 0);
6743         rdev->ih.enabled = false;
6744         rdev->ih.rptr = 0;
6745 }
6746
6747 /**
6748  * cik_disable_interrupt_state - Disable all interrupt sources
6749  *
6750  * @rdev: radeon_device pointer
6751  *
6752  * Clear all interrupt enable bits used by the driver (CIK).
6753  */
6754 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6755 {
6756         u32 tmp;
6757
6758         /* gfx ring */
6759         tmp = RREG32(CP_INT_CNTL_RING0) &
6760                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6761         WREG32(CP_INT_CNTL_RING0, tmp);
6762         /* sdma */
6763         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6764         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6765         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6766         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6767         /* compute queues */
6768         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6769         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6770         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6771         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6772         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6773         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6774         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6775         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6776         /* grbm */
6777         WREG32(GRBM_INT_CNTL, 0);
6778         /* vline/vblank, etc. */
6779         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6780         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6781         if (rdev->num_crtc >= 4) {
6782                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6783                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6784         }
6785         if (rdev->num_crtc >= 6) {
6786                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6787                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6788         }
6789         /* pflip */
6790         if (rdev->num_crtc >= 2) {
6791                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6792                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6793         }
6794         if (rdev->num_crtc >= 4) {
6795                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6796                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6797         }
6798         if (rdev->num_crtc >= 6) {
6799                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6800                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6801         }
6802
6803         /* dac hotplug */
6804         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6805
6806         /* digital hotplug */
6807         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6808         WREG32(DC_HPD1_INT_CONTROL, tmp);
6809         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6810         WREG32(DC_HPD2_INT_CONTROL, tmp);
6811         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6812         WREG32(DC_HPD3_INT_CONTROL, tmp);
6813         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6814         WREG32(DC_HPD4_INT_CONTROL, tmp);
6815         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6816         WREG32(DC_HPD5_INT_CONTROL, tmp);
6817         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6818         WREG32(DC_HPD6_INT_CONTROL, tmp);
6819
6820 }
6821
6822 /**
6823  * cik_irq_init - init and enable the interrupt ring
6824  *
6825  * @rdev: radeon_device pointer
6826  *
6827  * Allocate a ring buffer for the interrupt controller,
6828  * enable the RLC, disable interrupts, enable the IH
6829  * ring buffer and enable it (CIK).
6830  * Called at device load and reume.
6831  * Returns 0 for success, errors for failure.
6832  */
6833 static int cik_irq_init(struct radeon_device *rdev)
6834 {
6835         int ret = 0;
6836         int rb_bufsz;
6837         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6838
6839         /* allocate ring */
6840         ret = r600_ih_ring_alloc(rdev);
6841         if (ret)
6842                 return ret;
6843
6844         /* disable irqs */
6845         cik_disable_interrupts(rdev);
6846
6847         /* init rlc */
6848         ret = cik_rlc_resume(rdev);
6849         if (ret) {
6850                 r600_ih_ring_fini(rdev);
6851                 return ret;
6852         }
6853
6854         /* setup interrupt control */
6855         /* XXX this should actually be a bus address, not an MC address. same on older asics */
6856         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6857         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6858         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6859          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6860          */
6861         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6862         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6863         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6864         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6865
6866         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6867         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6868
6869         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6870                       IH_WPTR_OVERFLOW_CLEAR |
6871                       (rb_bufsz << 1));
6872
6873         if (rdev->wb.enabled)
6874                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6875
6876         /* set the writeback address whether it's enabled or not */
6877         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6878         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6879
6880         WREG32(IH_RB_CNTL, ih_rb_cntl);
6881
6882         /* set rptr, wptr to 0 */
6883         WREG32(IH_RB_RPTR, 0);
6884         WREG32(IH_RB_WPTR, 0);
6885
6886         /* Default settings for IH_CNTL (disabled at first) */
6887         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6888         /* RPTR_REARM only works if msi's are enabled */
6889         if (rdev->msi_enabled)
6890                 ih_cntl |= RPTR_REARM;
6891         WREG32(IH_CNTL, ih_cntl);
6892
6893         /* force the active interrupt state to all disabled */
6894         cik_disable_interrupt_state(rdev);
6895
6896         pci_enable_busmaster(rdev->pdev->dev);
6897
6898         /* enable irqs */
6899         cik_enable_interrupts(rdev);
6900
6901         return ret;
6902 }
6903
6904 /**
6905  * cik_irq_set - enable/disable interrupt sources
6906  *
6907  * @rdev: radeon_device pointer
6908  *
6909  * Enable interrupt sources on the GPU (vblanks, hpd,
6910  * etc.) (CIK).
6911  * Returns 0 for success, errors for failure.
6912  */
6913 int cik_irq_set(struct radeon_device *rdev)
6914 {
6915         u32 cp_int_cntl;
6916         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6917         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6918         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6919         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6920         u32 grbm_int_cntl = 0;
6921         u32 dma_cntl, dma_cntl1;
6922         u32 thermal_int;
6923
6924         if (!rdev->irq.installed) {
6925                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6926                 return -EINVAL;
6927         }
6928         /* don't enable anything if the ih is disabled */
6929         if (!rdev->ih.enabled) {
6930                 cik_disable_interrupts(rdev);
6931                 /* force the active interrupt state to all disabled */
6932                 cik_disable_interrupt_state(rdev);
6933                 return 0;
6934         }
6935
6936         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6937                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6938         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6939
6940         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6941         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6942         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6943         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6944         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6945         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6946
6947         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6948         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6949
6950         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6951         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6952         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6953         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6954         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6955         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6956         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6957         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6958
6959         if (rdev->flags & RADEON_IS_IGP)
6960                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6961                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6962         else
6963                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6964                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6965
6966         /* enable CP interrupts on all rings */
6967         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6968                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6969                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6970         }
6971         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6972                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6973                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6974                 if (ring->me == 1) {
6975                         switch (ring->pipe) {
6976                         case 0:
6977                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6978                                 break;
6979                         case 1:
6980                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6981                                 break;
6982                         case 2:
6983                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6984                                 break;
6985                         case 3:
6986                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6987                                 break;
6988                         default:
6989                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6990                                 break;
6991                         }
6992                 } else if (ring->me == 2) {
6993                         switch (ring->pipe) {
6994                         case 0:
6995                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6996                                 break;
6997                         case 1:
6998                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6999                                 break;
7000                         case 2:
7001                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7002                                 break;
7003                         case 3:
7004                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7005                                 break;
7006                         default:
7007                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7008                                 break;
7009                         }
7010                 } else {
7011                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7012                 }
7013         }
7014         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7015                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7016                 DRM_DEBUG("si_irq_set: sw int cp2\n");
7017                 if (ring->me == 1) {
7018                         switch (ring->pipe) {
7019                         case 0:
7020                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7021                                 break;
7022                         case 1:
7023                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7024                                 break;
7025                         case 2:
7026                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7027                                 break;
7028                         case 3:
7029                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7030                                 break;
7031                         default:
7032                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7033                                 break;
7034                         }
7035                 } else if (ring->me == 2) {
7036                         switch (ring->pipe) {
7037                         case 0:
7038                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7039                                 break;
7040                         case 1:
7041                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7042                                 break;
7043                         case 2:
7044                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7045                                 break;
7046                         case 3:
7047                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7048                                 break;
7049                         default:
7050                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7051                                 break;
7052                         }
7053                 } else {
7054                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7055                 }
7056         }
7057
7058         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7059                 DRM_DEBUG("cik_irq_set: sw int dma\n");
7060                 dma_cntl |= TRAP_ENABLE;
7061         }
7062
7063         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7064                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7065                 dma_cntl1 |= TRAP_ENABLE;
7066         }
7067
7068         if (rdev->irq.crtc_vblank_int[0] ||
7069             atomic_read(&rdev->irq.pflip[0])) {
7070                 DRM_DEBUG("cik_irq_set: vblank 0\n");
7071                 crtc1 |= VBLANK_INTERRUPT_MASK;
7072         }
7073         if (rdev->irq.crtc_vblank_int[1] ||
7074             atomic_read(&rdev->irq.pflip[1])) {
7075                 DRM_DEBUG("cik_irq_set: vblank 1\n");
7076                 crtc2 |= VBLANK_INTERRUPT_MASK;
7077         }
7078         if (rdev->irq.crtc_vblank_int[2] ||
7079             atomic_read(&rdev->irq.pflip[2])) {
7080                 DRM_DEBUG("cik_irq_set: vblank 2\n");
7081                 crtc3 |= VBLANK_INTERRUPT_MASK;
7082         }
7083         if (rdev->irq.crtc_vblank_int[3] ||
7084             atomic_read(&rdev->irq.pflip[3])) {
7085                 DRM_DEBUG("cik_irq_set: vblank 3\n");
7086                 crtc4 |= VBLANK_INTERRUPT_MASK;
7087         }
7088         if (rdev->irq.crtc_vblank_int[4] ||
7089             atomic_read(&rdev->irq.pflip[4])) {
7090                 DRM_DEBUG("cik_irq_set: vblank 4\n");
7091                 crtc5 |= VBLANK_INTERRUPT_MASK;
7092         }
7093         if (rdev->irq.crtc_vblank_int[5] ||
7094             atomic_read(&rdev->irq.pflip[5])) {
7095                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7096                 crtc6 |= VBLANK_INTERRUPT_MASK;
7097         }
7098         if (rdev->irq.hpd[0]) {
7099                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7100                 hpd1 |= DC_HPDx_INT_EN;
7101         }
7102         if (rdev->irq.hpd[1]) {
7103                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7104                 hpd2 |= DC_HPDx_INT_EN;
7105         }
7106         if (rdev->irq.hpd[2]) {
7107                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7108                 hpd3 |= DC_HPDx_INT_EN;
7109         }
7110         if (rdev->irq.hpd[3]) {
7111                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7112                 hpd4 |= DC_HPDx_INT_EN;
7113         }
7114         if (rdev->irq.hpd[4]) {
7115                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7116                 hpd5 |= DC_HPDx_INT_EN;
7117         }
7118         if (rdev->irq.hpd[5]) {
7119                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7120                 hpd6 |= DC_HPDx_INT_EN;
7121         }
7122
7123         if (rdev->irq.dpm_thermal) {
7124                 DRM_DEBUG("dpm thermal\n");
7125                 if (rdev->flags & RADEON_IS_IGP)
7126                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7127                 else
7128                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7129         }
7130
7131         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7132
7133         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7134         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7135
7136         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7137         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7138         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7139         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7140         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7141         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7142         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7143         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7144
7145         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7146
7147         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7148         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7149         if (rdev->num_crtc >= 4) {
7150                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7151                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7152         }
7153         if (rdev->num_crtc >= 6) {
7154                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7155                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7156         }
7157
7158         if (rdev->num_crtc >= 2) {
7159                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7160                        GRPH_PFLIP_INT_MASK);
7161                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7162                        GRPH_PFLIP_INT_MASK);
7163         }
7164         if (rdev->num_crtc >= 4) {
7165                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7166                        GRPH_PFLIP_INT_MASK);
7167                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7168                        GRPH_PFLIP_INT_MASK);
7169         }
7170         if (rdev->num_crtc >= 6) {
7171                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7172                        GRPH_PFLIP_INT_MASK);
7173                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7174                        GRPH_PFLIP_INT_MASK);
7175         }
7176
7177         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7178         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7179         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7180         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7181         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7182         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7183
7184         if (rdev->flags & RADEON_IS_IGP)
7185                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7186         else
7187                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7188
7189         return 0;
7190 }
7191
7192 /**
7193  * cik_irq_ack - ack interrupt sources
7194  *
7195  * @rdev: radeon_device pointer
7196  *
7197  * Ack interrupt sources on the GPU (vblanks, hpd,
7198  * etc.) (CIK).  Certain interrupts sources are sw
7199  * generated and do not require an explicit ack.
7200  */
7201 static inline void cik_irq_ack(struct radeon_device *rdev)
7202 {
7203         u32 tmp;
7204
7205         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7206         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7207         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7208         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7209         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7210         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7211         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7212
7213         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7214                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7215         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7216                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7217         if (rdev->num_crtc >= 4) {
7218                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7219                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7220                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7221                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7222         }
7223         if (rdev->num_crtc >= 6) {
7224                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7225                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7226                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7227                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7228         }
7229
7230         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7231                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7232                        GRPH_PFLIP_INT_CLEAR);
7233         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7234                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7235                        GRPH_PFLIP_INT_CLEAR);
7236         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7237                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7238         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7239                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7240         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7241                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7242         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7243                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7244
7245         if (rdev->num_crtc >= 4) {
7246                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7247                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7248                                GRPH_PFLIP_INT_CLEAR);
7249                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7250                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7251                                GRPH_PFLIP_INT_CLEAR);
7252                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7253                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7254                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7255                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7256                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7257                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7258                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7259                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7260         }
7261
7262         if (rdev->num_crtc >= 6) {
7263                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7264                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7265                                GRPH_PFLIP_INT_CLEAR);
7266                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7267                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7268                                GRPH_PFLIP_INT_CLEAR);
7269                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7270                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7271                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7272                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7273                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7274                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7275                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7276                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7277         }
7278
7279         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7280                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7281                 tmp |= DC_HPDx_INT_ACK;
7282                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7283         }
7284         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7285                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7286                 tmp |= DC_HPDx_INT_ACK;
7287                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7288         }
7289         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7290                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7291                 tmp |= DC_HPDx_INT_ACK;
7292                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7293         }
7294         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7295                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7296                 tmp |= DC_HPDx_INT_ACK;
7297                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7298         }
7299         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7300                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7301                 tmp |= DC_HPDx_INT_ACK;
7302                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7303         }
7304         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7305                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7306                 tmp |= DC_HPDx_INT_ACK;
7307                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7308         }
7309 }
7310
7311 /**
7312  * cik_irq_disable - disable interrupts
7313  *
7314  * @rdev: radeon_device pointer
7315  *
7316  * Disable interrupts on the hw (CIK).
7317  */
7318 static void cik_irq_disable(struct radeon_device *rdev)
7319 {
7320         cik_disable_interrupts(rdev);
7321         /* Wait and acknowledge irq */
7322         mdelay(1);
7323         cik_irq_ack(rdev);
7324         cik_disable_interrupt_state(rdev);
7325 }
7326
7327 /**
7328  * cik_irq_disable - disable interrupts for suspend
7329  *
7330  * @rdev: radeon_device pointer
7331  *
7332  * Disable interrupts and stop the RLC (CIK).
7333  * Used for suspend.
7334  */
7335 static void cik_irq_suspend(struct radeon_device *rdev)
7336 {
7337         cik_irq_disable(rdev);
7338         cik_rlc_stop(rdev);
7339 }
7340
7341 /**
7342  * cik_irq_fini - tear down interrupt support
7343  *
7344  * @rdev: radeon_device pointer
7345  *
7346  * Disable interrupts on the hw and free the IH ring
7347  * buffer (CIK).
7348  * Used for driver unload.
7349  */
7350 static void cik_irq_fini(struct radeon_device *rdev)
7351 {
7352         cik_irq_suspend(rdev);
7353         r600_ih_ring_fini(rdev);
7354 }
7355
7356 /**
7357  * cik_get_ih_wptr - get the IH ring buffer wptr
7358  *
7359  * @rdev: radeon_device pointer
7360  *
7361  * Get the IH ring buffer wptr from either the register
7362  * or the writeback memory buffer (CIK).  Also check for
7363  * ring buffer overflow and deal with it.
7364  * Used by cik_irq_process().
7365  * Returns the value of the wptr.
7366  */
7367 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7368 {
7369         u32 wptr, tmp;
7370
7371         if (rdev->wb.enabled)
7372                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7373         else
7374                 wptr = RREG32(IH_RB_WPTR);
7375
7376         if (wptr & RB_OVERFLOW) {
7377                 wptr &= ~RB_OVERFLOW;
7378                 /* When a ring buffer overflow happen start parsing interrupt
7379                  * from the last not overwritten vector (wptr + 16). Hopefully
7380                  * this should allow us to catchup.
7381                  */
7382                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7383                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
7384                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7385                 tmp = RREG32(IH_RB_CNTL);
7386                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7387                 WREG32(IH_RB_CNTL, tmp);
7388         }
7389         return (wptr & rdev->ih.ptr_mask);
7390 }
7391
7392 /*        CIK IV Ring
7393  * Each IV ring entry is 128 bits:
7394  * [7:0]    - interrupt source id
7395  * [31:8]   - reserved
7396  * [59:32]  - interrupt source data
7397  * [63:60]  - reserved
7398  * [71:64]  - RINGID
7399  *            CP:
7400  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7401  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7402  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7403  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7404  *            PIPE_ID - ME0 0=3D
7405  *                    - ME1&2 compute dispatcher (4 pipes each)
7406  *            SDMA:
7407  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7408  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7409  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7410  * [79:72]  - VMID
7411  * [95:80]  - PASID
7412  * [127:96] - reserved
7413  */
7414 /**
7415  * cik_irq_process - interrupt handler
7416  *
7417  * @rdev: radeon_device pointer
7418  *
7419  * Interrupt hander (CIK).  Walk the IH ring,
7420  * ack interrupts and schedule work to handle
7421  * interrupt events.
7422  * Returns irq process return code.
7423  */
7424 irqreturn_t cik_irq_process(struct radeon_device *rdev)
7425 {
7426         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7427         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7428         u32 wptr;
7429         u32 rptr;
7430         u32 src_id, src_data, ring_id;
7431         u8 me_id, pipe_id, queue_id;
7432         u32 ring_index;
7433         bool queue_hotplug = false;
7434         bool queue_reset = false;
7435         u32 addr, status, mc_client;
7436         bool queue_thermal = false;
7437
7438         if (!rdev->ih.enabled || rdev->shutdown)
7439                 return IRQ_NONE;
7440
7441         wptr = cik_get_ih_wptr(rdev);
7442
7443 restart_ih:
7444         /* is somebody else already processing irqs? */
7445         if (atomic_xchg(&rdev->ih.lock, 1))
7446                 return IRQ_NONE;
7447
7448         rptr = rdev->ih.rptr;
7449         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7450
7451         /* Order reading of wptr vs. reading of IH ring data */
7452         rmb();
7453
7454         /* display interrupts */
7455         cik_irq_ack(rdev);
7456
7457         while (rptr != wptr) {
7458                 /* wptr/rptr are in bytes! */
7459                 ring_index = rptr / 4;
7460                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7461                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7462                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7463
7464                 switch (src_id) {
7465                 case 1: /* D1 vblank/vline */
7466                         switch (src_data) {
7467                         case 0: /* D1 vblank */
7468                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7469                                         if (rdev->irq.crtc_vblank_int[0]) {
7470                                                 drm_handle_vblank(rdev->ddev, 0);
7471                                                 rdev->pm.vblank_sync = true;
7472                                                 wake_up(&rdev->irq.vblank_queue);
7473                                         }
7474                                         if (atomic_read(&rdev->irq.pflip[0]))
7475                                                 radeon_crtc_handle_vblank(rdev, 0);
7476                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7477                                         DRM_DEBUG("IH: D1 vblank\n");
7478                                 }
7479                                 break;
7480                         case 1: /* D1 vline */
7481                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7482                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7483                                         DRM_DEBUG("IH: D1 vline\n");
7484                                 }
7485                                 break;
7486                         default:
7487                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7488                                 break;
7489                         }
7490                         break;
7491                 case 2: /* D2 vblank/vline */
7492                         switch (src_data) {
7493                         case 0: /* D2 vblank */
7494                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7495                                         if (rdev->irq.crtc_vblank_int[1]) {
7496                                                 drm_handle_vblank(rdev->ddev, 1);
7497                                                 rdev->pm.vblank_sync = true;
7498                                                 wake_up(&rdev->irq.vblank_queue);
7499                                         }
7500                                         if (atomic_read(&rdev->irq.pflip[1]))
7501                                                 radeon_crtc_handle_vblank(rdev, 1);
7502                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7503                                         DRM_DEBUG("IH: D2 vblank\n");
7504                                 }
7505                                 break;
7506                         case 1: /* D2 vline */
7507                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7508                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7509                                         DRM_DEBUG("IH: D2 vline\n");
7510                                 }
7511                                 break;
7512                         default:
7513                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7514                                 break;
7515                         }
7516                         break;
7517                 case 3: /* D3 vblank/vline */
7518                         switch (src_data) {
7519                         case 0: /* D3 vblank */
7520                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7521                                         if (rdev->irq.crtc_vblank_int[2]) {
7522                                                 drm_handle_vblank(rdev->ddev, 2);
7523                                                 rdev->pm.vblank_sync = true;
7524                                                 wake_up(&rdev->irq.vblank_queue);
7525                                         }
7526                                         if (atomic_read(&rdev->irq.pflip[2]))
7527                                                 radeon_crtc_handle_vblank(rdev, 2);
7528                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7529                                         DRM_DEBUG("IH: D3 vblank\n");
7530                                 }
7531                                 break;
7532                         case 1: /* D3 vline */
7533                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7534                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7535                                         DRM_DEBUG("IH: D3 vline\n");
7536                                 }
7537                                 break;
7538                         default:
7539                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7540                                 break;
7541                         }
7542                         break;
7543                 case 4: /* D4 vblank/vline */
7544                         switch (src_data) {
7545                         case 0: /* D4 vblank */
7546                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7547                                         if (rdev->irq.crtc_vblank_int[3]) {
7548                                                 drm_handle_vblank(rdev->ddev, 3);
7549                                                 rdev->pm.vblank_sync = true;
7550                                                 wake_up(&rdev->irq.vblank_queue);
7551                                         }
7552                                         if (atomic_read(&rdev->irq.pflip[3]))
7553                                                 radeon_crtc_handle_vblank(rdev, 3);
7554                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7555                                         DRM_DEBUG("IH: D4 vblank\n");
7556                                 }
7557                                 break;
7558                         case 1: /* D4 vline */
7559                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7560                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7561                                         DRM_DEBUG("IH: D4 vline\n");
7562                                 }
7563                                 break;
7564                         default:
7565                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7566                                 break;
7567                         }
7568                         break;
7569                 case 5: /* D5 vblank/vline */
7570                         switch (src_data) {
7571                         case 0: /* D5 vblank */
7572                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7573                                         if (rdev->irq.crtc_vblank_int[4]) {
7574                                                 drm_handle_vblank(rdev->ddev, 4);
7575                                                 rdev->pm.vblank_sync = true;
7576                                                 wake_up(&rdev->irq.vblank_queue);
7577                                         }
7578                                         if (atomic_read(&rdev->irq.pflip[4]))
7579                                                 radeon_crtc_handle_vblank(rdev, 4);
7580                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7581                                         DRM_DEBUG("IH: D5 vblank\n");
7582                                 }
7583                                 break;
7584                         case 1: /* D5 vline */
7585                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7586                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7587                                         DRM_DEBUG("IH: D5 vline\n");
7588                                 }
7589                                 break;
7590                         default:
7591                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7592                                 break;
7593                         }
7594                         break;
7595                 case 6: /* D6 vblank/vline */
7596                         switch (src_data) {
7597                         case 0: /* D6 vblank */
7598                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7599                                         if (rdev->irq.crtc_vblank_int[5]) {
7600                                                 drm_handle_vblank(rdev->ddev, 5);
7601                                                 rdev->pm.vblank_sync = true;
7602                                                 wake_up(&rdev->irq.vblank_queue);
7603                                         }
7604                                         if (atomic_read(&rdev->irq.pflip[5]))
7605                                                 radeon_crtc_handle_vblank(rdev, 5);
7606                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7607                                         DRM_DEBUG("IH: D6 vblank\n");
7608                                 }
7609                                 break;
7610                         case 1: /* D6 vline */
7611                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7612                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7613                                         DRM_DEBUG("IH: D6 vline\n");
7614                                 }
7615                                 break;
7616                         default:
7617                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7618                                 break;
7619                         }
7620                         break;
7621                 case 8: /* D1 page flip */
7622                 case 10: /* D2 page flip */
7623                 case 12: /* D3 page flip */
7624                 case 14: /* D4 page flip */
7625                 case 16: /* D5 page flip */
7626                 case 18: /* D6 page flip */
7627                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7628                         if (radeon_use_pflipirq > 0)
7629                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7630                         break;
7631                 case 42: /* HPD hotplug */
7632                         switch (src_data) {
7633                         case 0:
7634                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7635                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7636                                         queue_hotplug = true;
7637                                         DRM_DEBUG("IH: HPD1\n");
7638                                 }
7639                                 break;
7640                         case 1:
7641                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7642                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7643                                         queue_hotplug = true;
7644                                         DRM_DEBUG("IH: HPD2\n");
7645                                 }
7646                                 break;
7647                         case 2:
7648                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7649                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7650                                         queue_hotplug = true;
7651                                         DRM_DEBUG("IH: HPD3\n");
7652                                 }
7653                                 break;
7654                         case 3:
7655                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7656                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7657                                         queue_hotplug = true;
7658                                         DRM_DEBUG("IH: HPD4\n");
7659                                 }
7660                                 break;
7661                         case 4:
7662                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7663                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7664                                         queue_hotplug = true;
7665                                         DRM_DEBUG("IH: HPD5\n");
7666                                 }
7667                                 break;
7668                         case 5:
7669                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7670                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7671                                         queue_hotplug = true;
7672                                         DRM_DEBUG("IH: HPD6\n");
7673                                 }
7674                                 break;
7675                         default:
7676                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7677                                 break;
7678                         }
7679                         break;
7680                 case 124: /* UVD */
7681                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7682                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7683                         break;
7684                 case 146:
7685                 case 147:
7686                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7687                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7688                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7689                         /* reset addr and status */
7690                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7691                         if (addr == 0x0 && status == 0x0)
7692                                 break;
7693                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7694                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7695                                 addr);
7696                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7697                                 status);
7698                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7699                         break;
7700                 case 167: /* VCE */
7701                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7702                         switch (src_data) {
7703                         case 0:
7704                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7705                                 break;
7706                         case 1:
7707                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7708                                 break;
7709                         default:
7710                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7711                                 break;
7712                         }
7713                         break;
7714                 case 176: /* GFX RB CP_INT */
7715                 case 177: /* GFX IB CP_INT */
7716                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7717                         break;
7718                 case 181: /* CP EOP event */
7719                         DRM_DEBUG("IH: CP EOP\n");
7720                         /* XXX check the bitfield order! */
7721                         me_id = (ring_id & 0x60) >> 5;
7722                         pipe_id = (ring_id & 0x18) >> 3;
7723                         queue_id = (ring_id & 0x7) >> 0;
7724                         switch (me_id) {
7725                         case 0:
7726                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7727                                 break;
7728                         case 1:
7729                         case 2:
7730                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7731                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7732                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7733                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7734                                 break;
7735                         }
7736                         break;
7737                 case 184: /* CP Privileged reg access */
7738                         DRM_ERROR("Illegal register access in command stream\n");
7739                         /* XXX check the bitfield order! */
7740                         me_id = (ring_id & 0x60) >> 5;
7741                         pipe_id = (ring_id & 0x18) >> 3;
7742                         queue_id = (ring_id & 0x7) >> 0;
7743                         switch (me_id) {
7744                         case 0:
7745                                 /* This results in a full GPU reset, but all we need to do is soft
7746                                  * reset the CP for gfx
7747                                  */
7748                                 queue_reset = true;
7749                                 break;
7750                         case 1:
7751                                 /* XXX compute */
7752                                 queue_reset = true;
7753                                 break;
7754                         case 2:
7755                                 /* XXX compute */
7756                                 queue_reset = true;
7757                                 break;
7758                         }
7759                         break;
7760                 case 185: /* CP Privileged inst */
7761                         DRM_ERROR("Illegal instruction in command stream\n");
7762                         /* XXX check the bitfield order! */
7763                         me_id = (ring_id & 0x60) >> 5;
7764                         pipe_id = (ring_id & 0x18) >> 3;
7765                         queue_id = (ring_id & 0x7) >> 0;
7766                         switch (me_id) {
7767                         case 0:
7768                                 /* This results in a full GPU reset, but all we need to do is soft
7769                                  * reset the CP for gfx
7770                                  */
7771                                 queue_reset = true;
7772                                 break;
7773                         case 1:
7774                                 /* XXX compute */
7775                                 queue_reset = true;
7776                                 break;
7777                         case 2:
7778                                 /* XXX compute */
7779                                 queue_reset = true;
7780                                 break;
7781                         }
7782                         break;
7783                 case 224: /* SDMA trap event */
7784                         /* XXX check the bitfield order! */
7785                         me_id = (ring_id & 0x3) >> 0;
7786                         queue_id = (ring_id & 0xc) >> 2;
7787                         DRM_DEBUG("IH: SDMA trap\n");
7788                         switch (me_id) {
7789                         case 0:
7790                                 switch (queue_id) {
7791                                 case 0:
7792                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7793                                         break;
7794                                 case 1:
7795                                         /* XXX compute */
7796                                         break;
7797                                 case 2:
7798                                         /* XXX compute */
7799                                         break;
7800                                 }
7801                                 break;
7802                         case 1:
7803                                 switch (queue_id) {
7804                                 case 0:
7805                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7806                                         break;
7807                                 case 1:
7808                                         /* XXX compute */
7809                                         break;
7810                                 case 2:
7811                                         /* XXX compute */
7812                                         break;
7813                                 }
7814                                 break;
7815                         }
7816                         break;
7817                 case 230: /* thermal low to high */
7818                         DRM_DEBUG("IH: thermal low to high\n");
7819                         rdev->pm.dpm.thermal.high_to_low = false;
7820                         queue_thermal = true;
7821                         break;
7822                 case 231: /* thermal high to low */
7823                         DRM_DEBUG("IH: thermal high to low\n");
7824                         rdev->pm.dpm.thermal.high_to_low = true;
7825                         queue_thermal = true;
7826                         break;
7827                 case 233: /* GUI IDLE */
7828                         DRM_DEBUG("IH: GUI idle\n");
7829                         break;
7830                 case 241: /* SDMA Privileged inst */
7831                 case 247: /* SDMA Privileged inst */
7832                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
7833                         /* XXX check the bitfield order! */
7834                         me_id = (ring_id & 0x3) >> 0;
7835                         queue_id = (ring_id & 0xc) >> 2;
7836                         switch (me_id) {
7837                         case 0:
7838                                 switch (queue_id) {
7839                                 case 0:
7840                                         queue_reset = true;
7841                                         break;
7842                                 case 1:
7843                                         /* XXX compute */
7844                                         queue_reset = true;
7845                                         break;
7846                                 case 2:
7847                                         /* XXX compute */
7848                                         queue_reset = true;
7849                                         break;
7850                                 }
7851                                 break;
7852                         case 1:
7853                                 switch (queue_id) {
7854                                 case 0:
7855                                         queue_reset = true;
7856                                         break;
7857                                 case 1:
7858                                         /* XXX compute */
7859                                         queue_reset = true;
7860                                         break;
7861                                 case 2:
7862                                         /* XXX compute */
7863                                         queue_reset = true;
7864                                         break;
7865                                 }
7866                                 break;
7867                         }
7868                         break;
7869                 default:
7870                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7871                         break;
7872                 }
7873
7874                 /* wptr/rptr are in bytes! */
7875                 rptr += 16;
7876                 rptr &= rdev->ih.ptr_mask;
7877                 WREG32(IH_RB_RPTR, rptr);
7878         }
7879         if (queue_hotplug)
7880                 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
7881         if (queue_reset) {
7882                 rdev->needs_reset = true;
7883                 wake_up_all(&rdev->fence_queue);
7884         }
7885         if (queue_thermal)
7886                 taskqueue_enqueue(rdev->tq, &rdev->pm.dpm.thermal.work);
7887         rdev->ih.rptr = rptr;
7888         atomic_set(&rdev->ih.lock, 0);
7889
7890         /* make sure wptr hasn't changed while processing */
7891         wptr = cik_get_ih_wptr(rdev);
7892         if (wptr != rptr)
7893                 goto restart_ih;
7894
7895         return IRQ_HANDLED;
7896 }
7897
7898 /*
7899  * startup/shutdown callbacks
7900  */
7901 /**
7902  * cik_startup - program the asic to a functional state
7903  *
7904  * @rdev: radeon_device pointer
7905  *
7906  * Programs the asic to a functional state (CIK).
7907  * Called by cik_init() and cik_resume().
7908  * Returns 0 for success, error for failure.
7909  */
7910 static int cik_startup(struct radeon_device *rdev)
7911 {
7912         struct radeon_ring *ring;
7913         u32 nop;
7914         int r;
7915
7916         /* enable pcie gen2/3 link */
7917         cik_pcie_gen3_enable(rdev);
7918         /* enable aspm */
7919         cik_program_aspm(rdev);
7920
7921         /* scratch needs to be initialized before MC */
7922         r = r600_vram_scratch_init(rdev);
7923         if (r)
7924                 return r;
7925
7926         cik_mc_program(rdev);
7927
7928         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7929                 r = ci_mc_load_microcode(rdev);
7930                 if (r) {
7931                         DRM_ERROR("Failed to load MC firmware!\n");
7932                         return r;
7933                 }
7934         }
7935
7936         r = cik_pcie_gart_enable(rdev);
7937         if (r)
7938                 return r;
7939         cik_gpu_init(rdev);
7940
7941         /* allocate rlc buffers */
7942         if (rdev->flags & RADEON_IS_IGP) {
7943                 if (rdev->family == CHIP_KAVERI) {
7944                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7945                         rdev->rlc.reg_list_size =
7946                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7947                 } else {
7948                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7949                         rdev->rlc.reg_list_size =
7950                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7951                 }
7952         }
7953         rdev->rlc.cs_data = ci_cs_data;
7954         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7955         r = sumo_rlc_init(rdev);
7956         if (r) {
7957                 DRM_ERROR("Failed to init rlc BOs!\n");
7958                 return r;
7959         }
7960
7961         /* allocate wb buffer */
7962         r = radeon_wb_init(rdev);
7963         if (r)
7964                 return r;
7965
7966         /* allocate mec buffers */
7967         r = cik_mec_init(rdev);
7968         if (r) {
7969                 DRM_ERROR("Failed to init MEC BOs!\n");
7970                 return r;
7971         }
7972
7973         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7974         if (r) {
7975                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7976                 return r;
7977         }
7978
7979         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7980         if (r) {
7981                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7982                 return r;
7983         }
7984
7985         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7986         if (r) {
7987                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7988                 return r;
7989         }
7990
7991         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7992         if (r) {
7993                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7994                 return r;
7995         }
7996
7997         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7998         if (r) {
7999                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8000                 return r;
8001         }
8002
8003         r = radeon_uvd_resume(rdev);
8004         if (!r) {
8005                 r = uvd_v4_2_resume(rdev);
8006                 if (!r) {
8007                         r = radeon_fence_driver_start_ring(rdev,
8008                                                            R600_RING_TYPE_UVD_INDEX);
8009                         if (r)
8010                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8011                 }
8012         }
8013         if (r)
8014                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8015
8016         r = radeon_vce_resume(rdev);
8017         if (!r) {
8018                 r = vce_v2_0_resume(rdev);
8019                 if (!r)
8020                         r = radeon_fence_driver_start_ring(rdev,
8021                                                            TN_RING_TYPE_VCE1_INDEX);
8022                 if (!r)
8023                         r = radeon_fence_driver_start_ring(rdev,
8024                                                            TN_RING_TYPE_VCE2_INDEX);
8025         }
8026         if (r) {
8027                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8028                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8029                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8030         }
8031
8032         /* Enable IRQ */
8033         if (!rdev->irq.installed) {
8034                 r = radeon_irq_kms_init(rdev);
8035                 if (r)
8036                         return r;
8037         }
8038
8039         r = cik_irq_init(rdev);
8040         if (r) {
8041                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8042                 radeon_irq_kms_fini(rdev);
8043                 return r;
8044         }
8045         cik_irq_set(rdev);
8046
8047         if (rdev->family == CHIP_HAWAII) {
8048                 nop = RADEON_CP_PACKET2;
8049         } else {
8050                 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8051         }
8052
8053         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8054         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8055                              nop);
8056         if (r)
8057                 return r;
8058
8059         /* set up the compute queues */
8060         /* type-2 packets are deprecated on MEC, use type-3 instead */
8061         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8062         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8063                              nop);
8064         if (r)
8065                 return r;
8066         ring->me = 1; /* first MEC */
8067         ring->pipe = 0; /* first pipe */
8068         ring->queue = 0; /* first queue */
8069         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8070
8071         /* type-2 packets are deprecated on MEC, use type-3 instead */
8072         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8073         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8074                              nop);
8075         if (r)
8076                 return r;
8077         /* dGPU only have 1 MEC */
8078         ring->me = 1; /* first MEC */
8079         ring->pipe = 0; /* first pipe */
8080         ring->queue = 1; /* second queue */
8081         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8082
8083         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8084         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8085                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8086         if (r)
8087                 return r;
8088
8089         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8090         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8091                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8092         if (r)
8093                 return r;
8094
8095         r = cik_cp_resume(rdev);
8096         if (r)
8097                 return r;
8098
8099         r = cik_sdma_resume(rdev);
8100         if (r)
8101                 return r;
8102
8103         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8104         if (ring->ring_size) {
8105                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8106                                      RADEON_CP_PACKET2);
8107                 if (!r)
8108                         r = uvd_v1_0_init(rdev);
8109                 if (r)
8110                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8111         }
8112
8113         r = -ENOENT;
8114
8115         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8116         if (ring->ring_size)
8117                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8118                                      VCE_CMD_NO_OP);
8119
8120         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8121         if (ring->ring_size)
8122                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8123                                      VCE_CMD_NO_OP);
8124
8125         if (!r)
8126                 r = vce_v1_0_init(rdev);
8127         else if (r != -ENOENT)
8128                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8129
8130         r = radeon_ib_pool_init(rdev);
8131         if (r) {
8132                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8133                 return r;
8134         }
8135
8136         r = radeon_vm_manager_init(rdev);
8137         if (r) {
8138                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8139                 return r;
8140         }
8141
8142         r = dce6_audio_init(rdev);
8143         if (r)
8144                 return r;
8145
8146         return 0;
8147 }
8148
8149 /**
8150  * cik_resume - resume the asic to a functional state
8151  *
8152  * @rdev: radeon_device pointer
8153  *
8154  * Programs the asic to a functional state (CIK).
8155  * Called at resume.
8156  * Returns 0 for success, error for failure.
8157  */
8158 int cik_resume(struct radeon_device *rdev)
8159 {
8160         int r;
8161
8162         /* post card */
8163         atom_asic_init(rdev->mode_info.atom_context);
8164
8165         /* init golden registers */
8166         cik_init_golden_registers(rdev);
8167
8168         if (rdev->pm.pm_method == PM_METHOD_DPM)
8169                 radeon_pm_resume(rdev);
8170
8171         rdev->accel_working = true;
8172         r = cik_startup(rdev);
8173         if (r) {
8174                 DRM_ERROR("cik startup failed on resume\n");
8175                 rdev->accel_working = false;
8176                 return r;
8177         }
8178
8179         return r;
8180
8181 }
8182
8183 /**
8184  * cik_suspend - suspend the asic
8185  *
8186  * @rdev: radeon_device pointer
8187  *
8188  * Bring the chip into a state suitable for suspend (CIK).
8189  * Called at suspend.
8190  * Returns 0 for success.
8191  */
8192 int cik_suspend(struct radeon_device *rdev)
8193 {
8194         radeon_pm_suspend(rdev);
8195         dce6_audio_fini(rdev);
8196         radeon_vm_manager_fini(rdev);
8197         cik_cp_enable(rdev, false);
8198         cik_sdma_enable(rdev, false);
8199         uvd_v1_0_fini(rdev);
8200         radeon_uvd_suspend(rdev);
8201         radeon_vce_suspend(rdev);
8202         cik_fini_pg(rdev);
8203         cik_fini_cg(rdev);
8204         cik_irq_suspend(rdev);
8205         radeon_wb_disable(rdev);
8206         cik_pcie_gart_disable(rdev);
8207         return 0;
8208 }
8209
8210 /* Plan is to move initialization in that function and use
8211  * helper function so that radeon_device_init pretty much
8212  * do nothing more than calling asic specific function. This
8213  * should also allow to remove a bunch of callback function
8214  * like vram_info.
8215  */
8216 /**
8217  * cik_init - asic specific driver and hw init
8218  *
8219  * @rdev: radeon_device pointer
8220  *
8221  * Setup asic specific driver variables and program the hw
8222  * to a functional state (CIK).
8223  * Called at driver startup.
8224  * Returns 0 for success, errors for failure.
8225  */
8226 int cik_init(struct radeon_device *rdev)
8227 {
8228         struct radeon_ring *ring;
8229         int r;
8230
8231         /* Read BIOS */
8232         if (!radeon_get_bios(rdev)) {
8233                 if (ASIC_IS_AVIVO(rdev))
8234                         return -EINVAL;
8235         }
8236         /* Must be an ATOMBIOS */
8237         if (!rdev->is_atom_bios) {
8238                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8239                 return -EINVAL;
8240         }
8241         r = radeon_atombios_init(rdev);
8242         if (r)
8243                 return r;
8244
8245         /* Post card if necessary */
8246         if (!radeon_card_posted(rdev)) {
8247                 if (!rdev->bios) {
8248                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8249                         return -EINVAL;
8250                 }
8251                 DRM_INFO("GPU not posted. posting now...\n");
8252                 atom_asic_init(rdev->mode_info.atom_context);
8253         }
8254         /* init golden registers */
8255         cik_init_golden_registers(rdev);
8256         /* Initialize scratch registers */
8257         cik_scratch_init(rdev);
8258         /* Initialize surface registers */
8259         radeon_surface_init(rdev);
8260         /* Initialize clocks */
8261         radeon_get_clock_info(rdev->ddev);
8262
8263         /* Fence driver */
8264         r = radeon_fence_driver_init(rdev);
8265         if (r)
8266                 return r;
8267
8268         /* initialize memory controller */
8269         r = cik_mc_init(rdev);
8270         if (r)
8271                 return r;
8272         /* Memory manager */
8273         r = radeon_bo_init(rdev);
8274         if (r)
8275                 return r;
8276
8277         if (rdev->flags & RADEON_IS_IGP) {
8278                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8279                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8280                         r = cik_init_microcode(rdev);
8281                         if (r) {
8282                                 DRM_ERROR("Failed to load firmware!\n");
8283                                 return r;
8284                         }
8285                 }
8286         } else {
8287                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8288                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8289                     !rdev->mc_fw) {
8290                         r = cik_init_microcode(rdev);
8291                         if (r) {
8292                                 DRM_ERROR("Failed to load firmware!\n");
8293                                 return r;
8294                         }
8295                 }
8296         }
8297
8298         /* Initialize power management */
8299         radeon_pm_init(rdev);
8300
8301         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8302         ring->ring_obj = NULL;
8303         r600_ring_init(rdev, ring, 1024 * 1024);
8304
8305         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8306         ring->ring_obj = NULL;
8307         r600_ring_init(rdev, ring, 1024 * 1024);
8308         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8309         if (r)
8310                 return r;
8311
8312         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8313         ring->ring_obj = NULL;
8314         r600_ring_init(rdev, ring, 1024 * 1024);
8315         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8316         if (r)
8317                 return r;
8318
8319         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8320         ring->ring_obj = NULL;
8321         r600_ring_init(rdev, ring, 256 * 1024);
8322
8323         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8324         ring->ring_obj = NULL;
8325         r600_ring_init(rdev, ring, 256 * 1024);
8326
8327         r = radeon_uvd_init(rdev);
8328         if (!r) {
8329                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8330                 ring->ring_obj = NULL;
8331                 r600_ring_init(rdev, ring, 4096);
8332         }
8333
8334         r = radeon_vce_init(rdev);
8335         if (!r) {
8336                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8337                 ring->ring_obj = NULL;
8338                 r600_ring_init(rdev, ring, 4096);
8339
8340                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8341                 ring->ring_obj = NULL;
8342                 r600_ring_init(rdev, ring, 4096);
8343         }
8344
8345         rdev->ih.ring_obj = NULL;
8346         r600_ih_ring_init(rdev, 64 * 1024);
8347
8348         r = r600_pcie_gart_init(rdev);
8349         if (r)
8350                 return r;
8351
8352         rdev->accel_working = true;
8353         r = cik_startup(rdev);
8354         if (r) {
8355                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8356                 cik_cp_fini(rdev);
8357                 cik_sdma_fini(rdev);
8358                 cik_irq_fini(rdev);
8359                 sumo_rlc_fini(rdev);
8360                 cik_mec_fini(rdev);
8361                 radeon_wb_fini(rdev);
8362                 radeon_ib_pool_fini(rdev);
8363                 radeon_vm_manager_fini(rdev);
8364                 radeon_irq_kms_fini(rdev);
8365                 cik_pcie_gart_fini(rdev);
8366                 rdev->accel_working = false;
8367         }
8368
8369         /* Don't start up if the MC ucode is missing.
8370          * The default clocks and voltages before the MC ucode
8371          * is loaded are not suffient for advanced operations.
8372          */
8373         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8374                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8375                 return -EINVAL;
8376         }
8377
8378         return 0;
8379 }
8380
8381 /**
8382  * cik_fini - asic specific driver and hw fini
8383  *
8384  * @rdev: radeon_device pointer
8385  *
8386  * Tear down the asic specific driver variables and program the hw
8387  * to an idle state (CIK).
8388  * Called at driver unload.
8389  */
8390 void cik_fini(struct radeon_device *rdev)
8391 {
8392         radeon_pm_fini(rdev);
8393         cik_cp_fini(rdev);
8394         cik_sdma_fini(rdev);
8395         cik_fini_pg(rdev);
8396         cik_fini_cg(rdev);
8397         cik_irq_fini(rdev);
8398         sumo_rlc_fini(rdev);
8399         cik_mec_fini(rdev);
8400         radeon_wb_fini(rdev);
8401         radeon_vm_manager_fini(rdev);
8402         radeon_ib_pool_fini(rdev);
8403         radeon_irq_kms_fini(rdev);
8404         uvd_v1_0_fini(rdev);
8405         radeon_uvd_fini(rdev);
8406         radeon_vce_fini(rdev);
8407         cik_pcie_gart_fini(rdev);
8408         r600_vram_scratch_fini(rdev);
8409         radeon_gem_fini(rdev);
8410         radeon_fence_driver_fini(rdev);
8411         radeon_bo_fini(rdev);
8412         radeon_atombios_fini(rdev);
8413         kfree(rdev->bios);
8414         rdev->bios = NULL;
8415 }
8416
8417 void dce8_program_fmt(struct drm_encoder *encoder)
8418 {
8419         struct drm_device *dev = encoder->dev;
8420         struct radeon_device *rdev = dev->dev_private;
8421         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8422         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8423         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8424         int bpc = 0;
8425         u32 tmp = 0;
8426         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8427
8428         if (connector) {
8429                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8430                 bpc = radeon_get_monitor_bpc(connector);
8431                 dither = radeon_connector->dither;
8432         }
8433
8434         /* LVDS/eDP FMT is set up by atom */
8435         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8436                 return;
8437
8438         /* not needed for analog */
8439         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8440             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8441                 return;
8442
8443         if (bpc == 0)
8444                 return;
8445
8446         switch (bpc) {
8447         case 6:
8448                 if (dither == RADEON_FMT_DITHER_ENABLE)
8449                         /* XXX sort out optimal dither settings */
8450                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8451                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8452                 else
8453                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8454                 break;
8455         case 8:
8456                 if (dither == RADEON_FMT_DITHER_ENABLE)
8457                         /* XXX sort out optimal dither settings */
8458                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8459                                 FMT_RGB_RANDOM_ENABLE |
8460                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8461                 else
8462                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8463                 break;
8464         case 10:
8465                 if (dither == RADEON_FMT_DITHER_ENABLE)
8466                         /* XXX sort out optimal dither settings */
8467                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8468                                 FMT_RGB_RANDOM_ENABLE |
8469                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8470                 else
8471                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8472                 break;
8473         default:
8474                 /* not needed */
8475                 break;
8476         }
8477
8478         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8479 }
8480
8481 /* display watermark setup */
8482 /**
8483  * dce8_line_buffer_adjust - Set up the line buffer
8484  *
8485  * @rdev: radeon_device pointer
8486  * @radeon_crtc: the selected display controller
8487  * @mode: the current display mode on the selected display
8488  * controller
8489  *
8490  * Setup up the line buffer allocation for
8491  * the selected display controller (CIK).
8492  * Returns the line buffer size in pixels.
8493  */
8494 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8495                                    struct radeon_crtc *radeon_crtc,
8496                                    struct drm_display_mode *mode)
8497 {
8498         u32 tmp, buffer_alloc, i;
8499         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8500         /*
8501          * Line Buffer Setup
8502          * There are 6 line buffers, one for each display controllers.
8503          * There are 3 partitions per LB. Select the number of partitions
8504          * to enable based on the display width.  For display widths larger
8505          * than 4096, you need use to use 2 display controllers and combine
8506          * them using the stereo blender.
8507          */
8508         if (radeon_crtc->base.enabled && mode) {
8509                 if (mode->crtc_hdisplay < 1920) {
8510                         tmp = 1;
8511                         buffer_alloc = 2;
8512                 } else if (mode->crtc_hdisplay < 2560) {
8513                         tmp = 2;
8514                         buffer_alloc = 2;
8515                 } else if (mode->crtc_hdisplay < 4096) {
8516                         tmp = 0;
8517                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8518                 } else {
8519                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8520                         tmp = 0;
8521                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8522                 }
8523         } else {
8524                 tmp = 1;
8525                 buffer_alloc = 0;
8526         }
8527
8528         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8529                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8530
8531         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8532                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8533         for (i = 0; i < rdev->usec_timeout; i++) {
8534                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8535                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8536                         break;
8537                 udelay(1);
8538         }
8539
8540         if (radeon_crtc->base.enabled && mode) {
8541                 switch (tmp) {
8542                 case 0:
8543                 default:
8544                         return 4096 * 2;
8545                 case 1:
8546                         return 1920 * 2;
8547                 case 2:
8548                         return 2560 * 2;
8549                 }
8550         }
8551
8552         /* controller not enabled, so no lb used */
8553         return 0;
8554 }
8555
8556 /**
8557  * cik_get_number_of_dram_channels - get the number of dram channels
8558  *
8559  * @rdev: radeon_device pointer
8560  *
8561  * Look up the number of video ram channels (CIK).
8562  * Used for display watermark bandwidth calculations
8563  * Returns the number of dram channels
8564  */
8565 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8566 {
8567         u32 tmp = RREG32(MC_SHARED_CHMAP);
8568
8569         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8570         case 0:
8571         default:
8572                 return 1;
8573         case 1:
8574                 return 2;
8575         case 2:
8576                 return 4;
8577         case 3:
8578                 return 8;
8579         case 4:
8580                 return 3;
8581         case 5:
8582                 return 6;
8583         case 6:
8584                 return 10;
8585         case 7:
8586                 return 12;
8587         case 8:
8588                 return 16;
8589         }
8590 }
8591
8592 struct dce8_wm_params {
8593         u32 dram_channels; /* number of dram channels */
8594         u32 yclk;          /* bandwidth per dram data pin in kHz */
8595         u32 sclk;          /* engine clock in kHz */
8596         u32 disp_clk;      /* display clock in kHz */
8597         u32 src_width;     /* viewport width */
8598         u32 active_time;   /* active display time in ns */
8599         u32 blank_time;    /* blank time in ns */
8600         bool interlaced;    /* mode is interlaced */
8601         fixed20_12 vsc;    /* vertical scale ratio */
8602         u32 num_heads;     /* number of active crtcs */
8603         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8604         u32 lb_size;       /* line buffer allocated to pipe */
8605         u32 vtaps;         /* vertical scaler taps */
8606 };
8607
8608 /**
8609  * dce8_dram_bandwidth - get the dram bandwidth
8610  *
8611  * @wm: watermark calculation data
8612  *
8613  * Calculate the raw dram bandwidth (CIK).
8614  * Used for display watermark bandwidth calculations
8615  * Returns the dram bandwidth in MBytes/s
8616  */
8617 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8618 {
8619         /* Calculate raw DRAM Bandwidth */
8620         fixed20_12 dram_efficiency; /* 0.7 */
8621         fixed20_12 yclk, dram_channels, bandwidth;
8622         fixed20_12 a;
8623
8624         a.full = dfixed_const(1000);
8625         yclk.full = dfixed_const(wm->yclk);
8626         yclk.full = dfixed_div(yclk, a);
8627         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8628         a.full = dfixed_const(10);
8629         dram_efficiency.full = dfixed_const(7);
8630         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8631         bandwidth.full = dfixed_mul(dram_channels, yclk);
8632         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8633
8634         return dfixed_trunc(bandwidth);
8635 }
8636
8637 /**
8638  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8639  *
8640  * @wm: watermark calculation data
8641  *
8642  * Calculate the dram bandwidth used for display (CIK).
8643  * Used for display watermark bandwidth calculations
8644  * Returns the dram bandwidth for display in MBytes/s
8645  */
8646 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8647 {
8648         /* Calculate DRAM Bandwidth and the part allocated to display. */
8649         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8650         fixed20_12 yclk, dram_channels, bandwidth;
8651         fixed20_12 a;
8652
8653         a.full = dfixed_const(1000);
8654         yclk.full = dfixed_const(wm->yclk);
8655         yclk.full = dfixed_div(yclk, a);
8656         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8657         a.full = dfixed_const(10);
8658         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8659         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8660         bandwidth.full = dfixed_mul(dram_channels, yclk);
8661         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8662
8663         return dfixed_trunc(bandwidth);
8664 }
8665
8666 /**
8667  * dce8_data_return_bandwidth - get the data return bandwidth
8668  *
8669  * @wm: watermark calculation data
8670  *
8671  * Calculate the data return bandwidth used for display (CIK).
8672  * Used for display watermark bandwidth calculations
8673  * Returns the data return bandwidth in MBytes/s
8674  */
8675 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8676 {
8677         /* Calculate the display Data return Bandwidth */
8678         fixed20_12 return_efficiency; /* 0.8 */
8679         fixed20_12 sclk, bandwidth;
8680         fixed20_12 a;
8681
8682         a.full = dfixed_const(1000);
8683         sclk.full = dfixed_const(wm->sclk);
8684         sclk.full = dfixed_div(sclk, a);
8685         a.full = dfixed_const(10);
8686         return_efficiency.full = dfixed_const(8);
8687         return_efficiency.full = dfixed_div(return_efficiency, a);
8688         a.full = dfixed_const(32);
8689         bandwidth.full = dfixed_mul(a, sclk);
8690         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8691
8692         return dfixed_trunc(bandwidth);
8693 }
8694
8695 /**
8696  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8697  *
8698  * @wm: watermark calculation data
8699  *
8700  * Calculate the dmif bandwidth used for display (CIK).
8701  * Used for display watermark bandwidth calculations
8702  * Returns the dmif bandwidth in MBytes/s
8703  */
8704 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8705 {
8706         /* Calculate the DMIF Request Bandwidth */
8707         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8708         fixed20_12 disp_clk, bandwidth;
8709         fixed20_12 a, b;
8710
8711         a.full = dfixed_const(1000);
8712         disp_clk.full = dfixed_const(wm->disp_clk);
8713         disp_clk.full = dfixed_div(disp_clk, a);
8714         a.full = dfixed_const(32);
8715         b.full = dfixed_mul(a, disp_clk);
8716
8717         a.full = dfixed_const(10);
8718         disp_clk_request_efficiency.full = dfixed_const(8);
8719         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8720
8721         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8722
8723         return dfixed_trunc(bandwidth);
8724 }
8725
8726 /**
8727  * dce8_available_bandwidth - get the min available bandwidth
8728  *
8729  * @wm: watermark calculation data
8730  *
8731  * Calculate the min available bandwidth used for display (CIK).
8732  * Used for display watermark bandwidth calculations
8733  * Returns the min available bandwidth in MBytes/s
8734  */
8735 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8736 {
8737         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8738         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8739         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8740         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8741
8742         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8743 }
8744
8745 /**
8746  * dce8_average_bandwidth - get the average available bandwidth
8747  *
8748  * @wm: watermark calculation data
8749  *
8750  * Calculate the average available bandwidth used for display (CIK).
8751  * Used for display watermark bandwidth calculations
8752  * Returns the average available bandwidth in MBytes/s
8753  */
8754 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8755 {
8756         /* Calculate the display mode Average Bandwidth
8757          * DisplayMode should contain the source and destination dimensions,
8758          * timing, etc.
8759          */
8760         fixed20_12 bpp;
8761         fixed20_12 line_time;
8762         fixed20_12 src_width;
8763         fixed20_12 bandwidth;
8764         fixed20_12 a;
8765
8766         a.full = dfixed_const(1000);
8767         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8768         line_time.full = dfixed_div(line_time, a);
8769         bpp.full = dfixed_const(wm->bytes_per_pixel);
8770         src_width.full = dfixed_const(wm->src_width);
8771         bandwidth.full = dfixed_mul(src_width, bpp);
8772         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8773         bandwidth.full = dfixed_div(bandwidth, line_time);
8774
8775         return dfixed_trunc(bandwidth);
8776 }
8777
8778 /**
8779  * dce8_latency_watermark - get the latency watermark
8780  *
8781  * @wm: watermark calculation data
8782  *
8783  * Calculate the latency watermark (CIK).
8784  * Used for display watermark bandwidth calculations
8785  * Returns the latency watermark in ns
8786  */
8787 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8788 {
8789         /* First calculate the latency in ns */
8790         u32 mc_latency = 2000; /* 2000 ns. */
8791         u32 available_bandwidth = dce8_available_bandwidth(wm);
8792         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8793         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8794         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8795         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8796                 (wm->num_heads * cursor_line_pair_return_time);
8797         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8798         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8799         u32 tmp, dmif_size = 12288;
8800         fixed20_12 a, b, c;
8801
8802         if (wm->num_heads == 0)
8803                 return 0;
8804
8805         a.full = dfixed_const(2);
8806         b.full = dfixed_const(1);
8807         if ((wm->vsc.full > a.full) ||
8808             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8809             (wm->vtaps >= 5) ||
8810             ((wm->vsc.full >= a.full) && wm->interlaced))
8811                 max_src_lines_per_dst_line = 4;
8812         else
8813                 max_src_lines_per_dst_line = 2;
8814
8815         a.full = dfixed_const(available_bandwidth);
8816         b.full = dfixed_const(wm->num_heads);
8817         a.full = dfixed_div(a, b);
8818
8819         b.full = dfixed_const(mc_latency + 512);
8820         c.full = dfixed_const(wm->disp_clk);
8821         b.full = dfixed_div(b, c);
8822
8823         c.full = dfixed_const(dmif_size);
8824         b.full = dfixed_div(c, b);
8825
8826         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8827
8828         b.full = dfixed_const(1000);
8829         c.full = dfixed_const(wm->disp_clk);
8830         b.full = dfixed_div(c, b);
8831         c.full = dfixed_const(wm->bytes_per_pixel);
8832         b.full = dfixed_mul(b, c);
8833
8834         lb_fill_bw = min(tmp, dfixed_trunc(b));
8835
8836         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8837         b.full = dfixed_const(1000);
8838         c.full = dfixed_const(lb_fill_bw);
8839         b.full = dfixed_div(c, b);
8840         a.full = dfixed_div(a, b);
8841         line_fill_time = dfixed_trunc(a);
8842
8843         if (line_fill_time < wm->active_time)
8844                 return latency;
8845         else
8846                 return latency + (line_fill_time - wm->active_time);
8847
8848 }
8849
8850 /**
8851  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8852  * average and available dram bandwidth
8853  *
8854  * @wm: watermark calculation data
8855  *
8856  * Check if the display average bandwidth fits in the display
8857  * dram bandwidth (CIK).
8858  * Used for display watermark bandwidth calculations
8859  * Returns true if the display fits, false if not.
8860  */
8861 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8862 {
8863         if (dce8_average_bandwidth(wm) <=
8864             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8865                 return true;
8866         else
8867                 return false;
8868 }
8869
8870 /**
8871  * dce8_average_bandwidth_vs_available_bandwidth - check
8872  * average and available bandwidth
8873  *
8874  * @wm: watermark calculation data
8875  *
8876  * Check if the display average bandwidth fits in the display
8877  * available bandwidth (CIK).
8878  * Used for display watermark bandwidth calculations
8879  * Returns true if the display fits, false if not.
8880  */
8881 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8882 {
8883         if (dce8_average_bandwidth(wm) <=
8884             (dce8_available_bandwidth(wm) / wm->num_heads))
8885                 return true;
8886         else
8887                 return false;
8888 }
8889
8890 /**
8891  * dce8_check_latency_hiding - check latency hiding
8892  *
8893  * @wm: watermark calculation data
8894  *
8895  * Check latency hiding (CIK).
8896  * Used for display watermark bandwidth calculations
8897  * Returns true if the display fits, false if not.
8898  */
8899 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8900 {
8901         u32 lb_partitions = wm->lb_size / wm->src_width;
8902         u32 line_time = wm->active_time + wm->blank_time;
8903         u32 latency_tolerant_lines;
8904         u32 latency_hiding;
8905         fixed20_12 a;
8906
8907         a.full = dfixed_const(1);
8908         if (wm->vsc.full > a.full)
8909                 latency_tolerant_lines = 1;
8910         else {
8911                 if (lb_partitions <= (wm->vtaps + 1))
8912                         latency_tolerant_lines = 1;
8913                 else
8914                         latency_tolerant_lines = 2;
8915         }
8916
8917         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8918
8919         if (dce8_latency_watermark(wm) <= latency_hiding)
8920                 return true;
8921         else
8922                 return false;
8923 }
8924
8925 /**
8926  * dce8_program_watermarks - program display watermarks
8927  *
8928  * @rdev: radeon_device pointer
8929  * @radeon_crtc: the selected display controller
8930  * @lb_size: line buffer size
8931  * @num_heads: number of display controllers in use
8932  *
8933  * Calculate and program the display watermarks for the
8934  * selected display controller (CIK).
8935  */
8936 static void dce8_program_watermarks(struct radeon_device *rdev,
8937                                     struct radeon_crtc *radeon_crtc,
8938                                     u32 lb_size, u32 num_heads)
8939 {
8940         struct drm_display_mode *mode = &radeon_crtc->base.mode;
8941         struct dce8_wm_params wm_low, wm_high;
8942         u32 pixel_period;
8943         u32 line_time = 0;
8944         u32 latency_watermark_a = 0, latency_watermark_b = 0;
8945         u32 tmp, wm_mask;
8946
8947         if (radeon_crtc->base.enabled && num_heads && mode) {
8948                 pixel_period = 1000000 / (u32)mode->clock;
8949                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8950
8951                 /* watermark for high clocks */
8952                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8953                     rdev->pm.dpm_enabled) {
8954                         wm_high.yclk =
8955                                 radeon_dpm_get_mclk(rdev, false) * 10;
8956                         wm_high.sclk =
8957                                 radeon_dpm_get_sclk(rdev, false) * 10;
8958                 } else {
8959                         wm_high.yclk = rdev->pm.current_mclk * 10;
8960                         wm_high.sclk = rdev->pm.current_sclk * 10;
8961                 }
8962
8963                 wm_high.disp_clk = mode->clock;
8964                 wm_high.src_width = mode->crtc_hdisplay;
8965                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8966                 wm_high.blank_time = line_time - wm_high.active_time;
8967                 wm_high.interlaced = false;
8968                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8969                         wm_high.interlaced = true;
8970                 wm_high.vsc = radeon_crtc->vsc;
8971                 wm_high.vtaps = 1;
8972                 if (radeon_crtc->rmx_type != RMX_OFF)
8973                         wm_high.vtaps = 2;
8974                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8975                 wm_high.lb_size = lb_size;
8976                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8977                 wm_high.num_heads = num_heads;
8978
8979                 /* set for high clocks */
8980                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8981
8982                 /* possibly force display priority to high */
8983                 /* should really do this at mode validation time... */
8984                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8985                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8986                     !dce8_check_latency_hiding(&wm_high) ||
8987                     (rdev->disp_priority == 2)) {
8988                         DRM_DEBUG_KMS("force priority to high\n");
8989                 }
8990
8991                 /* watermark for low clocks */
8992                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8993                     rdev->pm.dpm_enabled) {
8994                         wm_low.yclk =
8995                                 radeon_dpm_get_mclk(rdev, true) * 10;
8996                         wm_low.sclk =
8997                                 radeon_dpm_get_sclk(rdev, true) * 10;
8998                 } else {
8999                         wm_low.yclk = rdev->pm.current_mclk * 10;
9000                         wm_low.sclk = rdev->pm.current_sclk * 10;
9001                 }
9002
9003                 wm_low.disp_clk = mode->clock;
9004                 wm_low.src_width = mode->crtc_hdisplay;
9005                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9006                 wm_low.blank_time = line_time - wm_low.active_time;
9007                 wm_low.interlaced = false;
9008                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9009                         wm_low.interlaced = true;
9010                 wm_low.vsc = radeon_crtc->vsc;
9011                 wm_low.vtaps = 1;
9012                 if (radeon_crtc->rmx_type != RMX_OFF)
9013                         wm_low.vtaps = 2;
9014                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9015                 wm_low.lb_size = lb_size;
9016                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9017                 wm_low.num_heads = num_heads;
9018
9019                 /* set for low clocks */
9020                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9021
9022                 /* possibly force display priority to high */
9023                 /* should really do this at mode validation time... */
9024                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9025                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9026                     !dce8_check_latency_hiding(&wm_low) ||
9027                     (rdev->disp_priority == 2)) {
9028                         DRM_DEBUG_KMS("force priority to high\n");
9029                 }
9030         }
9031
9032         /* select wm A */
9033         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9034         tmp = wm_mask;
9035         tmp &= ~LATENCY_WATERMARK_MASK(3);
9036         tmp |= LATENCY_WATERMARK_MASK(1);
9037         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9038         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9039                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9040                 LATENCY_HIGH_WATERMARK(line_time)));
9041         /* select wm B */
9042         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9043         tmp &= ~LATENCY_WATERMARK_MASK(3);
9044         tmp |= LATENCY_WATERMARK_MASK(2);
9045         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9046         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9047                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9048                 LATENCY_HIGH_WATERMARK(line_time)));
9049         /* restore original selection */
9050         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9051
9052         /* save values for DPM */
9053         radeon_crtc->line_time = line_time;
9054         radeon_crtc->wm_high = latency_watermark_a;
9055         radeon_crtc->wm_low = latency_watermark_b;
9056 }
9057
9058 /**
9059  * dce8_bandwidth_update - program display watermarks
9060  *
9061  * @rdev: radeon_device pointer
9062  *
9063  * Calculate and program the display watermarks and line
9064  * buffer allocation (CIK).
9065  */
9066 void dce8_bandwidth_update(struct radeon_device *rdev)
9067 {
9068         struct drm_display_mode *mode = NULL;
9069         u32 num_heads = 0, lb_size;
9070         int i;
9071
9072         if (!rdev->mode_info.mode_config_initialized)
9073                 return;
9074
9075         radeon_update_display_priority(rdev);
9076
9077         for (i = 0; i < rdev->num_crtc; i++) {
9078                 if (rdev->mode_info.crtcs[i]->base.enabled)
9079                         num_heads++;
9080         }
9081         for (i = 0; i < rdev->num_crtc; i++) {
9082                 mode = &rdev->mode_info.crtcs[i]->base.mode;
9083                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9084                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9085         }
9086 }
9087
9088 /**
9089  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9090  *
9091  * @rdev: radeon_device pointer
9092  *
9093  * Fetches a GPU clock counter snapshot (SI).
9094  * Returns the 64 bit clock counter snapshot.
9095  */
9096 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9097 {
9098         uint64_t clock;
9099
9100         spin_lock(&rdev->gpu_clock_mutex);
9101         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9102         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9103                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9104         spin_unlock(&rdev->gpu_clock_mutex);
9105         return clock;
9106 }
9107
9108 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9109                               u32 cntl_reg, u32 status_reg)
9110 {
9111         int r, i;
9112         struct atom_clock_dividers dividers;
9113         uint32_t tmp;
9114
9115         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9116                                            clock, false, &dividers);
9117         if (r)
9118                 return r;
9119
9120         tmp = RREG32_SMC(cntl_reg);
9121         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9122         tmp |= dividers.post_divider;
9123         WREG32_SMC(cntl_reg, tmp);
9124
9125         for (i = 0; i < 100; i++) {
9126                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9127                         break;
9128                 mdelay(10);
9129         }
9130         if (i == 100)
9131                 return -ETIMEDOUT;
9132
9133         return 0;
9134 }
9135
9136 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9137 {
9138         int r = 0;
9139
9140         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9141         if (r)
9142                 return r;
9143
9144         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9145         return r;
9146 }
9147
9148 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9149 {
9150         int r, i;
9151         struct atom_clock_dividers dividers;
9152         u32 tmp;
9153
9154         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9155                                            ecclk, false, &dividers);
9156         if (r)
9157                 return r;
9158
9159         for (i = 0; i < 100; i++) {
9160                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9161                         break;
9162                 mdelay(10);
9163         }
9164         if (i == 100)
9165                 return -ETIMEDOUT;
9166
9167         tmp = RREG32_SMC(CG_ECLK_CNTL);
9168         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9169         tmp |= dividers.post_divider;
9170         WREG32_SMC(CG_ECLK_CNTL, tmp);
9171
9172         for (i = 0; i < 100; i++) {
9173                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9174                         break;
9175                 mdelay(10);
9176         }
9177         if (i == 100)
9178                 return -ETIMEDOUT;
9179
9180         return 0;
9181 }
9182
9183 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9184 {
9185         struct pci_dev *root = rdev->pdev->bus->self;
9186         int bridge_pos, gpu_pos;
9187         u32 speed_cntl, mask, current_data_rate;
9188         int ret, i;
9189         u16 tmp16;
9190
9191         if (radeon_pcie_gen2 == 0)
9192                 return;
9193
9194         if (rdev->flags & RADEON_IS_IGP)
9195                 return;
9196
9197         if (!(rdev->flags & RADEON_IS_PCIE))
9198                 return;
9199
9200         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9201         if (ret != 0)
9202                 return;
9203
9204         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9205                 return;
9206
9207         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9208         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9209                 LC_CURRENT_DATA_RATE_SHIFT;
9210         if (mask & DRM_PCIE_SPEED_80) {
9211                 if (current_data_rate == 2) {
9212                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9213                         return;
9214                 }
9215                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9216         } else if (mask & DRM_PCIE_SPEED_50) {
9217                 if (current_data_rate == 1) {
9218                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9219                         return;
9220                 }
9221                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9222         }
9223
9224         bridge_pos = pci_get_pciecap_ptr(root->dev);
9225         if (!bridge_pos)
9226                 return;
9227
9228         gpu_pos = pci_get_pciecap_ptr(rdev->pdev->dev);
9229         if (!gpu_pos)
9230                 return;
9231
9232         if (mask & DRM_PCIE_SPEED_80) {
9233                 /* re-try equalization if gen3 is not already enabled */
9234                 if (current_data_rate != 2) {
9235                         u16 bridge_cfg, gpu_cfg;
9236                         u16 bridge_cfg2, gpu_cfg2;
9237                         u32 max_lw, current_lw, tmp;
9238
9239                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9240                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9241
9242                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9243                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9244
9245                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9246                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9247
9248                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9249                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9250                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9251
9252                         if (current_lw < max_lw) {
9253                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9254                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9255                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9256                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9257                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9258                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9259                                 }
9260                         }
9261
9262                         for (i = 0; i < 10; i++) {
9263                                 /* check status */
9264                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9265                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9266                                         break;
9267
9268                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9269                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9270
9271                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9272                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9273
9274                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9275                                 tmp |= LC_SET_QUIESCE;
9276                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9277
9278                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9279                                 tmp |= LC_REDO_EQ;
9280                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9281
9282                                 mdelay(100);
9283
9284                                 /* linkctl */
9285                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9286                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9287                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9288                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9289
9290                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9291                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9292                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9293                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9294
9295                                 /* linkctl2 */
9296                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9297                                 tmp16 &= ~((1 << 4) | (7 << 9));
9298                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9299                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9300
9301                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9302                                 tmp16 &= ~((1 << 4) | (7 << 9));
9303                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9304                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9305
9306                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9307                                 tmp &= ~LC_SET_QUIESCE;
9308                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9309                         }
9310                 }
9311         }
9312
9313         /* set the link speed */
9314         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9315         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9316         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9317
9318         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9319         tmp16 &= ~0xf;
9320         if (mask & DRM_PCIE_SPEED_80)
9321                 tmp16 |= 3; /* gen3 */
9322         else if (mask & DRM_PCIE_SPEED_50)
9323                 tmp16 |= 2; /* gen2 */
9324         else
9325                 tmp16 |= 1; /* gen1 */
9326         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9327
9328         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9329         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9330         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9331
9332         for (i = 0; i < rdev->usec_timeout; i++) {
9333                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9334                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9335                         break;
9336                 udelay(1);
9337         }
9338 }
9339
9340 static void cik_program_aspm(struct radeon_device *rdev)
9341 {
9342         u32 data, orig;
9343         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9344         bool disable_clkreq = false;
9345
9346         if (radeon_aspm == 0)
9347                 return;
9348
9349         /* XXX double check IGPs */
9350         if (rdev->flags & RADEON_IS_IGP)
9351                 return;
9352
9353         if (!(rdev->flags & RADEON_IS_PCIE))
9354                 return;
9355
9356         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9357         data &= ~LC_XMIT_N_FTS_MASK;
9358         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9359         if (orig != data)
9360                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9361
9362         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9363         data |= LC_GO_TO_RECOVERY;
9364         if (orig != data)
9365                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9366
9367         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9368         data |= P_IGNORE_EDB_ERR;
9369         if (orig != data)
9370                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9371
9372         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9373         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9374         data |= LC_PMI_TO_L1_DIS;
9375         if (!disable_l0s)
9376                 data |= LC_L0S_INACTIVITY(7);
9377
9378         if (!disable_l1) {
9379                 data |= LC_L1_INACTIVITY(7);
9380                 data &= ~LC_PMI_TO_L1_DIS;
9381                 if (orig != data)
9382                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9383
9384                 if (!disable_plloff_in_l1) {
9385                         bool clk_req_support;
9386
9387                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9388                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9389                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9390                         if (orig != data)
9391                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9392
9393                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9394                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9395                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9396                         if (orig != data)
9397                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9398
9399                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9400                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9401                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9402                         if (orig != data)
9403                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9404
9405                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9406                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9407                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9408                         if (orig != data)
9409                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9410
9411                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9412                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9413                         data |= LC_DYN_LANES_PWR_STATE(3);
9414                         if (orig != data)
9415                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9416
9417                         if (!disable_clkreq) {
9418 #ifdef zMN_TODO
9419                                 struct pci_dev *root = rdev->pdev->bus->self;
9420                                 u32 lnkcap;
9421
9422                                 clk_req_support = false;
9423                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9424                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9425                                         clk_req_support = true;
9426 #else
9427                                 clk_req_support = false;
9428 #endif
9429                         } else {
9430                                 clk_req_support = false;
9431                         }
9432
9433                         if (clk_req_support) {
9434                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9435                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9436                                 if (orig != data)
9437                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9438
9439                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9440                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9441                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9442                                 if (orig != data)
9443                                         WREG32_SMC(THM_CLK_CNTL, data);
9444
9445                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9446                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9447                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9448                                 if (orig != data)
9449                                         WREG32_SMC(MISC_CLK_CTRL, data);
9450
9451                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9452                                 data &= ~BCLK_AS_XCLK;
9453                                 if (orig != data)
9454                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9455
9456                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9457                                 data &= ~FORCE_BIF_REFCLK_EN;
9458                                 if (orig != data)
9459                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9460
9461                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9462                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9463                                 data |= MPLL_CLKOUT_SEL(4);
9464                                 if (orig != data)
9465                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9466                         }
9467                 }
9468         } else {
9469                 if (orig != data)
9470                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9471         }
9472
9473         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9474         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9475         if (orig != data)
9476                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9477
9478         if (!disable_l0s) {
9479                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9480                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9481                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9482                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9483                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9484                                 data &= ~LC_L0S_INACTIVITY_MASK;
9485                                 if (orig != data)
9486                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9487                         }
9488                 }
9489         }
9490 }