sys/dev/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/module.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include <drm/radeon_drm.h>
  30 #include "nid.h"
  31 #include "atom.h"
  32 #include "ni_reg.h"
  33 #include "cayman_blit_shaders.h"
  34 #include "radeon_ucode.h"
  35 #include "clearstate_cayman.h"
  36
  37 static const u32 tn_rlc_save_restore_register_list[] =
  38 {
  39         0x98fc,
  40         0x98f0,
  41         0x9834,
  42         0x9838,
  43         0x9870,
  44         0x9874,
  45         0x8a14,
  46         0x8b24,
  47         0x8bcc,
  48         0x8b10,
  49         0x8c30,
  50         0x8d00,
  51         0x8d04,
  52         0x8c00,
  53         0x8c04,
  54         0x8c10,
  55         0x8c14,
  56         0x8d8c,
  57         0x8cf0,
  58         0x8e38,
  59         0x9508,
  60         0x9688,
  61         0x9608,
  62         0x960c,
  63         0x9610,
  64         0x9614,
  65         0x88c4,
  66         0x8978,
  67         0x88d4,
  68         0x900c,
  69         0x9100,
  70         0x913c,
  71         0x90e8,
  72         0x9354,
  73         0xa008,
  74         0x98f8,
  75         0x9148,
  76         0x914c,
  77         0x3f94,
  78         0x98f4,
  79         0x9b7c,
  80         0x3f8c,
  81         0x8950,
  82         0x8954,
  83         0x8a18,
  84         0x8b28,
  85         0x9144,
  86         0x3f90,
  87         0x915c,
  88         0x9160,
  89         0x9178,
  90         0x917c,
  91         0x9180,
  92         0x918c,
  93         0x9190,
  94         0x9194,
  95         0x9198,
  96         0x919c,
  97         0x91a8,
  98         0x91ac,
  99         0x91b0,
 100         0x91b4,
 101         0x91b8,
 102         0x91c4,
 103         0x91c8,
 104         0x91cc,
 105         0x91d0,
 106         0x91d4,
 107         0x91e0,
 108         0x91e4,
 109         0x91ec,
 110         0x91f0,
 111         0x91f4,
 112         0x9200,
 113         0x9204,
 114         0x929c,
 115         0x8030,
 116         0x9150,
 117         0x9a60,
 118         0x920c,
 119         0x9210,
 120         0x9228,
 121         0x922c,
 122         0x9244,
 123         0x9248,
 124         0x91e8,
 125         0x9294,
 126         0x9208,
 127         0x9224,
 128         0x9240,
 129         0x9220,
 130         0x923c,
 131         0x9258,
 132         0x9744,
 133         0xa200,
 134         0xa204,
 135         0xa208,
 136         0xa20c,
 137         0x8d58,
 138         0x9030,
 139         0x9034,
 140         0x9038,
 141         0x903c,
 142         0x9040,
 143         0x9654,
 144         0x897c,
 145         0xa210,
 146         0xa214,
 147         0x9868,
 148         0xa02c,
 149         0x9664,
 150         0x9698,
 151         0x949c,
 152         0x8e10,
 153         0x8e18,
 154         0x8c50,
 155         0x8c58,
 156         0x8c60,
 157         0x8c68,
 158         0x89b4,
 159         0x9830,
 160         0x802c,
 161 };
 162
 163 /* Firmware Names */
 164 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 165 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 166 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 167 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 168 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 169 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 170 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 171 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 172 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 173 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 174 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 175 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 176 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 177 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 178 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 179 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 180 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 181 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 182 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 183 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 184 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 185
 186
 187 static const u32 cayman_golden_registers2[] =
 188 {
 189         0x3e5c, 0xffffffff, 0x00000000,
 190         0x3e48, 0xffffffff, 0x00000000,
 191         0x3e4c, 0xffffffff, 0x00000000,
 192         0x3e64, 0xffffffff, 0x00000000,
 193         0x3e50, 0xffffffff, 0x00000000,
 194         0x3e60, 0xffffffff, 0x00000000
 195 };
 196
 197 static const u32 cayman_golden_registers[] =
 198 {
 199         0x5eb4, 0xffffffff, 0x00000002,
 200         0x5e78, 0x8f311ff1, 0x001000f0,
 201         0x3f90, 0xffff0000, 0xff000000,
 202         0x9148, 0xffff0000, 0xff000000,
 203         0x3f94, 0xffff0000, 0xff000000,
 204         0x914c, 0xffff0000, 0xff000000,
 205         0xc78, 0x00000080, 0x00000080,
 206         0xbd4, 0x70073777, 0x00011003,
 207         0xd02c, 0xbfffff1f, 0x08421000,
 208         0xd0b8, 0x73773777, 0x02011003,
 209         0x5bc0, 0x00200000, 0x50100000,
 210         0x98f8, 0x33773777, 0x02011003,
 211         0x98fc, 0xffffffff, 0x76541032,
 212         0x7030, 0x31000311, 0x00000011,
 213         0x2f48, 0x33773777, 0x42010001,
 214         0x6b28, 0x00000010, 0x00000012,
 215         0x7728, 0x00000010, 0x00000012,
 216         0x10328, 0x00000010, 0x00000012,
 217         0x10f28, 0x00000010, 0x00000012,
 218         0x11b28, 0x00000010, 0x00000012,
 219         0x12728, 0x00000010, 0x00000012,
 220         0x240c, 0x000007ff, 0x00000000,
 221         0x8a14, 0xf000001f, 0x00000007,
 222         0x8b24, 0x3fff3fff, 0x00ff0fff,
 223         0x8b10, 0x0000ff0f, 0x00000000,
 224         0x28a4c, 0x07ffffff, 0x06000000,
 225         0x10c, 0x00000001, 0x00010003,
 226         0xa02c, 0xffffffff, 0x0000009b,
 227         0x913c, 0x0000010f, 0x01000100,
 228         0x8c04, 0xf8ff00ff, 0x40600060,
 229         0x28350, 0x00000f01, 0x00000000,
 230         0x9508, 0x3700001f, 0x00000002,
 231         0x960c, 0xffffffff, 0x54763210,
 232         0x88c4, 0x001f3ae3, 0x00000082,
 233         0x88d0, 0xffffffff, 0x0f40df40,
 234         0x88d4, 0x0000001f, 0x00000010,
 235         0x8974, 0xffffffff, 0x00000000
 236 };
 237
 238 static const u32 dvst_golden_registers2[] =
 239 {
 240         0x8f8, 0xffffffff, 0,
 241         0x8fc, 0x00380000, 0,
 242         0x8f8, 0xffffffff, 1,
 243         0x8fc, 0x0e000000, 0
 244 };
 245
 246 static const u32 dvst_golden_registers[] =
 247 {
 248         0x690, 0x3fff3fff, 0x20c00033,
 249         0x918c, 0x0fff0fff, 0x00010006,
 250         0x91a8, 0x0fff0fff, 0x00010006,
 251         0x9150, 0xffffdfff, 0x6e944040,
 252         0x917c, 0x0fff0fff, 0x00030002,
 253         0x9198, 0x0fff0fff, 0x00030002,
 254         0x915c, 0x0fff0fff, 0x00010000,
 255         0x3f90, 0xffff0001, 0xff000000,
 256         0x9178, 0x0fff0fff, 0x00070000,
 257         0x9194, 0x0fff0fff, 0x00070000,
 258         0x9148, 0xffff0001, 0xff000000,
 259         0x9190, 0x0fff0fff, 0x00090008,
 260         0x91ac, 0x0fff0fff, 0x00090008,
 261         0x3f94, 0xffff0000, 0xff000000,
 262         0x914c, 0xffff0000, 0xff000000,
 263         0x929c, 0x00000fff, 0x00000001,
 264         0x55e4, 0xff607fff, 0xfc000100,
 265         0x8a18, 0xff000fff, 0x00000100,
 266         0x8b28, 0xff000fff, 0x00000100,
 267         0x9144, 0xfffc0fff, 0x00000100,
 268         0x6ed8, 0x00010101, 0x00010000,
 269         0x9830, 0xffffffff, 0x00000000,
 270         0x9834, 0xf00fffff, 0x00000400,
 271         0x9838, 0xfffffffe, 0x00000000,
 272         0xd0c0, 0xff000fff, 0x00000100,
 273         0xd02c, 0xbfffff1f, 0x08421000,
 274         0xd0b8, 0x73773777, 0x12010001,
 275         0x5bb0, 0x000000f0, 0x00000070,
 276         0x98f8, 0x73773777, 0x12010001,
 277         0x98fc, 0xffffffff, 0x00000010,
 278         0x9b7c, 0x00ff0000, 0x00fc0000,
 279         0x8030, 0x00001f0f, 0x0000100a,
 280         0x2f48, 0x73773777, 0x12010001,
 281         0x2408, 0x00030000, 0x000c007f,
 282         0x8a14, 0xf000003f, 0x00000007,
 283         0x8b24, 0x3fff3fff, 0x00ff0fff,
 284         0x8b10, 0x0000ff0f, 0x00000000,
 285         0x28a4c, 0x07ffffff, 0x06000000,
 286         0x4d8, 0x00000fff, 0x00000100,
 287         0xa008, 0xffffffff, 0x00010000,
 288         0x913c, 0xffff03ff, 0x01000100,
 289         0x8c00, 0x000000ff, 0x00000003,
 290         0x8c04, 0xf8ff00ff, 0x40600060,
 291         0x8cf0, 0x1fff1fff, 0x08e00410,
 292         0x28350, 0x00000f01, 0x00000000,
 293         0x9508, 0xf700071f, 0x00000002,
 294         0x960c, 0xffffffff, 0x54763210,
 295         0x20ef8, 0x01ff01ff, 0x00000002,
 296         0x20e98, 0xfffffbff, 0x00200000,
 297         0x2015c, 0xffffffff, 0x00000f40,
 298         0x88c4, 0x001f3ae3, 0x00000082,
 299         0x8978, 0x3fffffff, 0x04050140,
 300         0x88d4, 0x0000001f, 0x00000010,
 301         0x8974, 0xffffffff, 0x00000000
 302 };
 303
 304 static const u32 scrapper_golden_registers[] =
 305 {
 306         0x690, 0x3fff3fff, 0x20c00033,
 307         0x918c, 0x0fff0fff, 0x00010006,
 308         0x918c, 0x0fff0fff, 0x00010006,
 309         0x91a8, 0x0fff0fff, 0x00010006,
 310         0x91a8, 0x0fff0fff, 0x00010006,
 311         0x9150, 0xffffdfff, 0x6e944040,
 312         0x9150, 0xffffdfff, 0x6e944040,
 313         0x917c, 0x0fff0fff, 0x00030002,
 314         0x917c, 0x0fff0fff, 0x00030002,
 315         0x9198, 0x0fff0fff, 0x00030002,
 316         0x9198, 0x0fff0fff, 0x00030002,
 317         0x915c, 0x0fff0fff, 0x00010000,
 318         0x915c, 0x0fff0fff, 0x00010000,
 319         0x3f90, 0xffff0001, 0xff000000,
 320         0x3f90, 0xffff0001, 0xff000000,
 321         0x9178, 0x0fff0fff, 0x00070000,
 322         0x9178, 0x0fff0fff, 0x00070000,
 323         0x9194, 0x0fff0fff, 0x00070000,
 324         0x9194, 0x0fff0fff, 0x00070000,
 325         0x9148, 0xffff0001, 0xff000000,
 326         0x9148, 0xffff0001, 0xff000000,
 327         0x9190, 0x0fff0fff, 0x00090008,
 328         0x9190, 0x0fff0fff, 0x00090008,
 329         0x91ac, 0x0fff0fff, 0x00090008,
 330         0x91ac, 0x0fff0fff, 0x00090008,
 331         0x3f94, 0xffff0000, 0xff000000,
 332         0x3f94, 0xffff0000, 0xff000000,
 333         0x914c, 0xffff0000, 0xff000000,
 334         0x914c, 0xffff0000, 0xff000000,
 335         0x929c, 0x00000fff, 0x00000001,
 336         0x929c, 0x00000fff, 0x00000001,
 337         0x55e4, 0xff607fff, 0xfc000100,
 338         0x8a18, 0xff000fff, 0x00000100,
 339         0x8a18, 0xff000fff, 0x00000100,
 340         0x8b28, 0xff000fff, 0x00000100,
 341         0x8b28, 0xff000fff, 0x00000100,
 342         0x9144, 0xfffc0fff, 0x00000100,
 343         0x9144, 0xfffc0fff, 0x00000100,
 344         0x6ed8, 0x00010101, 0x00010000,
 345         0x9830, 0xffffffff, 0x00000000,
 346         0x9830, 0xffffffff, 0x00000000,
 347         0x9834, 0xf00fffff, 0x00000400,
 348         0x9834, 0xf00fffff, 0x00000400,
 349         0x9838, 0xfffffffe, 0x00000000,
 350         0x9838, 0xfffffffe, 0x00000000,
 351         0xd0c0, 0xff000fff, 0x00000100,
 352         0xd02c, 0xbfffff1f, 0x08421000,
 353         0xd02c, 0xbfffff1f, 0x08421000,
 354         0xd0b8, 0x73773777, 0x12010001,
 355         0xd0b8, 0x73773777, 0x12010001,
 356         0x5bb0, 0x000000f0, 0x00000070,
 357         0x98f8, 0x73773777, 0x12010001,
 358         0x98f8, 0x73773777, 0x12010001,
 359         0x98fc, 0xffffffff, 0x00000010,
 360         0x98fc, 0xffffffff, 0x00000010,
 361         0x9b7c, 0x00ff0000, 0x00fc0000,
 362         0x9b7c, 0x00ff0000, 0x00fc0000,
 363         0x8030, 0x00001f0f, 0x0000100a,
 364         0x8030, 0x00001f0f, 0x0000100a,
 365         0x2f48, 0x73773777, 0x12010001,
 366         0x2f48, 0x73773777, 0x12010001,
 367         0x2408, 0x00030000, 0x000c007f,
 368         0x8a14, 0xf000003f, 0x00000007,
 369         0x8a14, 0xf000003f, 0x00000007,
 370         0x8b24, 0x3fff3fff, 0x00ff0fff,
 371         0x8b24, 0x3fff3fff, 0x00ff0fff,
 372         0x8b10, 0x0000ff0f, 0x00000000,
 373         0x8b10, 0x0000ff0f, 0x00000000,
 374         0x28a4c, 0x07ffffff, 0x06000000,
 375         0x28a4c, 0x07ffffff, 0x06000000,
 376         0x4d8, 0x00000fff, 0x00000100,
 377         0x4d8, 0x00000fff, 0x00000100,
 378         0xa008, 0xffffffff, 0x00010000,
 379         0xa008, 0xffffffff, 0x00010000,
 380         0x913c, 0xffff03ff, 0x01000100,
 381         0x913c, 0xffff03ff, 0x01000100,
 382         0x90e8, 0x001fffff, 0x010400c0,
 383         0x8c00, 0x000000ff, 0x00000003,
 384         0x8c00, 0x000000ff, 0x00000003,
 385         0x8c04, 0xf8ff00ff, 0x40600060,
 386         0x8c04, 0xf8ff00ff, 0x40600060,
 387         0x8c30, 0x0000000f, 0x00040005,
 388         0x8cf0, 0x1fff1fff, 0x08e00410,
 389         0x8cf0, 0x1fff1fff, 0x08e00410,
 390         0x900c, 0x00ffffff, 0x0017071f,
 391         0x28350, 0x00000f01, 0x00000000,
 392         0x28350, 0x00000f01, 0x00000000,
 393         0x9508, 0xf700071f, 0x00000002,
 394         0x9508, 0xf700071f, 0x00000002,
 395         0x9688, 0x00300000, 0x0017000f,
 396         0x960c, 0xffffffff, 0x54763210,
 397         0x960c, 0xffffffff, 0x54763210,
 398         0x20ef8, 0x01ff01ff, 0x00000002,
 399         0x20e98, 0xfffffbff, 0x00200000,
 400         0x2015c, 0xffffffff, 0x00000f40,
 401         0x88c4, 0x001f3ae3, 0x00000082,
 402         0x88c4, 0x001f3ae3, 0x00000082,
 403         0x8978, 0x3fffffff, 0x04050140,
 404         0x8978, 0x3fffffff, 0x04050140,
 405         0x88d4, 0x0000001f, 0x00000010,
 406         0x88d4, 0x0000001f, 0x00000010,
 407         0x8974, 0xffffffff, 0x00000000,
 408         0x8974, 0xffffffff, 0x00000000
 409 };
 410
 411 static void ni_init_golden_registers(struct radeon_device *rdev)
 412 {
 413         switch (rdev->family) {
 414         case CHIP_CAYMAN:
 415                 radeon_program_register_sequence(rdev,
 416                                                  cayman_golden_registers,
 417                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 418                 radeon_program_register_sequence(rdev,
 419                                                  cayman_golden_registers2,
 420                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 421                 break;
 422         case CHIP_ARUBA:
 423                 if ((rdev->pdev->device == 0x9900) ||
 424                     (rdev->pdev->device == 0x9901) ||
 425                     (rdev->pdev->device == 0x9903) ||
 426                     (rdev->pdev->device == 0x9904) ||
 427                     (rdev->pdev->device == 0x9905) ||
 428                     (rdev->pdev->device == 0x9906) ||
 429                     (rdev->pdev->device == 0x9907) ||
 430                     (rdev->pdev->device == 0x9908) ||
 431                     (rdev->pdev->device == 0x9909) ||
 432                     (rdev->pdev->device == 0x990A) ||
 433                     (rdev->pdev->device == 0x990B) ||
 434                     (rdev->pdev->device == 0x990C) ||
 435                     (rdev->pdev->device == 0x990D) ||
 436                     (rdev->pdev->device == 0x990E) ||
 437                     (rdev->pdev->device == 0x990F) ||
 438                     (rdev->pdev->device == 0x9910) ||
 439                     (rdev->pdev->device == 0x9913) ||
 440                     (rdev->pdev->device == 0x9917) ||
 441                     (rdev->pdev->device == 0x9918)) {
 442                         radeon_program_register_sequence(rdev,
 443                                                          dvst_golden_registers,
 444                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 445                         radeon_program_register_sequence(rdev,
 446                                                          dvst_golden_registers2,
 447                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 448                 } else {
 449                         radeon_program_register_sequence(rdev,
 450                                                          scrapper_golden_registers,
 451                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 452                         radeon_program_register_sequence(rdev,
 453                                                          dvst_golden_registers2,
 454                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 455                 }
 456                 break;
 457         default:
 458                 break;
 459         }
 460 }
 461
 462 #define BTC_IO_MC_REGS_SIZE 29
 463
 464 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 465         {0x00000077, 0xff010100},
 466         {0x00000078, 0x00000000},
 467         {0x00000079, 0x00001434},
 468         {0x0000007a, 0xcc08ec08},
 469         {0x0000007b, 0x00040000},
 470         {0x0000007c, 0x000080c0},
 471         {0x0000007d, 0x09000000},
 472         {0x0000007e, 0x00210404},
 473         {0x00000081, 0x08a8e800},
 474         {0x00000082, 0x00030444},
 475         {0x00000083, 0x00000000},
 476         {0x00000085, 0x00000001},
 477         {0x00000086, 0x00000002},
 478         {0x00000087, 0x48490000},
 479         {0x00000088, 0x20244647},
 480         {0x00000089, 0x00000005},
 481         {0x0000008b, 0x66030000},
 482         {0x0000008c, 0x00006603},
 483         {0x0000008d, 0x00000100},
 484         {0x0000008f, 0x00001c0a},
 485         {0x00000090, 0xff000001},
 486         {0x00000094, 0x00101101},
 487         {0x00000095, 0x00000fff},
 488         {0x00000096, 0x00116fff},
 489         {0x00000097, 0x60010000},
 490         {0x00000098, 0x10010000},
 491         {0x00000099, 0x00006000},
 492         {0x0000009a, 0x00001000},
 493         {0x0000009f, 0x00946a00}
 494 };
 495
 496 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 497         {0x00000077, 0xff010100},
 498         {0x00000078, 0x00000000},
 499         {0x00000079, 0x00001434},
 500         {0x0000007a, 0xcc08ec08},
 501         {0x0000007b, 0x00040000},
 502         {0x0000007c, 0x000080c0},
 503         {0x0000007d, 0x09000000},
 504         {0x0000007e, 0x00210404},
 505         {0x00000081, 0x08a8e800},
 506         {0x00000082, 0x00030444},
 507         {0x00000083, 0x00000000},
 508         {0x00000085, 0x00000001},
 509         {0x00000086, 0x00000002},
 510         {0x00000087, 0x48490000},
 511         {0x00000088, 0x20244647},
 512         {0x00000089, 0x00000005},
 513         {0x0000008b, 0x66030000},
 514         {0x0000008c, 0x00006603},
 515         {0x0000008d, 0x00000100},
 516         {0x0000008f, 0x00001c0a},
 517         {0x00000090, 0xff000001},
 518         {0x00000094, 0x00101101},
 519         {0x00000095, 0x00000fff},
 520         {0x00000096, 0x00116fff},
 521         {0x00000097, 0x60010000},
 522         {0x00000098, 0x10010000},
 523         {0x00000099, 0x00006000},
 524         {0x0000009a, 0x00001000},
 525         {0x0000009f, 0x00936a00}
 526 };
 527
 528 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 529         {0x00000077, 0xff010100},
 530         {0x00000078, 0x00000000},
 531         {0x00000079, 0x00001434},
 532         {0x0000007a, 0xcc08ec08},
 533         {0x0000007b, 0x00040000},
 534         {0x0000007c, 0x000080c0},
 535         {0x0000007d, 0x09000000},
 536         {0x0000007e, 0x00210404},
 537         {0x00000081, 0x08a8e800},
 538         {0x00000082, 0x00030444},
 539         {0x00000083, 0x00000000},
 540         {0x00000085, 0x00000001},
 541         {0x00000086, 0x00000002},
 542         {0x00000087, 0x48490000},
 543         {0x00000088, 0x20244647},
 544         {0x00000089, 0x00000005},
 545         {0x0000008b, 0x66030000},
 546         {0x0000008c, 0x00006603},
 547         {0x0000008d, 0x00000100},
 548         {0x0000008f, 0x00001c0a},
 549         {0x00000090, 0xff000001},
 550         {0x00000094, 0x00101101},
 551         {0x00000095, 0x00000fff},
 552         {0x00000096, 0x00116fff},
 553         {0x00000097, 0x60010000},
 554         {0x00000098, 0x10010000},
 555         {0x00000099, 0x00006000},
 556         {0x0000009a, 0x00001000},
 557         {0x0000009f, 0x00916a00}
 558 };
 559
 560 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 561         {0x00000077, 0xff010100},
 562         {0x00000078, 0x00000000},
 563         {0x00000079, 0x00001434},
 564         {0x0000007a, 0xcc08ec08},
 565         {0x0000007b, 0x00040000},
 566         {0x0000007c, 0x000080c0},
 567         {0x0000007d, 0x09000000},
 568         {0x0000007e, 0x00210404},
 569         {0x00000081, 0x08a8e800},
 570         {0x00000082, 0x00030444},
 571         {0x00000083, 0x00000000},
 572         {0x00000085, 0x00000001},
 573         {0x00000086, 0x00000002},
 574         {0x00000087, 0x48490000},
 575         {0x00000088, 0x20244647},
 576         {0x00000089, 0x00000005},
 577         {0x0000008b, 0x66030000},
 578         {0x0000008c, 0x00006603},
 579         {0x0000008d, 0x00000100},
 580         {0x0000008f, 0x00001c0a},
 581         {0x00000090, 0xff000001},
 582         {0x00000094, 0x00101101},
 583         {0x00000095, 0x00000fff},
 584         {0x00000096, 0x00116fff},
 585         {0x00000097, 0x60010000},
 586         {0x00000098, 0x10010000},
 587         {0x00000099, 0x00006000},
 588         {0x0000009a, 0x00001000},
 589         {0x0000009f, 0x00976b00}
 590 };
 591
 592 int ni_mc_load_microcode(struct radeon_device *rdev)
 593 {
 594         const __be32 *fw_data;
 595         u32 mem_type, running, blackout = 0;
 596         u32 *io_mc_regs;
 597         int i, ucode_size, regs_size;
 598
 599         if (!rdev->mc_fw)
 600                 return -EINVAL;
 601
 602         switch (rdev->family) {
 603         case CHIP_BARTS:
 604                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 605                 ucode_size = BTC_MC_UCODE_SIZE;
 606                 regs_size = BTC_IO_MC_REGS_SIZE;
 607                 break;
 608         case CHIP_TURKS:
 609                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 610                 ucode_size = BTC_MC_UCODE_SIZE;
 611                 regs_size = BTC_IO_MC_REGS_SIZE;
 612                 break;
 613         case CHIP_CAICOS:
 614         default:
 615                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 616                 ucode_size = BTC_MC_UCODE_SIZE;
 617                 regs_size = BTC_IO_MC_REGS_SIZE;
 618                 break;
 619         case CHIP_CAYMAN:
 620                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 621                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 622                 regs_size = BTC_IO_MC_REGS_SIZE;
 623                 break;
 624         }
 625
 626         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 627         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 628
 629         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 630                 if (running) {
 631                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 632                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 633                 }
 634
 635                 /* reset the engine and set to writable */
 636                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 637                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 638
 639                 /* load mc io regs */
 640                 for (i = 0; i < regs_size; i++) {
 641                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 642                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 643                 }
 644                 /* load the MC ucode */
 645                 fw_data = (const __be32 *)rdev->mc_fw->data;
 646                 for (i = 0; i < ucode_size; i++)
 647                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 648
 649                 /* put the engine back into the active state */
 650                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 651                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 652                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 653
 654                 /* wait for training to complete */
 655                 for (i = 0; i < rdev->usec_timeout; i++) {
 656                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 657                                 break;
 658                         udelay(1);
 659                 }
 660
 661                 if (running)
 662                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 663         }
 664
 665         return 0;
 666 }
 667
 668 int ni_init_microcode(struct radeon_device *rdev)
 669 {
 670         const char *chip_name;
 671         const char *rlc_chip_name;
 672         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 673         size_t smc_req_size = 0;
 674         char fw_name[30];
 675         int err;
 676
 677         DRM_DEBUG("\n");
 678
 679         switch (rdev->family) {
 680         case CHIP_BARTS:
 681                 chip_name = "BARTS";
 682                 rlc_chip_name = "BTC";
 683                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 684                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 685                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 686                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 687                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 688                 break;
 689         case CHIP_TURKS:
 690                 chip_name = "TURKS";
 691                 rlc_chip_name = "BTC";
 692                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 693                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 694                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 695                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 696                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 697                 break;
 698         case CHIP_CAICOS:
 699                 chip_name = "CAICOS";
 700                 rlc_chip_name = "BTC";
 701                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 702                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 703                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 704                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 705                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 706                 break;
 707         case CHIP_CAYMAN:
 708                 chip_name = "CAYMAN";
 709                 rlc_chip_name = "CAYMAN";
 710                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 711                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 712                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 713                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 714                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 715                 break;
 716         case CHIP_ARUBA:
 717                 chip_name = "ARUBA";
 718                 rlc_chip_name = "ARUBA";
 719                 /* pfp/me same size as CAYMAN */
 720                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 721                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 722                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 723                 mc_req_size = 0;
 724                 break;
 725         default: BUG();
 726         }
 727
 728         DRM_INFO("Loading %s Microcode\n", chip_name);
 729
 730         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
 731         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 732         if (err)
 733                 goto out;
 734         if (rdev->pfp_fw->datasize != pfp_req_size) {
 735                 printk(KERN_ERR
 736                        "ni_pfp: Bogus length %zu in firmware \"%s\"\n",
 737                        rdev->pfp_fw->datasize, fw_name);
 738                 err = -EINVAL;
 739                 goto out;
 740         }
 741
 742         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
 743         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 744         if (err)
 745                 goto out;
 746         if (rdev->me_fw->datasize != me_req_size) {
 747                 printk(KERN_ERR
 748                        "ni_me: Bogus length %zu in firmware \"%s\"\n",
 749                        rdev->me_fw->datasize, fw_name);
 750                 err = -EINVAL;
 751         }
 752
 753         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc",
 754                   rlc_chip_name);
 755         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 756         if (err)
 757                 goto out;
 758         if (rdev->rlc_fw->datasize != rlc_req_size) {
 759                 printk(KERN_ERR
 760                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 761                        rdev->rlc_fw->datasize, fw_name);
 762                 err = -EINVAL;
 763         }
 764
 765         /* no MC ucode on TN */
 766         if (!(rdev->flags & RADEON_IS_IGP)) {
 767                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
 768                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 769                 if (err)
 770                         goto out;
 771                 if (rdev->mc_fw->datasize != mc_req_size) {
 772                         printk(KERN_ERR
 773                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 774                                rdev->mc_fw->datasize, fw_name);
 775                         err = -EINVAL;
 776                 }
 777         }
 778
 779         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 780                 ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_smc", chip_name);
 781                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 782                 if (err) {
 783                         printk(KERN_ERR
 784                                "smc: error loading firmware \"%s\"\n",
 785                                fw_name);
 786                         release_firmware(rdev->smc_fw);
 787                         rdev->smc_fw = NULL;
 788                         err = 0;
 789                 } else if (rdev->smc_fw->datasize != smc_req_size) {
 790                         printk(KERN_ERR
 791                                "ni_smc: Bogus length %zu in firmware \"%s\"\n",
 792                                rdev->smc_fw->datasize, fw_name);
 793                         err = -EINVAL;
 794                 }
 795         }
 796
 797 out:
 798         if (err) {
 799                 if (err != -EINVAL)
 800                         printk(KERN_ERR
 801                                "ni_cp: Failed to load firmware \"%s\"\n",
 802                                fw_name);
 803                 release_firmware(rdev->pfp_fw);
 804                 rdev->pfp_fw = NULL;
 805                 release_firmware(rdev->me_fw);
 806                 rdev->me_fw = NULL;
 807                 release_firmware(rdev->rlc_fw);
 808                 rdev->rlc_fw = NULL;
 809                 release_firmware(rdev->mc_fw);
 810                 rdev->mc_fw = NULL;
 811                 release_firmware(rdev->smc_fw);
 812                 rdev->smc_fw = NULL;
 813         }
 814         return err;
 815 }
 816
 817 /**
 818  * ni_fini_microcode - drop the firmwares image references
 819  *
 820  * @rdev: radeon_device pointer
 821  *
 822  * Drop the pfp, me, mc and rlc firmwares image references.
 823  * Called at driver shutdown.
 824  */
 825 void ni_fini_microcode(struct radeon_device *rdev)
 826 {
 827         release_firmware(rdev->pfp_fw);
 828         rdev->pfp_fw = NULL;
 829         release_firmware(rdev->me_fw);
 830         rdev->me_fw = NULL;
 831         release_firmware(rdev->rlc_fw);
 832         rdev->rlc_fw = NULL;
 833         release_firmware(rdev->mc_fw);
 834         rdev->mc_fw = NULL;
 835         release_firmware(rdev->smc_fw);
 836         rdev->smc_fw = NULL;
 837 }
 838
 839 int tn_get_temp(struct radeon_device *rdev)
 840 {
 841         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 842         int actual_temp = (temp / 8) - 49;
 843
 844         return actual_temp * 1000;
 845 }
 846
 847 /*
 848  * Core functions
 849  */
 850 static void cayman_gpu_init(struct radeon_device *rdev)
 851 {
 852         u32 gb_addr_config = 0;
 853         u32 mc_shared_chmap, mc_arb_ramcfg;
 854         u32 cgts_tcc_disable;
 855         u32 sx_debug_1;
 856         u32 smx_dc_ctl0;
 857         u32 cgts_sm_ctrl_reg;
 858         u32 hdp_host_path_cntl;
 859         u32 tmp;
 860         u32 disabled_rb_mask;
 861         int i, j;
 862
 863         switch (rdev->family) {
 864         case CHIP_CAYMAN:
 865                 rdev->config.cayman.max_shader_engines = 2;
 866                 rdev->config.cayman.max_pipes_per_simd = 4;
 867                 rdev->config.cayman.max_tile_pipes = 8;
 868                 rdev->config.cayman.max_simds_per_se = 12;
 869                 rdev->config.cayman.max_backends_per_se = 4;
 870                 rdev->config.cayman.max_texture_channel_caches = 8;
 871                 rdev->config.cayman.max_gprs = 256;
 872                 rdev->config.cayman.max_threads = 256;
 873                 rdev->config.cayman.max_gs_threads = 32;
 874                 rdev->config.cayman.max_stack_entries = 512;
 875                 rdev->config.cayman.sx_num_of_sets = 8;
 876                 rdev->config.cayman.sx_max_export_size = 256;
 877                 rdev->config.cayman.sx_max_export_pos_size = 64;
 878                 rdev->config.cayman.sx_max_export_smx_size = 192;
 879                 rdev->config.cayman.max_hw_contexts = 8;
 880                 rdev->config.cayman.sq_num_cf_insts = 2;
 881
 882                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 883                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 884                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 885                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 886                 break;
 887         case CHIP_ARUBA:
 888         default:
 889                 rdev->config.cayman.max_shader_engines = 1;
 890                 rdev->config.cayman.max_pipes_per_simd = 4;
 891                 rdev->config.cayman.max_tile_pipes = 2;
 892                 if ((rdev->pdev->device == 0x9900) ||
 893                     (rdev->pdev->device == 0x9901) ||
 894                     (rdev->pdev->device == 0x9905) ||
 895                     (rdev->pdev->device == 0x9906) ||
 896                     (rdev->pdev->device == 0x9907) ||
 897                     (rdev->pdev->device == 0x9908) ||
 898                     (rdev->pdev->device == 0x9909) ||
 899                     (rdev->pdev->device == 0x990B) ||
 900                     (rdev->pdev->device == 0x990C) ||
 901                     (rdev->pdev->device == 0x990F) ||
 902                     (rdev->pdev->device == 0x9910) ||
 903                     (rdev->pdev->device == 0x9917) ||
 904                     (rdev->pdev->device == 0x9999) ||
 905                     (rdev->pdev->device == 0x999C)) {
 906                         rdev->config.cayman.max_simds_per_se = 6;
 907                         rdev->config.cayman.max_backends_per_se = 2;
 908                         rdev->config.cayman.max_hw_contexts = 8;
 909                         rdev->config.cayman.sx_max_export_size = 256;
 910                         rdev->config.cayman.sx_max_export_pos_size = 64;
 911                         rdev->config.cayman.sx_max_export_smx_size = 192;
 912                 } else if ((rdev->pdev->device == 0x9903) ||
 913                            (rdev->pdev->device == 0x9904) ||
 914                            (rdev->pdev->device == 0x990A) ||
 915                            (rdev->pdev->device == 0x990D) ||
 916                            (rdev->pdev->device == 0x990E) ||
 917                            (rdev->pdev->device == 0x9913) ||
 918                            (rdev->pdev->device == 0x9918) ||
 919                            (rdev->pdev->device == 0x999D)) {
 920                         rdev->config.cayman.max_simds_per_se = 4;
 921                         rdev->config.cayman.max_backends_per_se = 2;
 922                         rdev->config.cayman.max_hw_contexts = 8;
 923                         rdev->config.cayman.sx_max_export_size = 256;
 924                         rdev->config.cayman.sx_max_export_pos_size = 64;
 925                         rdev->config.cayman.sx_max_export_smx_size = 192;
 926                 } else if ((rdev->pdev->device == 0x9919) ||
 927                            (rdev->pdev->device == 0x9990) ||
 928                            (rdev->pdev->device == 0x9991) ||
 929                            (rdev->pdev->device == 0x9994) ||
 930                            (rdev->pdev->device == 0x9995) ||
 931                            (rdev->pdev->device == 0x9996) ||
 932                            (rdev->pdev->device == 0x999A) ||
 933                            (rdev->pdev->device == 0x99A0)) {
 934                         rdev->config.cayman.max_simds_per_se = 3;
 935                         rdev->config.cayman.max_backends_per_se = 1;
 936                         rdev->config.cayman.max_hw_contexts = 4;
 937                         rdev->config.cayman.sx_max_export_size = 128;
 938                         rdev->config.cayman.sx_max_export_pos_size = 32;
 939                         rdev->config.cayman.sx_max_export_smx_size = 96;
 940                 } else {
 941                         rdev->config.cayman.max_simds_per_se = 2;
 942                         rdev->config.cayman.max_backends_per_se = 1;
 943                         rdev->config.cayman.max_hw_contexts = 4;
 944                         rdev->config.cayman.sx_max_export_size = 128;
 945                         rdev->config.cayman.sx_max_export_pos_size = 32;
 946                         rdev->config.cayman.sx_max_export_smx_size = 96;
 947                 }
 948                 rdev->config.cayman.max_texture_channel_caches = 2;
 949                 rdev->config.cayman.max_gprs = 256;
 950                 rdev->config.cayman.max_threads = 256;
 951                 rdev->config.cayman.max_gs_threads = 32;
 952                 rdev->config.cayman.max_stack_entries = 512;
 953                 rdev->config.cayman.sx_num_of_sets = 8;
 954                 rdev->config.cayman.sq_num_cf_insts = 2;
 955
 956                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 957                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 958                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 959                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 960                 break;
 961         }
 962
 963         /* Initialize HDP */
 964         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 965                 WREG32((0x2c14 + j), 0x00000000);
 966                 WREG32((0x2c18 + j), 0x00000000);
 967                 WREG32((0x2c1c + j), 0x00000000);
 968                 WREG32((0x2c20 + j), 0x00000000);
 969                 WREG32((0x2c24 + j), 0x00000000);
 970         }
 971
 972         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 973
 974         evergreen_fix_pci_max_read_req_size(rdev);
 975
 976         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 977         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 978
 979         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 980         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 981         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 982                 rdev->config.cayman.mem_row_size_in_kb = 4;
 983         /* XXX use MC settings? */
 984         rdev->config.cayman.shader_engine_tile_size = 32;
 985         rdev->config.cayman.num_gpus = 1;
 986         rdev->config.cayman.multi_gpu_tile_size = 64;
 987
 988         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 989         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 990         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 991         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 992         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 993         rdev->config.cayman.num_shader_engines = tmp + 1;
 994         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 995         rdev->config.cayman.num_gpus = tmp + 1;
 996         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 997         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 998         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 999         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1000
1001
1002         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1003          * not have bank info, so create a custom tiling dword.
1004          * bits 3:0   num_pipes
1005          * bits 7:4   num_banks
1006          * bits 11:8  group_size
1007          * bits 15:12 row_size
1008          */
1009         rdev->config.cayman.tile_config = 0;
1010         switch (rdev->config.cayman.num_tile_pipes) {
1011         case 1:
1012         default:
1013                 rdev->config.cayman.tile_config |= (0 << 0);
1014                 break;
1015         case 2:
1016                 rdev->config.cayman.tile_config |= (1 << 0);
1017                 break;
1018         case 4:
1019                 rdev->config.cayman.tile_config |= (2 << 0);
1020                 break;
1021         case 8:
1022                 rdev->config.cayman.tile_config |= (3 << 0);
1023                 break;
1024         }
1025
1026         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1027         if (rdev->flags & RADEON_IS_IGP)
1028                 rdev->config.cayman.tile_config |= 1 << 4;
1029         else {
1030                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1031                 case 0: /* four banks */
1032                         rdev->config.cayman.tile_config |= 0 << 4;
1033                         break;
1034                 case 1: /* eight banks */
1035                         rdev->config.cayman.tile_config |= 1 << 4;
1036                         break;
1037                 case 2: /* sixteen banks */
1038                 default:
1039                         rdev->config.cayman.tile_config |= 2 << 4;
1040                         break;
1041                 }
1042         }
1043         rdev->config.cayman.tile_config |=
1044                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1045         rdev->config.cayman.tile_config |=
1046                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1047
1048         tmp = 0;
1049         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1050                 u32 rb_disable_bitmap;
1051
1052                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1053                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1054                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1055                 tmp <<= 4;
1056                 tmp |= rb_disable_bitmap;
1057         }
1058         /* enabled rb are just the one not disabled :) */
1059         disabled_rb_mask = tmp;
1060         tmp = 0;
1061         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1062                 tmp |= (1 << i);
1063         /* if all the backends are disabled, fix it up here */
1064         if ((disabled_rb_mask & tmp) == tmp) {
1065                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1066                         disabled_rb_mask &= ~(1 << i);
1067         }
1068
1069         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1070                 u32 simd_disable_bitmap;
1071
1072                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1073                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1074                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1075                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1076                 tmp <<= 16;
1077                 tmp |= simd_disable_bitmap;
1078         }
1079         rdev->config.cayman.active_simds = hweight32(~tmp);
1080
1081         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1082         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1083
1084         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1085         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1086         if (ASIC_IS_DCE6(rdev))
1087                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1088         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1089         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1090         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1091         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1092         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1093         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1094
1095         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1096             (rdev->flags & RADEON_IS_IGP)) {
1097                 if ((disabled_rb_mask & 3) == 1) {
1098                         /* RB0 disabled, RB1 enabled */
1099                         tmp = 0x11111111;
1100                 } else {
1101                         /* RB1 disabled, RB0 enabled */
1102                         tmp = 0x00000000;
1103                 }
1104         } else {
1105                 tmp = gb_addr_config & NUM_PIPES_MASK;
1106                 tmp = r6xx_remap_render_backend(rdev, tmp,
1107                                                 rdev->config.cayman.max_backends_per_se *
1108                                                 rdev->config.cayman.max_shader_engines,
1109                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1110         }
1111         WREG32(GB_BACKEND_MAP, tmp);
1112
1113         cgts_tcc_disable = 0xffff0000;
1114         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1115                 cgts_tcc_disable &= ~(1 << (16 + i));
1116         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1117         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1118         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1119         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1120
1121         /* reprogram the shader complex */
1122         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1123         for (i = 0; i < 16; i++)
1124                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1125         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1126
1127         /* set HW defaults for 3D engine */
1128         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1129
1130         sx_debug_1 = RREG32(SX_DEBUG_1);
1131         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1132         WREG32(SX_DEBUG_1, sx_debug_1);
1133
1134         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1135         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1136         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1137         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1138
1139         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1140
1141         /* need to be explicitly zero-ed */
1142         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1143         WREG32(SQ_LSTMP_RING_BASE, 0);
1144         WREG32(SQ_HSTMP_RING_BASE, 0);
1145         WREG32(SQ_ESTMP_RING_BASE, 0);
1146         WREG32(SQ_GSTMP_RING_BASE, 0);
1147         WREG32(SQ_VSTMP_RING_BASE, 0);
1148         WREG32(SQ_PSTMP_RING_BASE, 0);
1149
1150         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1151
1152         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1153                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1154                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1155
1156         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1157                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1158                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1159
1160
1161         WREG32(VGT_NUM_INSTANCES, 1);
1162
1163         WREG32(CP_PERFMON_CNTL, 0);
1164
1165         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1166                                   FETCH_FIFO_HIWATER(0x4) |
1167                                   DONE_FIFO_HIWATER(0xe0) |
1168                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1169
1170         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1171         WREG32(SQ_CONFIG, (VC_ENABLE |
1172                            EXPORT_SRC_C |
1173                            GFX_PRIO(0) |
1174                            CS1_PRIO(0) |
1175                            CS2_PRIO(1)));
1176         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1177
1178         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1179                                           FORCE_EOV_MAX_REZ_CNT(255)));
1180
1181         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1182                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1183
1184         WREG32(VGT_GS_VERTEX_REUSE, 16);
1185         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1186
1187         WREG32(CB_PERF_CTR0_SEL_0, 0);
1188         WREG32(CB_PERF_CTR0_SEL_1, 0);
1189         WREG32(CB_PERF_CTR1_SEL_0, 0);
1190         WREG32(CB_PERF_CTR1_SEL_1, 0);
1191         WREG32(CB_PERF_CTR2_SEL_0, 0);
1192         WREG32(CB_PERF_CTR2_SEL_1, 0);
1193         WREG32(CB_PERF_CTR3_SEL_0, 0);
1194         WREG32(CB_PERF_CTR3_SEL_1, 0);
1195
1196         tmp = RREG32(HDP_MISC_CNTL);
1197         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1198         WREG32(HDP_MISC_CNTL, tmp);
1199
1200         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1201         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1202
1203         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1204
1205         udelay(50);
1206
1207         /* set clockgating golden values on TN */
1208         if (rdev->family == CHIP_ARUBA) {
1209                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1210                 tmp &= ~0x00380000;
1211                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1212                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1213                 tmp &= ~0x0e000000;
1214                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1215         }
1216 }
1217
1218 /*
1219  * GART
1220  */
1221 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1222 {
1223         /* flush hdp cache */
1224         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1225
1226         /* bits 0-7 are the VM contexts0-7 */
1227         WREG32(VM_INVALIDATE_REQUEST, 1);
1228 }
1229
1230 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1231 {
1232         int i, r;
1233
1234         if (rdev->gart.robj == NULL) {
1235                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1236                 return -EINVAL;
1237         }
1238         r = radeon_gart_table_vram_pin(rdev);
1239         if (r)
1240                 return r;
1241         /* Setup TLB control */
1242         WREG32(MC_VM_MX_L1_TLB_CNTL,
1243                (0xA << 7) |
1244                ENABLE_L1_TLB |
1245                ENABLE_L1_FRAGMENT_PROCESSING |
1246                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1247                ENABLE_ADVANCED_DRIVER_MODEL |
1248                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1249         /* Setup L2 cache */
1250         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1251                ENABLE_L2_FRAGMENT_PROCESSING |
1252                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1253                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1254                EFFECTIVE_L2_QUEUE_SIZE(7) |
1255                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1256         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1257         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1258                BANK_SELECT(6) |
1259                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1260         /* setup context0 */
1261         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1262         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1263         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1264         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1265                         (u32)(rdev->dummy_page.addr >> 12));
1266         WREG32(VM_CONTEXT0_CNTL2, 0);
1267         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1268                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1269
1270         WREG32(0x15D4, 0);
1271         WREG32(0x15D8, 0);
1272         WREG32(0x15DC, 0);
1273
1274         /* empty context1-7 */
1275         /* Assign the pt base to something valid for now; the pts used for
1276          * the VMs are determined by the application and setup and assigned
1277          * on the fly in the vm part of radeon_gart.c
1278          */
1279         for (i = 1; i < 8; i++) {
1280                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1281                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1282                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1283                        rdev->vm_manager.saved_table_addr[i]);
1284         }
1285
1286         /* enable context1-7 */
1287         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1288                (u32)(rdev->dummy_page.addr >> 12));
1289         WREG32(VM_CONTEXT1_CNTL2, 4);
1290         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1291                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1292                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1293                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1294                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1295                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1296                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1297                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1298                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1299                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1300                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1301                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1302                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1303                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1304
1305         cayman_pcie_gart_tlb_flush(rdev);
1306         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1307                  (unsigned)(rdev->mc.gtt_size >> 20),
1308                  (unsigned long long)rdev->gart.table_addr);
1309         rdev->gart.ready = true;
1310         return 0;
1311 }
1312
1313 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1314 {
1315         unsigned i;
1316
1317         for (i = 1; i < 8; ++i) {
1318                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1319                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1320         }
1321
1322         /* Disable all tables */
1323         WREG32(VM_CONTEXT0_CNTL, 0);
1324         WREG32(VM_CONTEXT1_CNTL, 0);
1325         /* Setup TLB control */
1326         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1327                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1328                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1329         /* Setup L2 cache */
1330         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1331                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1332                EFFECTIVE_L2_QUEUE_SIZE(7) |
1333                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1334         WREG32(VM_L2_CNTL2, 0);
1335         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1336                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1337         radeon_gart_table_vram_unpin(rdev);
1338 }
1339
1340 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1341 {
1342         cayman_pcie_gart_disable(rdev);
1343         radeon_gart_table_vram_free(rdev);
1344         radeon_gart_fini(rdev);
1345 }
1346
1347 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1348                               int ring, u32 cp_int_cntl)
1349 {
1350         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1351
1352         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1353         WREG32(CP_INT_CNTL, cp_int_cntl);
1354 }
1355
1356 /*
1357  * CP.
1358  */
1359 void cayman_fence_ring_emit(struct radeon_device *rdev,
1360                             struct radeon_fence *fence)
1361 {
1362         struct radeon_ring *ring = &rdev->ring[fence->ring];
1363         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1364         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1365                 PACKET3_SH_ACTION_ENA;
1366
1367         /* flush read cache over gart for this vmid */
1368         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1369         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1370         radeon_ring_write(ring, 0xFFFFFFFF);
1371         radeon_ring_write(ring, 0);
1372         radeon_ring_write(ring, 10); /* poll interval */
1373         /* EVENT_WRITE_EOP - flush caches, send int */
1374         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1375         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1376         radeon_ring_write(ring, lower_32_bits(addr));
1377         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1378         radeon_ring_write(ring, fence->seq);
1379         radeon_ring_write(ring, 0);
1380 }
1381
1382 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1383 {
1384         struct radeon_ring *ring = &rdev->ring[ib->ring];
1385         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1386                 PACKET3_SH_ACTION_ENA;
1387
1388         /* set to DX10/11 mode */
1389         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1390         radeon_ring_write(ring, 1);
1391
1392         if (ring->rptr_save_reg) {
1393                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1394                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1395                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1396                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1397                 radeon_ring_write(ring, next_rptr);
1398         }
1399
1400         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1401         radeon_ring_write(ring,
1402 #ifdef __BIG_ENDIAN
1403                           (2 << 0) |
1404 #endif
1405                           (ib->gpu_addr & 0xFFFFFFFC));
1406         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1407         radeon_ring_write(ring, ib->length_dw |
1408                           (ib->vm ? (ib->vm->id << 24) : 0));
1409
1410         /* flush read cache over gart for this vmid */
1411         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1412         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1413         radeon_ring_write(ring, 0xFFFFFFFF);
1414         radeon_ring_write(ring, 0);
1415         radeon_ring_write(ring, ((ib->vm ? ib->vm->id : 0) << 24) | 10); /* poll interval */
1416 }
1417
1418 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1419 {
1420         if (enable)
1421                 WREG32(CP_ME_CNTL, 0);
1422         else {
1423                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1424                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1425                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1426                 WREG32(SCRATCH_UMSK, 0);
1427                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1428         }
1429 }
1430
1431 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1432                         struct radeon_ring *ring)
1433 {
1434         u32 rptr;
1435
1436         if (rdev->wb.enabled)
1437                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1438         else {
1439                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1440                         rptr = RREG32(CP_RB0_RPTR);
1441                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1442                         rptr = RREG32(CP_RB1_RPTR);
1443                 else
1444                         rptr = RREG32(CP_RB2_RPTR);
1445         }
1446
1447         return rptr;
1448 }
1449
1450 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1451                         struct radeon_ring *ring)
1452 {
1453         u32 wptr;
1454
1455         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1456                 wptr = RREG32(CP_RB0_WPTR);
1457         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1458                 wptr = RREG32(CP_RB1_WPTR);
1459         else
1460                 wptr = RREG32(CP_RB2_WPTR);
1461
1462         return wptr;
1463 }
1464
1465 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1466                          struct radeon_ring *ring)
1467 {
1468         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1469                 WREG32(CP_RB0_WPTR, ring->wptr);
1470                 (void)RREG32(CP_RB0_WPTR);
1471         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1472                 WREG32(CP_RB1_WPTR, ring->wptr);
1473                 (void)RREG32(CP_RB1_WPTR);
1474         } else {
1475                 WREG32(CP_RB2_WPTR, ring->wptr);
1476                 (void)RREG32(CP_RB2_WPTR);
1477         }
1478 }
1479
1480 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1481 {
1482         const __be32 *fw_data;
1483         int i;
1484
1485         if (!rdev->me_fw || !rdev->pfp_fw)
1486                 return -EINVAL;
1487
1488         cayman_cp_enable(rdev, false);
1489
1490         fw_data = (const __be32 *)rdev->pfp_fw->data;
1491         WREG32(CP_PFP_UCODE_ADDR, 0);
1492         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1493                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1494         WREG32(CP_PFP_UCODE_ADDR, 0);
1495
1496         fw_data = (const __be32 *)rdev->me_fw->data;
1497         WREG32(CP_ME_RAM_WADDR, 0);
1498         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1499                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1500
1501         WREG32(CP_PFP_UCODE_ADDR, 0);
1502         WREG32(CP_ME_RAM_WADDR, 0);
1503         WREG32(CP_ME_RAM_RADDR, 0);
1504         return 0;
1505 }
1506
1507 static int cayman_cp_start(struct radeon_device *rdev)
1508 {
1509         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1510         int r, i;
1511
1512         r = radeon_ring_lock(rdev, ring, 7);
1513         if (r) {
1514                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1515                 return r;
1516         }
1517         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1518         radeon_ring_write(ring, 0x1);
1519         radeon_ring_write(ring, 0x0);
1520         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1521         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1522         radeon_ring_write(ring, 0);
1523         radeon_ring_write(ring, 0);
1524         radeon_ring_unlock_commit(rdev, ring, false);
1525
1526         cayman_cp_enable(rdev, true);
1527
1528         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1529         if (r) {
1530                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1531                 return r;
1532         }
1533
1534         /* setup clear context state */
1535         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1536         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1537
1538         for (i = 0; i < cayman_default_size; i++)
1539                 radeon_ring_write(ring, cayman_default_state[i]);
1540
1541         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1542         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1543
1544         /* set clear context state */
1545         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1546         radeon_ring_write(ring, 0);
1547
1548         /* SQ_VTX_BASE_VTX_LOC */
1549         radeon_ring_write(ring, 0xc0026f00);
1550         radeon_ring_write(ring, 0x00000000);
1551         radeon_ring_write(ring, 0x00000000);
1552         radeon_ring_write(ring, 0x00000000);
1553
1554         /* Clear consts */
1555         radeon_ring_write(ring, 0xc0036f00);
1556         radeon_ring_write(ring, 0x00000bc4);
1557         radeon_ring_write(ring, 0xffffffff);
1558         radeon_ring_write(ring, 0xffffffff);
1559         radeon_ring_write(ring, 0xffffffff);
1560
1561         radeon_ring_write(ring, 0xc0026900);
1562         radeon_ring_write(ring, 0x00000316);
1563         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1564         radeon_ring_write(ring, 0x00000010); /*  */
1565
1566         radeon_ring_unlock_commit(rdev, ring, false);
1567
1568         /* XXX init other rings */
1569
1570         return 0;
1571 }
1572
1573 static void cayman_cp_fini(struct radeon_device *rdev)
1574 {
1575         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1576         cayman_cp_enable(rdev, false);
1577         radeon_ring_fini(rdev, ring);
1578         radeon_scratch_free(rdev, ring->rptr_save_reg);
1579 }
1580
1581 static int cayman_cp_resume(struct radeon_device *rdev)
1582 {
1583         static const int ridx[] = {
1584                 RADEON_RING_TYPE_GFX_INDEX,
1585                 CAYMAN_RING_TYPE_CP1_INDEX,
1586                 CAYMAN_RING_TYPE_CP2_INDEX
1587         };
1588         static const unsigned cp_rb_cntl[] = {
1589                 CP_RB0_CNTL,
1590                 CP_RB1_CNTL,
1591                 CP_RB2_CNTL,
1592         };
1593         static const unsigned cp_rb_rptr_addr[] = {
1594                 CP_RB0_RPTR_ADDR,
1595                 CP_RB1_RPTR_ADDR,
1596                 CP_RB2_RPTR_ADDR
1597         };
1598         static const unsigned cp_rb_rptr_addr_hi[] = {
1599                 CP_RB0_RPTR_ADDR_HI,
1600                 CP_RB1_RPTR_ADDR_HI,
1601                 CP_RB2_RPTR_ADDR_HI
1602         };
1603         static const unsigned cp_rb_base[] = {
1604                 CP_RB0_BASE,
1605                 CP_RB1_BASE,
1606                 CP_RB2_BASE
1607         };
1608         static const unsigned cp_rb_rptr[] = {
1609                 CP_RB0_RPTR,
1610                 CP_RB1_RPTR,
1611                 CP_RB2_RPTR
1612         };
1613         static const unsigned cp_rb_wptr[] = {
1614                 CP_RB0_WPTR,
1615                 CP_RB1_WPTR,
1616                 CP_RB2_WPTR
1617         };
1618         struct radeon_ring *ring;
1619         int i, r;
1620
1621         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1622         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1623                                  SOFT_RESET_PA |
1624                                  SOFT_RESET_SH |
1625                                  SOFT_RESET_VGT |
1626                                  SOFT_RESET_SPI |
1627                                  SOFT_RESET_SX));
1628         RREG32(GRBM_SOFT_RESET);
1629         mdelay(15);
1630         WREG32(GRBM_SOFT_RESET, 0);
1631         RREG32(GRBM_SOFT_RESET);
1632
1633         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1634         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1635
1636         /* Set the write pointer delay */
1637         WREG32(CP_RB_WPTR_DELAY, 0);
1638
1639         WREG32(CP_DEBUG, (1 << 27));
1640
1641         /* set the wb address whether it's enabled or not */
1642         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1643         WREG32(SCRATCH_UMSK, 0xff);
1644
1645         for (i = 0; i < 3; ++i) {
1646                 uint32_t rb_cntl;
1647                 uint64_t addr;
1648
1649                 /* Set ring buffer size */
1650                 ring = &rdev->ring[ridx[i]];
1651                 rb_cntl = order_base_2(ring->ring_size / 8);
1652                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1653 #ifdef __BIG_ENDIAN
1654                 rb_cntl |= BUF_SWAP_32BIT;
1655 #endif
1656                 WREG32(cp_rb_cntl[i], rb_cntl);
1657
1658                 /* set the wb address whether it's enabled or not */
1659                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1660                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1661                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1662         }
1663
1664         /* set the rb base addr, this causes an internal reset of ALL rings */
1665         for (i = 0; i < 3; ++i) {
1666                 ring = &rdev->ring[ridx[i]];
1667                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1668         }
1669
1670         for (i = 0; i < 3; ++i) {
1671                 /* Initialize the ring buffer's read and write pointers */
1672                 ring = &rdev->ring[ridx[i]];
1673                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1674
1675                 ring->wptr = 0;
1676                 WREG32(cp_rb_rptr[i], 0);
1677                 WREG32(cp_rb_wptr[i], ring->wptr);
1678
1679                 mdelay(1);
1680                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1681         }
1682
1683         /* start the rings */
1684         cayman_cp_start(rdev);
1685         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1686         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1687         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1688         /* this only test cp0 */
1689         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1690         if (r) {
1691                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1692                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1693                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1694                 return r;
1695         }
1696
1697         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1698                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1699
1700         return 0;
1701 }
1702
1703 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1704 {
1705         u32 reset_mask = 0;
1706         u32 tmp;
1707
1708         /* GRBM_STATUS */
1709         tmp = RREG32(GRBM_STATUS);
1710         if (tmp & (PA_BUSY | SC_BUSY |
1711                    SH_BUSY | SX_BUSY |
1712                    TA_BUSY | VGT_BUSY |
1713                    DB_BUSY | CB_BUSY |
1714                    GDS_BUSY | SPI_BUSY |
1715                    IA_BUSY | IA_BUSY_NO_DMA))
1716                 reset_mask |= RADEON_RESET_GFX;
1717
1718         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1719                    CP_BUSY | CP_COHERENCY_BUSY))
1720                 reset_mask |= RADEON_RESET_CP;
1721
1722         if (tmp & GRBM_EE_BUSY)
1723                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1724
1725         /* DMA_STATUS_REG 0 */
1726         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1727         if (!(tmp & DMA_IDLE))
1728                 reset_mask |= RADEON_RESET_DMA;
1729
1730         /* DMA_STATUS_REG 1 */
1731         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1732         if (!(tmp & DMA_IDLE))
1733                 reset_mask |= RADEON_RESET_DMA1;
1734
1735         /* SRBM_STATUS2 */
1736         tmp = RREG32(SRBM_STATUS2);
1737         if (tmp & DMA_BUSY)
1738                 reset_mask |= RADEON_RESET_DMA;
1739
1740         if (tmp & DMA1_BUSY)
1741                 reset_mask |= RADEON_RESET_DMA1;
1742
1743         /* SRBM_STATUS */
1744         tmp = RREG32(SRBM_STATUS);
1745         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1746                 reset_mask |= RADEON_RESET_RLC;
1747
1748         if (tmp & IH_BUSY)
1749                 reset_mask |= RADEON_RESET_IH;
1750
1751         if (tmp & SEM_BUSY)
1752                 reset_mask |= RADEON_RESET_SEM;
1753
1754         if (tmp & GRBM_RQ_PENDING)
1755                 reset_mask |= RADEON_RESET_GRBM;
1756
1757         if (tmp & VMC_BUSY)
1758                 reset_mask |= RADEON_RESET_VMC;
1759
1760         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1761                    MCC_BUSY | MCD_BUSY))
1762                 reset_mask |= RADEON_RESET_MC;
1763
1764         if (evergreen_is_display_hung(rdev))
1765                 reset_mask |= RADEON_RESET_DISPLAY;
1766
1767         /* VM_L2_STATUS */
1768         tmp = RREG32(VM_L2_STATUS);
1769         if (tmp & L2_BUSY)
1770                 reset_mask |= RADEON_RESET_VMC;
1771
1772         /* Skip MC reset as it's mostly likely not hung, just busy */
1773         if (reset_mask & RADEON_RESET_MC) {
1774                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1775                 reset_mask &= ~RADEON_RESET_MC;
1776         }
1777
1778         return reset_mask;
1779 }
1780
1781 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1782 {
1783         struct evergreen_mc_save save;
1784         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1785         u32 tmp;
1786
1787         if (reset_mask == 0)
1788                 return;
1789
1790         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1791
1792         evergreen_print_gpu_status_regs(rdev);
1793         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1794                  RREG32(0x14F8));
1795         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1796                  RREG32(0x14D8));
1797         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1798                  RREG32(0x14FC));
1799         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1800                  RREG32(0x14DC));
1801
1802         /* Disable CP parsing/prefetching */
1803         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1804
1805         if (reset_mask & RADEON_RESET_DMA) {
1806                 /* dma0 */
1807                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1808                 tmp &= ~DMA_RB_ENABLE;
1809                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1810         }
1811
1812         if (reset_mask & RADEON_RESET_DMA1) {
1813                 /* dma1 */
1814                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1815                 tmp &= ~DMA_RB_ENABLE;
1816                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1817         }
1818
1819         udelay(50);
1820
1821         evergreen_mc_stop(rdev, &save);
1822         if (evergreen_mc_wait_for_idle(rdev)) {
1823                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1824         }
1825
1826         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1827                 grbm_soft_reset = SOFT_RESET_CB |
1828                         SOFT_RESET_DB |
1829                         SOFT_RESET_GDS |
1830                         SOFT_RESET_PA |
1831                         SOFT_RESET_SC |
1832                         SOFT_RESET_SPI |
1833                         SOFT_RESET_SH |
1834                         SOFT_RESET_SX |
1835                         SOFT_RESET_TC |
1836                         SOFT_RESET_TA |
1837                         SOFT_RESET_VGT |
1838                         SOFT_RESET_IA;
1839         }
1840
1841         if (reset_mask & RADEON_RESET_CP) {
1842                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1843
1844                 srbm_soft_reset |= SOFT_RESET_GRBM;
1845         }
1846
1847         if (reset_mask & RADEON_RESET_DMA)
1848                 srbm_soft_reset |= SOFT_RESET_DMA;
1849
1850         if (reset_mask & RADEON_RESET_DMA1)
1851                 srbm_soft_reset |= SOFT_RESET_DMA1;
1852
1853         if (reset_mask & RADEON_RESET_DISPLAY)
1854                 srbm_soft_reset |= SOFT_RESET_DC;
1855
1856         if (reset_mask & RADEON_RESET_RLC)
1857                 srbm_soft_reset |= SOFT_RESET_RLC;
1858
1859         if (reset_mask & RADEON_RESET_SEM)
1860                 srbm_soft_reset |= SOFT_RESET_SEM;
1861
1862         if (reset_mask & RADEON_RESET_IH)
1863                 srbm_soft_reset |= SOFT_RESET_IH;
1864
1865         if (reset_mask & RADEON_RESET_GRBM)
1866                 srbm_soft_reset |= SOFT_RESET_GRBM;
1867
1868         if (reset_mask & RADEON_RESET_VMC)
1869                 srbm_soft_reset |= SOFT_RESET_VMC;
1870
1871         if (!(rdev->flags & RADEON_IS_IGP)) {
1872                 if (reset_mask & RADEON_RESET_MC)
1873                         srbm_soft_reset |= SOFT_RESET_MC;
1874         }
1875
1876         if (grbm_soft_reset) {
1877                 tmp = RREG32(GRBM_SOFT_RESET);
1878                 tmp |= grbm_soft_reset;
1879                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1880                 WREG32(GRBM_SOFT_RESET, tmp);
1881                 tmp = RREG32(GRBM_SOFT_RESET);
1882
1883                 udelay(50);
1884
1885                 tmp &= ~grbm_soft_reset;
1886                 WREG32(GRBM_SOFT_RESET, tmp);
1887                 tmp = RREG32(GRBM_SOFT_RESET);
1888         }
1889
1890         if (srbm_soft_reset) {
1891                 tmp = RREG32(SRBM_SOFT_RESET);
1892                 tmp |= srbm_soft_reset;
1893                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1894                 WREG32(SRBM_SOFT_RESET, tmp);
1895                 tmp = RREG32(SRBM_SOFT_RESET);
1896
1897                 udelay(50);
1898
1899                 tmp &= ~srbm_soft_reset;
1900                 WREG32(SRBM_SOFT_RESET, tmp);
1901                 tmp = RREG32(SRBM_SOFT_RESET);
1902         }
1903
1904         /* Wait a little for things to settle down */
1905         udelay(50);
1906
1907         evergreen_mc_resume(rdev, &save);
1908         udelay(50);
1909
1910         evergreen_print_gpu_status_regs(rdev);
1911 }
1912
1913 int cayman_asic_reset(struct radeon_device *rdev)
1914 {
1915         u32 reset_mask;
1916
1917         reset_mask = cayman_gpu_check_soft_reset(rdev);
1918
1919         if (reset_mask)
1920                 r600_set_bios_scratch_engine_hung(rdev, true);
1921
1922         cayman_gpu_soft_reset(rdev, reset_mask);
1923
1924         reset_mask = cayman_gpu_check_soft_reset(rdev);
1925
1926         if (reset_mask)
1927                 evergreen_gpu_pci_config_reset(rdev);
1928
1929         r600_set_bios_scratch_engine_hung(rdev, false);
1930
1931         return 0;
1932 }
1933
1934 /**
1935  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1936  *
1937  * @rdev: radeon_device pointer
1938  * @ring: radeon_ring structure holding ring information
1939  *
1940  * Check if the GFX engine is locked up.
1941  * Returns true if the engine appears to be locked up, false if not.
1942  */
1943 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1944 {
1945         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1946
1947         if (!(reset_mask & (RADEON_RESET_GFX |
1948                             RADEON_RESET_COMPUTE |
1949                             RADEON_RESET_CP))) {
1950                 radeon_ring_lockup_update(rdev, ring);
1951                 return false;
1952         }
1953         return radeon_ring_test_lockup(rdev, ring);
1954 }
1955
1956 static int cayman_startup(struct radeon_device *rdev)
1957 {
1958         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1959         int r;
1960
1961         /* enable pcie gen2 link */
1962         evergreen_pcie_gen2_enable(rdev);
1963         /* enable aspm */
1964         evergreen_program_aspm(rdev);
1965
1966         /* scratch needs to be initialized before MC */
1967         r = r600_vram_scratch_init(rdev);
1968         if (r)
1969                 return r;
1970
1971         evergreen_mc_program(rdev);
1972
1973         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1974                 r = ni_mc_load_microcode(rdev);
1975                 if (r) {
1976                         DRM_ERROR("Failed to load MC firmware!\n");
1977                         return r;
1978                 }
1979         }
1980
1981         r = cayman_pcie_gart_enable(rdev);
1982         if (r)
1983                 return r;
1984         cayman_gpu_init(rdev);
1985
1986         /* allocate rlc buffers */
1987         if (rdev->flags & RADEON_IS_IGP) {
1988                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1989                 rdev->rlc.reg_list_size =
1990                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1991                 rdev->rlc.cs_data = cayman_cs_data;
1992                 r = sumo_rlc_init(rdev);
1993                 if (r) {
1994                         DRM_ERROR("Failed to init rlc BOs!\n");
1995                         return r;
1996                 }
1997         }
1998
1999         /* allocate wb buffer */
2000         r = radeon_wb_init(rdev);
2001         if (r)
2002                 return r;
2003
2004         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005         if (r) {
2006                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2007                 return r;
2008         }
2009
2010         r = uvd_v2_2_resume(rdev);
2011         if (!r) {
2012                 r = radeon_fence_driver_start_ring(rdev,
2013                                                    R600_RING_TYPE_UVD_INDEX);
2014                 if (r)
2015                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2016         }
2017         if (r)
2018                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2019
2020         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2021         if (r) {
2022                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2023                 return r;
2024         }
2025
2026         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2027         if (r) {
2028                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2029                 return r;
2030         }
2031
2032         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2033         if (r) {
2034                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2035                 return r;
2036         }
2037
2038         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2039         if (r) {
2040                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2041                 return r;
2042         }
2043
2044         /* Enable IRQ */
2045         if (!rdev->irq.installed) {
2046                 r = radeon_irq_kms_init(rdev);
2047                 if (r)
2048                         return r;
2049         }
2050
2051         r = r600_irq_init(rdev);
2052         if (r) {
2053                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2054                 radeon_irq_kms_fini(rdev);
2055                 return r;
2056         }
2057         evergreen_irq_set(rdev);
2058
2059         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2060                              RADEON_CP_PACKET2);
2061         if (r)
2062                 return r;
2063
2064         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2065         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2066                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2067         if (r)
2068                 return r;
2069
2070         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2071         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2072                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2073         if (r)
2074                 return r;
2075
2076         r = cayman_cp_load_microcode(rdev);
2077         if (r)
2078                 return r;
2079         r = cayman_cp_resume(rdev);
2080         if (r)
2081                 return r;
2082
2083         r = cayman_dma_resume(rdev);
2084         if (r)
2085                 return r;
2086
2087         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2088         if (ring->ring_size) {
2089                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2090                                      RADEON_CP_PACKET2);
2091                 if (!r)
2092                         r = uvd_v1_0_init(rdev);
2093                 if (r)
2094                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2095         }
2096
2097         r = radeon_ib_pool_init(rdev);
2098         if (r) {
2099                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2100                 return r;
2101         }
2102
2103         r = radeon_vm_manager_init(rdev);
2104         if (r) {
2105                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2106                 return r;
2107         }
2108
2109         if (ASIC_IS_DCE6(rdev)) {
2110                 r = dce6_audio_init(rdev);
2111                 if (r)
2112                         return r;
2113         } else {
2114                 r = r600_audio_init(rdev);
2115                 if (r)
2116                         return r;
2117         }
2118
2119         return 0;
2120 }
2121
2122 int cayman_resume(struct radeon_device *rdev)
2123 {
2124         int r;
2125
2126         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2127          * posting will perform necessary task to bring back GPU into good
2128          * shape.
2129          */
2130         /* post card */
2131         atom_asic_init(rdev->mode_info.atom_context);
2132
2133         /* init golden registers */
2134         ni_init_golden_registers(rdev);
2135
2136         if (rdev->pm.pm_method == PM_METHOD_DPM)
2137                 radeon_pm_resume(rdev);
2138
2139         rdev->accel_working = true;
2140         r = cayman_startup(rdev);
2141         if (r) {
2142                 DRM_ERROR("cayman startup failed on resume\n");
2143                 rdev->accel_working = false;
2144                 return r;
2145         }
2146         return r;
2147 }
2148
2149 int cayman_suspend(struct radeon_device *rdev)
2150 {
2151         radeon_pm_suspend(rdev);
2152         if (ASIC_IS_DCE6(rdev))
2153                 dce6_audio_fini(rdev);
2154         else
2155                 r600_audio_fini(rdev);
2156         radeon_vm_manager_fini(rdev);
2157         cayman_cp_enable(rdev, false);
2158         cayman_dma_stop(rdev);
2159         uvd_v1_0_fini(rdev);
2160         radeon_uvd_suspend(rdev);
2161         evergreen_irq_suspend(rdev);
2162         radeon_wb_disable(rdev);
2163         cayman_pcie_gart_disable(rdev);
2164         return 0;
2165 }
2166
2167 /* Plan is to move initialization in that function and use
2168  * helper function so that radeon_device_init pretty much
2169  * do nothing more than calling asic specific function. This
2170  * should also allow to remove a bunch of callback function
2171  * like vram_info.
2172  */
2173 int cayman_init(struct radeon_device *rdev)
2174 {
2175         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2176         int r;
2177
2178         /* Read BIOS */
2179         if (!radeon_get_bios(rdev)) {
2180                 if (ASIC_IS_AVIVO(rdev))
2181                         return -EINVAL;
2182         }
2183         /* Must be an ATOMBIOS */
2184         if (!rdev->is_atom_bios) {
2185                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2186                 return -EINVAL;
2187         }
2188         r = radeon_atombios_init(rdev);
2189         if (r)
2190                 return r;
2191
2192         /* Post card if necessary */
2193         if (!radeon_card_posted(rdev)) {
2194                 if (!rdev->bios) {
2195                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2196                         return -EINVAL;
2197                 }
2198                 DRM_INFO("GPU not posted. posting now...\n");
2199                 atom_asic_init(rdev->mode_info.atom_context);
2200         }
2201         /* init golden registers */
2202         ni_init_golden_registers(rdev);
2203         /* Initialize scratch registers */
2204         r600_scratch_init(rdev);
2205         /* Initialize surface registers */
2206         radeon_surface_init(rdev);
2207         /* Initialize clocks */
2208         radeon_get_clock_info(rdev->ddev);
2209         /* Fence driver */
2210         r = radeon_fence_driver_init(rdev);
2211         if (r)
2212                 return r;
2213         /* initialize memory controller */
2214         r = evergreen_mc_init(rdev);
2215         if (r)
2216                 return r;
2217         /* Memory manager */
2218         r = radeon_bo_init(rdev);
2219         if (r)
2220                 return r;
2221
2222         if (rdev->flags & RADEON_IS_IGP) {
2223                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2224                         r = ni_init_microcode(rdev);
2225                         if (r) {
2226                                 DRM_ERROR("Failed to load firmware!\n");
2227                                 return r;
2228                         }
2229                 }
2230         } else {
2231                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2232                         r = ni_init_microcode(rdev);
2233                         if (r) {
2234                                 DRM_ERROR("Failed to load firmware!\n");
2235                                 return r;
2236                         }
2237                 }
2238         }
2239
2240         /* Initialize power management */
2241         radeon_pm_init(rdev);
2242
2243         ring->ring_obj = NULL;
2244         r600_ring_init(rdev, ring, 1024 * 1024);
2245
2246         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2247         ring->ring_obj = NULL;
2248         r600_ring_init(rdev, ring, 64 * 1024);
2249
2250         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2251         ring->ring_obj = NULL;
2252         r600_ring_init(rdev, ring, 64 * 1024);
2253
2254         r = radeon_uvd_init(rdev);
2255         if (!r) {
2256                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2257                 ring->ring_obj = NULL;
2258                 r600_ring_init(rdev, ring, 4096);
2259         }
2260
2261         rdev->ih.ring_obj = NULL;
2262         r600_ih_ring_init(rdev, 64 * 1024);
2263
2264         r = r600_pcie_gart_init(rdev);
2265         if (r)
2266                 return r;
2267
2268         rdev->accel_working = true;
2269         r = cayman_startup(rdev);
2270         if (r) {
2271                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2272                 cayman_cp_fini(rdev);
2273                 cayman_dma_fini(rdev);
2274                 r600_irq_fini(rdev);
2275                 if (rdev->flags & RADEON_IS_IGP)
2276                         sumo_rlc_fini(rdev);
2277                 radeon_wb_fini(rdev);
2278                 radeon_ib_pool_fini(rdev);
2279                 radeon_vm_manager_fini(rdev);
2280                 radeon_irq_kms_fini(rdev);
2281                 cayman_pcie_gart_fini(rdev);
2282                 rdev->accel_working = false;
2283         }
2284
2285         /* Don't start up if the MC ucode is missing.
2286          * The default clocks and voltages before the MC ucode
2287          * is loaded are not suffient for advanced operations.
2288          *
2289          * We can skip this check for TN, because there is no MC
2290          * ucode.
2291          */
2292         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2293                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2294                 return -EINVAL;
2295         }
2296
2297         return 0;
2298 }
2299
2300 void cayman_fini(struct radeon_device *rdev)
2301 {
2302         radeon_pm_fini(rdev);
2303         cayman_cp_fini(rdev);
2304         cayman_dma_fini(rdev);
2305         r600_irq_fini(rdev);
2306         if (rdev->flags & RADEON_IS_IGP)
2307                 sumo_rlc_fini(rdev);
2308         radeon_wb_fini(rdev);
2309         radeon_vm_manager_fini(rdev);
2310         radeon_ib_pool_fini(rdev);
2311         radeon_irq_kms_fini(rdev);
2312         uvd_v1_0_fini(rdev);
2313         radeon_uvd_fini(rdev);
2314         cayman_pcie_gart_fini(rdev);
2315         r600_vram_scratch_fini(rdev);
2316         radeon_gem_fini(rdev);
2317         radeon_fence_driver_fini(rdev);
2318         radeon_bo_fini(rdev);
2319         radeon_atombios_fini(rdev);
2320         ni_fini_microcode(rdev);
2321         kfree(rdev->bios);
2322         rdev->bios = NULL;
2323 }
2324
2325 /*
2326  * vm
2327  */
2328 int cayman_vm_init(struct radeon_device *rdev)
2329 {
2330         /* number of VMs */
2331         rdev->vm_manager.nvm = 8;
2332         /* base offset of vram pages */
2333         if (rdev->flags & RADEON_IS_IGP) {
2334                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2335                 tmp <<= 22;
2336                 rdev->vm_manager.vram_base_offset = tmp;
2337         } else
2338                 rdev->vm_manager.vram_base_offset = 0;
2339         return 0;
2340 }
2341
2342 void cayman_vm_fini(struct radeon_device *rdev)
2343 {
2344 }
2345
2346 /**
2347  * cayman_vm_decode_fault - print human readable fault info
2348  *
2349  * @rdev: radeon_device pointer
2350  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2351  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2352  *
2353  * Print human readable fault information (cayman/TN).
2354  */
2355 void cayman_vm_decode_fault(struct radeon_device *rdev,
2356                             u32 status, u32 addr)
2357 {
2358         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2359         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2360         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2361         char *block;
2362
2363         switch (mc_id) {
2364         case 32:
2365         case 16:
2366         case 96:
2367         case 80:
2368         case 160:
2369         case 144:
2370         case 224:
2371         case 208:
2372                 block = "CB";
2373                 break;
2374         case 33:
2375         case 17:
2376         case 97:
2377         case 81:
2378         case 161:
2379         case 145:
2380         case 225:
2381         case 209:
2382                 block = "CB_FMASK";
2383                 break;
2384         case 34:
2385         case 18:
2386         case 98:
2387         case 82:
2388         case 162:
2389         case 146:
2390         case 226:
2391         case 210:
2392                 block = "CB_CMASK";
2393                 break;
2394         case 35:
2395         case 19:
2396         case 99:
2397         case 83:
2398         case 163:
2399         case 147:
2400         case 227:
2401         case 211:
2402                 block = "CB_IMMED";
2403                 break;
2404         case 36:
2405         case 20:
2406         case 100:
2407         case 84:
2408         case 164:
2409         case 148:
2410         case 228:
2411         case 212:
2412                 block = "DB";
2413                 break;
2414         case 37:
2415         case 21:
2416         case 101:
2417         case 85:
2418         case 165:
2419         case 149:
2420         case 229:
2421         case 213:
2422                 block = "DB_HTILE";
2423                 break;
2424         case 38:
2425         case 22:
2426         case 102:
2427         case 86:
2428         case 166:
2429         case 150:
2430         case 230:
2431         case 214:
2432                 block = "SX";
2433                 break;
2434         case 39:
2435         case 23:
2436         case 103:
2437         case 87:
2438         case 167:
2439         case 151:
2440         case 231:
2441         case 215:
2442                 block = "DB_STEN";
2443                 break;
2444         case 40:
2445         case 24:
2446         case 104:
2447         case 88:
2448         case 232:
2449         case 216:
2450         case 168:
2451         case 152:
2452                 block = "TC_TFETCH";
2453                 break;
2454         case 41:
2455         case 25:
2456         case 105:
2457         case 89:
2458         case 233:
2459         case 217:
2460         case 169:
2461         case 153:
2462                 block = "TC_VFETCH";
2463                 break;
2464         case 42:
2465         case 26:
2466         case 106:
2467         case 90:
2468         case 234:
2469         case 218:
2470         case 170:
2471         case 154:
2472                 block = "VC";
2473                 break;
2474         case 112:
2475                 block = "CP";
2476                 break;
2477         case 113:
2478         case 114:
2479                 block = "SH";
2480                 break;
2481         case 115:
2482                 block = "VGT";
2483                 break;
2484         case 178:
2485                 block = "IH";
2486                 break;
2487         case 51:
2488                 block = "RLC";
2489                 break;
2490         case 55:
2491                 block = "DMA";
2492                 break;
2493         case 56:
2494                 block = "HDP";
2495                 break;
2496         default:
2497                 block = "unknown";
2498                 break;
2499         }
2500
2501         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2502                protections, vmid, addr,
2503                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2504                block, mc_id);
2505 }
2506
2507 /**
2508  * cayman_vm_flush - vm flush using the CP
2509  *
2510  * @rdev: radeon_device pointer
2511  *
2512  * Update the page table base and flush the VM TLB
2513  * using the CP (cayman-si).
2514  */
2515 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2516 {
2517         struct radeon_ring *ring = &rdev->ring[ridx];
2518
2519         if (vm == NULL)
2520                 return;
2521
2522         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2523         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2524
2525         /* flush hdp cache */
2526         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2527         radeon_ring_write(ring, 0x1);
2528
2529         /* bits 0-7 are the VM contexts0-7 */
2530         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2531         radeon_ring_write(ring, 1 << vm->id);
2532
2533         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2534         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2535         radeon_ring_write(ring, 0x0);
2536 }