sys/dev/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include "radeon_audio.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 #define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
  39 #define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
  40 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
  41
  42 /*
  43  * Indirect registers accessor
  44  */
  45 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
  46 {
  47         unsigned long flags;
  48         u32 r;
  49
  50         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  51         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  52         r = RREG32(EVERGREEN_CG_IND_DATA);
  53         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  54         return r;
  55 }
  56
  57 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  58 {
  59         unsigned long flags;
  60
  61         spin_lock_irqsave(&rdev->cg_idx_lock, flags);
  62         WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
  63         WREG32(EVERGREEN_CG_IND_DATA, (v));
  64         spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
  65 }
  66
  67 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
  68 {
  69         unsigned long flags;
  70         u32 r;
  71
  72         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  73         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  74         r = RREG32(EVERGREEN_PIF_PHY0_DATA);
  75         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  76         return r;
  77 }
  78
  79 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  80 {
  81         unsigned long flags;
  82
  83         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  84         WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
  85         WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
  86         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  87 }
  88
  89 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
  90 {
  91         unsigned long flags;
  92         u32 r;
  93
  94         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
  95         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
  96         r = RREG32(EVERGREEN_PIF_PHY1_DATA);
  97         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
  98         return r;
  99 }
 100
 101 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 102 {
 103         unsigned long flags;
 104
 105         spin_lock_irqsave(&rdev->pif_idx_lock, flags);
 106         WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
 107         WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
 108         spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
 109 }
 110
 111 static const u32 crtc_offsets[6] =
 112 {
 113         EVERGREEN_CRTC0_REGISTER_OFFSET,
 114         EVERGREEN_CRTC1_REGISTER_OFFSET,
 115         EVERGREEN_CRTC2_REGISTER_OFFSET,
 116         EVERGREEN_CRTC3_REGISTER_OFFSET,
 117         EVERGREEN_CRTC4_REGISTER_OFFSET,
 118         EVERGREEN_CRTC5_REGISTER_OFFSET
 119 };
 120
 121 #include "clearstate_evergreen.h"
 122
 123 static const u32 sumo_rlc_save_restore_register_list[] =
 124 {
 125         0x98fc,
 126         0x9830,
 127         0x9834,
 128         0x9838,
 129         0x9870,
 130         0x9874,
 131         0x8a14,
 132         0x8b24,
 133         0x8bcc,
 134         0x8b10,
 135         0x8d00,
 136         0x8d04,
 137         0x8c00,
 138         0x8c04,
 139         0x8c08,
 140         0x8c0c,
 141         0x8d8c,
 142         0x8c20,
 143         0x8c24,
 144         0x8c28,
 145         0x8c18,
 146         0x8c1c,
 147         0x8cf0,
 148         0x8e2c,
 149         0x8e38,
 150         0x8c30,
 151         0x9508,
 152         0x9688,
 153         0x9608,
 154         0x960c,
 155         0x9610,
 156         0x9614,
 157         0x88c4,
 158         0x88d4,
 159         0xa008,
 160         0x900c,
 161         0x9100,
 162         0x913c,
 163         0x98f8,
 164         0x98f4,
 165         0x9b7c,
 166         0x3f8c,
 167         0x8950,
 168         0x8954,
 169         0x8a18,
 170         0x8b28,
 171         0x9144,
 172         0x9148,
 173         0x914c,
 174         0x3f90,
 175         0x3f94,
 176         0x915c,
 177         0x9160,
 178         0x9178,
 179         0x917c,
 180         0x9180,
 181         0x918c,
 182         0x9190,
 183         0x9194,
 184         0x9198,
 185         0x919c,
 186         0x91a8,
 187         0x91ac,
 188         0x91b0,
 189         0x91b4,
 190         0x91b8,
 191         0x91c4,
 192         0x91c8,
 193         0x91cc,
 194         0x91d0,
 195         0x91d4,
 196         0x91e0,
 197         0x91e4,
 198         0x91ec,
 199         0x91f0,
 200         0x91f4,
 201         0x9200,
 202         0x9204,
 203         0x929c,
 204         0x9150,
 205         0x802c,
 206 };
 207
 208 static void evergreen_gpu_init(struct radeon_device *rdev);
 209
 210 static const u32 evergreen_golden_registers[] =
 211 {
 212         0x3f90, 0xffff0000, 0xff000000,
 213         0x9148, 0xffff0000, 0xff000000,
 214         0x3f94, 0xffff0000, 0xff000000,
 215         0x914c, 0xffff0000, 0xff000000,
 216         0x9b7c, 0xffffffff, 0x00000000,
 217         0x8a14, 0xffffffff, 0x00000007,
 218         0x8b10, 0xffffffff, 0x00000000,
 219         0x960c, 0xffffffff, 0x54763210,
 220         0x88c4, 0xffffffff, 0x000000c2,
 221         0x88d4, 0xffffffff, 0x00000010,
 222         0x8974, 0xffffffff, 0x00000000,
 223         0xc78, 0x00000080, 0x00000080,
 224         0x5eb4, 0xffffffff, 0x00000002,
 225         0x5e78, 0xffffffff, 0x001000f0,
 226         0x6104, 0x01000300, 0x00000000,
 227         0x5bc0, 0x00300000, 0x00000000,
 228         0x7030, 0xffffffff, 0x00000011,
 229         0x7c30, 0xffffffff, 0x00000011,
 230         0x10830, 0xffffffff, 0x00000011,
 231         0x11430, 0xffffffff, 0x00000011,
 232         0x12030, 0xffffffff, 0x00000011,
 233         0x12c30, 0xffffffff, 0x00000011,
 234         0xd02c, 0xffffffff, 0x08421000,
 235         0x240c, 0xffffffff, 0x00000380,
 236         0x8b24, 0xffffffff, 0x00ff0fff,
 237         0x28a4c, 0x06000000, 0x06000000,
 238         0x10c, 0x00000001, 0x00000001,
 239         0x8d00, 0xffffffff, 0x100e4848,
 240         0x8d04, 0xffffffff, 0x00164745,
 241         0x8c00, 0xffffffff, 0xe4000003,
 242         0x8c04, 0xffffffff, 0x40600060,
 243         0x8c08, 0xffffffff, 0x001c001c,
 244         0x8cf0, 0xffffffff, 0x08e00620,
 245         0x8c20, 0xffffffff, 0x00800080,
 246         0x8c24, 0xffffffff, 0x00800080,
 247         0x8c18, 0xffffffff, 0x20202078,
 248         0x8c1c, 0xffffffff, 0x00001010,
 249         0x28350, 0xffffffff, 0x00000000,
 250         0xa008, 0xffffffff, 0x00010000,
 251         0x5c4, 0xffffffff, 0x00000001,
 252         0x9508, 0xffffffff, 0x00000002,
 253         0x913c, 0x0000000f, 0x0000000a
 254 };
 255
 256 static const u32 evergreen_golden_registers2[] =
 257 {
 258         0x2f4c, 0xffffffff, 0x00000000,
 259         0x54f4, 0xffffffff, 0x00000000,
 260         0x54f0, 0xffffffff, 0x00000000,
 261         0x5498, 0xffffffff, 0x00000000,
 262         0x549c, 0xffffffff, 0x00000000,
 263         0x5494, 0xffffffff, 0x00000000,
 264         0x53cc, 0xffffffff, 0x00000000,
 265         0x53c8, 0xffffffff, 0x00000000,
 266         0x53c4, 0xffffffff, 0x00000000,
 267         0x53c0, 0xffffffff, 0x00000000,
 268         0x53bc, 0xffffffff, 0x00000000,
 269         0x53b8, 0xffffffff, 0x00000000,
 270         0x53b4, 0xffffffff, 0x00000000,
 271         0x53b0, 0xffffffff, 0x00000000
 272 };
 273
 274 static const u32 cypress_mgcg_init[] =
 275 {
 276         0x802c, 0xffffffff, 0xc0000000,
 277         0x5448, 0xffffffff, 0x00000100,
 278         0x55e4, 0xffffffff, 0x00000100,
 279         0x160c, 0xffffffff, 0x00000100,
 280         0x5644, 0xffffffff, 0x00000100,
 281         0xc164, 0xffffffff, 0x00000100,
 282         0x8a18, 0xffffffff, 0x00000100,
 283         0x897c, 0xffffffff, 0x06000100,
 284         0x8b28, 0xffffffff, 0x00000100,
 285         0x9144, 0xffffffff, 0x00000100,
 286         0x9a60, 0xffffffff, 0x00000100,
 287         0x9868, 0xffffffff, 0x00000100,
 288         0x8d58, 0xffffffff, 0x00000100,
 289         0x9510, 0xffffffff, 0x00000100,
 290         0x949c, 0xffffffff, 0x00000100,
 291         0x9654, 0xffffffff, 0x00000100,
 292         0x9030, 0xffffffff, 0x00000100,
 293         0x9034, 0xffffffff, 0x00000100,
 294         0x9038, 0xffffffff, 0x00000100,
 295         0x903c, 0xffffffff, 0x00000100,
 296         0x9040, 0xffffffff, 0x00000100,
 297         0xa200, 0xffffffff, 0x00000100,
 298         0xa204, 0xffffffff, 0x00000100,
 299         0xa208, 0xffffffff, 0x00000100,
 300         0xa20c, 0xffffffff, 0x00000100,
 301         0x971c, 0xffffffff, 0x00000100,
 302         0x977c, 0xffffffff, 0x00000100,
 303         0x3f80, 0xffffffff, 0x00000100,
 304         0xa210, 0xffffffff, 0x00000100,
 305         0xa214, 0xffffffff, 0x00000100,
 306         0x4d8, 0xffffffff, 0x00000100,
 307         0x9784, 0xffffffff, 0x00000100,
 308         0x9698, 0xffffffff, 0x00000100,
 309         0x4d4, 0xffffffff, 0x00000200,
 310         0x30cc, 0xffffffff, 0x00000100,
 311         0xd0c0, 0xffffffff, 0xff000100,
 312         0x802c, 0xffffffff, 0x40000000,
 313         0x915c, 0xffffffff, 0x00010000,
 314         0x9160, 0xffffffff, 0x00030002,
 315         0x9178, 0xffffffff, 0x00070000,
 316         0x917c, 0xffffffff, 0x00030002,
 317         0x9180, 0xffffffff, 0x00050004,
 318         0x918c, 0xffffffff, 0x00010006,
 319         0x9190, 0xffffffff, 0x00090008,
 320         0x9194, 0xffffffff, 0x00070000,
 321         0x9198, 0xffffffff, 0x00030002,
 322         0x919c, 0xffffffff, 0x00050004,
 323         0x91a8, 0xffffffff, 0x00010006,
 324         0x91ac, 0xffffffff, 0x00090008,
 325         0x91b0, 0xffffffff, 0x00070000,
 326         0x91b4, 0xffffffff, 0x00030002,
 327         0x91b8, 0xffffffff, 0x00050004,
 328         0x91c4, 0xffffffff, 0x00010006,
 329         0x91c8, 0xffffffff, 0x00090008,
 330         0x91cc, 0xffffffff, 0x00070000,
 331         0x91d0, 0xffffffff, 0x00030002,
 332         0x91d4, 0xffffffff, 0x00050004,
 333         0x91e0, 0xffffffff, 0x00010006,
 334         0x91e4, 0xffffffff, 0x00090008,
 335         0x91e8, 0xffffffff, 0x00000000,
 336         0x91ec, 0xffffffff, 0x00070000,
 337         0x91f0, 0xffffffff, 0x00030002,
 338         0x91f4, 0xffffffff, 0x00050004,
 339         0x9200, 0xffffffff, 0x00010006,
 340         0x9204, 0xffffffff, 0x00090008,
 341         0x9208, 0xffffffff, 0x00070000,
 342         0x920c, 0xffffffff, 0x00030002,
 343         0x9210, 0xffffffff, 0x00050004,
 344         0x921c, 0xffffffff, 0x00010006,
 345         0x9220, 0xffffffff, 0x00090008,
 346         0x9224, 0xffffffff, 0x00070000,
 347         0x9228, 0xffffffff, 0x00030002,
 348         0x922c, 0xffffffff, 0x00050004,
 349         0x9238, 0xffffffff, 0x00010006,
 350         0x923c, 0xffffffff, 0x00090008,
 351         0x9240, 0xffffffff, 0x00070000,
 352         0x9244, 0xffffffff, 0x00030002,
 353         0x9248, 0xffffffff, 0x00050004,
 354         0x9254, 0xffffffff, 0x00010006,
 355         0x9258, 0xffffffff, 0x00090008,
 356         0x925c, 0xffffffff, 0x00070000,
 357         0x9260, 0xffffffff, 0x00030002,
 358         0x9264, 0xffffffff, 0x00050004,
 359         0x9270, 0xffffffff, 0x00010006,
 360         0x9274, 0xffffffff, 0x00090008,
 361         0x9278, 0xffffffff, 0x00070000,
 362         0x927c, 0xffffffff, 0x00030002,
 363         0x9280, 0xffffffff, 0x00050004,
 364         0x928c, 0xffffffff, 0x00010006,
 365         0x9290, 0xffffffff, 0x00090008,
 366         0x9294, 0xffffffff, 0x00000000,
 367         0x929c, 0xffffffff, 0x00000001,
 368         0x802c, 0xffffffff, 0x40010000,
 369         0x915c, 0xffffffff, 0x00010000,
 370         0x9160, 0xffffffff, 0x00030002,
 371         0x9178, 0xffffffff, 0x00070000,
 372         0x917c, 0xffffffff, 0x00030002,
 373         0x9180, 0xffffffff, 0x00050004,
 374         0x918c, 0xffffffff, 0x00010006,
 375         0x9190, 0xffffffff, 0x00090008,
 376         0x9194, 0xffffffff, 0x00070000,
 377         0x9198, 0xffffffff, 0x00030002,
 378         0x919c, 0xffffffff, 0x00050004,
 379         0x91a8, 0xffffffff, 0x00010006,
 380         0x91ac, 0xffffffff, 0x00090008,
 381         0x91b0, 0xffffffff, 0x00070000,
 382         0x91b4, 0xffffffff, 0x00030002,
 383         0x91b8, 0xffffffff, 0x00050004,
 384         0x91c4, 0xffffffff, 0x00010006,
 385         0x91c8, 0xffffffff, 0x00090008,
 386         0x91cc, 0xffffffff, 0x00070000,
 387         0x91d0, 0xffffffff, 0x00030002,
 388         0x91d4, 0xffffffff, 0x00050004,
 389         0x91e0, 0xffffffff, 0x00010006,
 390         0x91e4, 0xffffffff, 0x00090008,
 391         0x91e8, 0xffffffff, 0x00000000,
 392         0x91ec, 0xffffffff, 0x00070000,
 393         0x91f0, 0xffffffff, 0x00030002,
 394         0x91f4, 0xffffffff, 0x00050004,
 395         0x9200, 0xffffffff, 0x00010006,
 396         0x9204, 0xffffffff, 0x00090008,
 397         0x9208, 0xffffffff, 0x00070000,
 398         0x920c, 0xffffffff, 0x00030002,
 399         0x9210, 0xffffffff, 0x00050004,
 400         0x921c, 0xffffffff, 0x00010006,
 401         0x9220, 0xffffffff, 0x00090008,
 402         0x9224, 0xffffffff, 0x00070000,
 403         0x9228, 0xffffffff, 0x00030002,
 404         0x922c, 0xffffffff, 0x00050004,
 405         0x9238, 0xffffffff, 0x00010006,
 406         0x923c, 0xffffffff, 0x00090008,
 407         0x9240, 0xffffffff, 0x00070000,
 408         0x9244, 0xffffffff, 0x00030002,
 409         0x9248, 0xffffffff, 0x00050004,
 410         0x9254, 0xffffffff, 0x00010006,
 411         0x9258, 0xffffffff, 0x00090008,
 412         0x925c, 0xffffffff, 0x00070000,
 413         0x9260, 0xffffffff, 0x00030002,
 414         0x9264, 0xffffffff, 0x00050004,
 415         0x9270, 0xffffffff, 0x00010006,
 416         0x9274, 0xffffffff, 0x00090008,
 417         0x9278, 0xffffffff, 0x00070000,
 418         0x927c, 0xffffffff, 0x00030002,
 419         0x9280, 0xffffffff, 0x00050004,
 420         0x928c, 0xffffffff, 0x00010006,
 421         0x9290, 0xffffffff, 0x00090008,
 422         0x9294, 0xffffffff, 0x00000000,
 423         0x929c, 0xffffffff, 0x00000001,
 424         0x802c, 0xffffffff, 0xc0000000
 425 };
 426
 427 static const u32 redwood_mgcg_init[] =
 428 {
 429         0x802c, 0xffffffff, 0xc0000000,
 430         0x5448, 0xffffffff, 0x00000100,
 431         0x55e4, 0xffffffff, 0x00000100,
 432         0x160c, 0xffffffff, 0x00000100,
 433         0x5644, 0xffffffff, 0x00000100,
 434         0xc164, 0xffffffff, 0x00000100,
 435         0x8a18, 0xffffffff, 0x00000100,
 436         0x897c, 0xffffffff, 0x06000100,
 437         0x8b28, 0xffffffff, 0x00000100,
 438         0x9144, 0xffffffff, 0x00000100,
 439         0x9a60, 0xffffffff, 0x00000100,
 440         0x9868, 0xffffffff, 0x00000100,
 441         0x8d58, 0xffffffff, 0x00000100,
 442         0x9510, 0xffffffff, 0x00000100,
 443         0x949c, 0xffffffff, 0x00000100,
 444         0x9654, 0xffffffff, 0x00000100,
 445         0x9030, 0xffffffff, 0x00000100,
 446         0x9034, 0xffffffff, 0x00000100,
 447         0x9038, 0xffffffff, 0x00000100,
 448         0x903c, 0xffffffff, 0x00000100,
 449         0x9040, 0xffffffff, 0x00000100,
 450         0xa200, 0xffffffff, 0x00000100,
 451         0xa204, 0xffffffff, 0x00000100,
 452         0xa208, 0xffffffff, 0x00000100,
 453         0xa20c, 0xffffffff, 0x00000100,
 454         0x971c, 0xffffffff, 0x00000100,
 455         0x977c, 0xffffffff, 0x00000100,
 456         0x3f80, 0xffffffff, 0x00000100,
 457         0xa210, 0xffffffff, 0x00000100,
 458         0xa214, 0xffffffff, 0x00000100,
 459         0x4d8, 0xffffffff, 0x00000100,
 460         0x9784, 0xffffffff, 0x00000100,
 461         0x9698, 0xffffffff, 0x00000100,
 462         0x4d4, 0xffffffff, 0x00000200,
 463         0x30cc, 0xffffffff, 0x00000100,
 464         0xd0c0, 0xffffffff, 0xff000100,
 465         0x802c, 0xffffffff, 0x40000000,
 466         0x915c, 0xffffffff, 0x00010000,
 467         0x9160, 0xffffffff, 0x00030002,
 468         0x9178, 0xffffffff, 0x00070000,
 469         0x917c, 0xffffffff, 0x00030002,
 470         0x9180, 0xffffffff, 0x00050004,
 471         0x918c, 0xffffffff, 0x00010006,
 472         0x9190, 0xffffffff, 0x00090008,
 473         0x9194, 0xffffffff, 0x00070000,
 474         0x9198, 0xffffffff, 0x00030002,
 475         0x919c, 0xffffffff, 0x00050004,
 476         0x91a8, 0xffffffff, 0x00010006,
 477         0x91ac, 0xffffffff, 0x00090008,
 478         0x91b0, 0xffffffff, 0x00070000,
 479         0x91b4, 0xffffffff, 0x00030002,
 480         0x91b8, 0xffffffff, 0x00050004,
 481         0x91c4, 0xffffffff, 0x00010006,
 482         0x91c8, 0xffffffff, 0x00090008,
 483         0x91cc, 0xffffffff, 0x00070000,
 484         0x91d0, 0xffffffff, 0x00030002,
 485         0x91d4, 0xffffffff, 0x00050004,
 486         0x91e0, 0xffffffff, 0x00010006,
 487         0x91e4, 0xffffffff, 0x00090008,
 488         0x91e8, 0xffffffff, 0x00000000,
 489         0x91ec, 0xffffffff, 0x00070000,
 490         0x91f0, 0xffffffff, 0x00030002,
 491         0x91f4, 0xffffffff, 0x00050004,
 492         0x9200, 0xffffffff, 0x00010006,
 493         0x9204, 0xffffffff, 0x00090008,
 494         0x9294, 0xffffffff, 0x00000000,
 495         0x929c, 0xffffffff, 0x00000001,
 496         0x802c, 0xffffffff, 0xc0000000
 497 };
 498
 499 static const u32 cedar_golden_registers[] =
 500 {
 501         0x3f90, 0xffff0000, 0xff000000,
 502         0x9148, 0xffff0000, 0xff000000,
 503         0x3f94, 0xffff0000, 0xff000000,
 504         0x914c, 0xffff0000, 0xff000000,
 505         0x9b7c, 0xffffffff, 0x00000000,
 506         0x8a14, 0xffffffff, 0x00000007,
 507         0x8b10, 0xffffffff, 0x00000000,
 508         0x960c, 0xffffffff, 0x54763210,
 509         0x88c4, 0xffffffff, 0x000000c2,
 510         0x88d4, 0xffffffff, 0x00000000,
 511         0x8974, 0xffffffff, 0x00000000,
 512         0xc78, 0x00000080, 0x00000080,
 513         0x5eb4, 0xffffffff, 0x00000002,
 514         0x5e78, 0xffffffff, 0x001000f0,
 515         0x6104, 0x01000300, 0x00000000,
 516         0x5bc0, 0x00300000, 0x00000000,
 517         0x7030, 0xffffffff, 0x00000011,
 518         0x7c30, 0xffffffff, 0x00000011,
 519         0x10830, 0xffffffff, 0x00000011,
 520         0x11430, 0xffffffff, 0x00000011,
 521         0xd02c, 0xffffffff, 0x08421000,
 522         0x240c, 0xffffffff, 0x00000380,
 523         0x8b24, 0xffffffff, 0x00ff0fff,
 524         0x28a4c, 0x06000000, 0x06000000,
 525         0x10c, 0x00000001, 0x00000001,
 526         0x8d00, 0xffffffff, 0x100e4848,
 527         0x8d04, 0xffffffff, 0x00164745,
 528         0x8c00, 0xffffffff, 0xe4000003,
 529         0x8c04, 0xffffffff, 0x40600060,
 530         0x8c08, 0xffffffff, 0x001c001c,
 531         0x8cf0, 0xffffffff, 0x08e00410,
 532         0x8c20, 0xffffffff, 0x00800080,
 533         0x8c24, 0xffffffff, 0x00800080,
 534         0x8c18, 0xffffffff, 0x20202078,
 535         0x8c1c, 0xffffffff, 0x00001010,
 536         0x28350, 0xffffffff, 0x00000000,
 537         0xa008, 0xffffffff, 0x00010000,
 538         0x5c4, 0xffffffff, 0x00000001,
 539         0x9508, 0xffffffff, 0x00000002
 540 };
 541
 542 static const u32 cedar_mgcg_init[] =
 543 {
 544         0x802c, 0xffffffff, 0xc0000000,
 545         0x5448, 0xffffffff, 0x00000100,
 546         0x55e4, 0xffffffff, 0x00000100,
 547         0x160c, 0xffffffff, 0x00000100,
 548         0x5644, 0xffffffff, 0x00000100,
 549         0xc164, 0xffffffff, 0x00000100,
 550         0x8a18, 0xffffffff, 0x00000100,
 551         0x897c, 0xffffffff, 0x06000100,
 552         0x8b28, 0xffffffff, 0x00000100,
 553         0x9144, 0xffffffff, 0x00000100,
 554         0x9a60, 0xffffffff, 0x00000100,
 555         0x9868, 0xffffffff, 0x00000100,
 556         0x8d58, 0xffffffff, 0x00000100,
 557         0x9510, 0xffffffff, 0x00000100,
 558         0x949c, 0xffffffff, 0x00000100,
 559         0x9654, 0xffffffff, 0x00000100,
 560         0x9030, 0xffffffff, 0x00000100,
 561         0x9034, 0xffffffff, 0x00000100,
 562         0x9038, 0xffffffff, 0x00000100,
 563         0x903c, 0xffffffff, 0x00000100,
 564         0x9040, 0xffffffff, 0x00000100,
 565         0xa200, 0xffffffff, 0x00000100,
 566         0xa204, 0xffffffff, 0x00000100,
 567         0xa208, 0xffffffff, 0x00000100,
 568         0xa20c, 0xffffffff, 0x00000100,
 569         0x971c, 0xffffffff, 0x00000100,
 570         0x977c, 0xffffffff, 0x00000100,
 571         0x3f80, 0xffffffff, 0x00000100,
 572         0xa210, 0xffffffff, 0x00000100,
 573         0xa214, 0xffffffff, 0x00000100,
 574         0x4d8, 0xffffffff, 0x00000100,
 575         0x9784, 0xffffffff, 0x00000100,
 576         0x9698, 0xffffffff, 0x00000100,
 577         0x4d4, 0xffffffff, 0x00000200,
 578         0x30cc, 0xffffffff, 0x00000100,
 579         0xd0c0, 0xffffffff, 0xff000100,
 580         0x802c, 0xffffffff, 0x40000000,
 581         0x915c, 0xffffffff, 0x00010000,
 582         0x9178, 0xffffffff, 0x00050000,
 583         0x917c, 0xffffffff, 0x00030002,
 584         0x918c, 0xffffffff, 0x00010004,
 585         0x9190, 0xffffffff, 0x00070006,
 586         0x9194, 0xffffffff, 0x00050000,
 587         0x9198, 0xffffffff, 0x00030002,
 588         0x91a8, 0xffffffff, 0x00010004,
 589         0x91ac, 0xffffffff, 0x00070006,
 590         0x91e8, 0xffffffff, 0x00000000,
 591         0x9294, 0xffffffff, 0x00000000,
 592         0x929c, 0xffffffff, 0x00000001,
 593         0x802c, 0xffffffff, 0xc0000000
 594 };
 595
 596 static const u32 juniper_mgcg_init[] =
 597 {
 598         0x802c, 0xffffffff, 0xc0000000,
 599         0x5448, 0xffffffff, 0x00000100,
 600         0x55e4, 0xffffffff, 0x00000100,
 601         0x160c, 0xffffffff, 0x00000100,
 602         0x5644, 0xffffffff, 0x00000100,
 603         0xc164, 0xffffffff, 0x00000100,
 604         0x8a18, 0xffffffff, 0x00000100,
 605         0x897c, 0xffffffff, 0x06000100,
 606         0x8b28, 0xffffffff, 0x00000100,
 607         0x9144, 0xffffffff, 0x00000100,
 608         0x9a60, 0xffffffff, 0x00000100,
 609         0x9868, 0xffffffff, 0x00000100,
 610         0x8d58, 0xffffffff, 0x00000100,
 611         0x9510, 0xffffffff, 0x00000100,
 612         0x949c, 0xffffffff, 0x00000100,
 613         0x9654, 0xffffffff, 0x00000100,
 614         0x9030, 0xffffffff, 0x00000100,
 615         0x9034, 0xffffffff, 0x00000100,
 616         0x9038, 0xffffffff, 0x00000100,
 617         0x903c, 0xffffffff, 0x00000100,
 618         0x9040, 0xffffffff, 0x00000100,
 619         0xa200, 0xffffffff, 0x00000100,
 620         0xa204, 0xffffffff, 0x00000100,
 621         0xa208, 0xffffffff, 0x00000100,
 622         0xa20c, 0xffffffff, 0x00000100,
 623         0x971c, 0xffffffff, 0x00000100,
 624         0xd0c0, 0xffffffff, 0xff000100,
 625         0x802c, 0xffffffff, 0x40000000,
 626         0x915c, 0xffffffff, 0x00010000,
 627         0x9160, 0xffffffff, 0x00030002,
 628         0x9178, 0xffffffff, 0x00070000,
 629         0x917c, 0xffffffff, 0x00030002,
 630         0x9180, 0xffffffff, 0x00050004,
 631         0x918c, 0xffffffff, 0x00010006,
 632         0x9190, 0xffffffff, 0x00090008,
 633         0x9194, 0xffffffff, 0x00070000,
 634         0x9198, 0xffffffff, 0x00030002,
 635         0x919c, 0xffffffff, 0x00050004,
 636         0x91a8, 0xffffffff, 0x00010006,
 637         0x91ac, 0xffffffff, 0x00090008,
 638         0x91b0, 0xffffffff, 0x00070000,
 639         0x91b4, 0xffffffff, 0x00030002,
 640         0x91b8, 0xffffffff, 0x00050004,
 641         0x91c4, 0xffffffff, 0x00010006,
 642         0x91c8, 0xffffffff, 0x00090008,
 643         0x91cc, 0xffffffff, 0x00070000,
 644         0x91d0, 0xffffffff, 0x00030002,
 645         0x91d4, 0xffffffff, 0x00050004,
 646         0x91e0, 0xffffffff, 0x00010006,
 647         0x91e4, 0xffffffff, 0x00090008,
 648         0x91e8, 0xffffffff, 0x00000000,
 649         0x91ec, 0xffffffff, 0x00070000,
 650         0x91f0, 0xffffffff, 0x00030002,
 651         0x91f4, 0xffffffff, 0x00050004,
 652         0x9200, 0xffffffff, 0x00010006,
 653         0x9204, 0xffffffff, 0x00090008,
 654         0x9208, 0xffffffff, 0x00070000,
 655         0x920c, 0xffffffff, 0x00030002,
 656         0x9210, 0xffffffff, 0x00050004,
 657         0x921c, 0xffffffff, 0x00010006,
 658         0x9220, 0xffffffff, 0x00090008,
 659         0x9224, 0xffffffff, 0x00070000,
 660         0x9228, 0xffffffff, 0x00030002,
 661         0x922c, 0xffffffff, 0x00050004,
 662         0x9238, 0xffffffff, 0x00010006,
 663         0x923c, 0xffffffff, 0x00090008,
 664         0x9240, 0xffffffff, 0x00070000,
 665         0x9244, 0xffffffff, 0x00030002,
 666         0x9248, 0xffffffff, 0x00050004,
 667         0x9254, 0xffffffff, 0x00010006,
 668         0x9258, 0xffffffff, 0x00090008,
 669         0x925c, 0xffffffff, 0x00070000,
 670         0x9260, 0xffffffff, 0x00030002,
 671         0x9264, 0xffffffff, 0x00050004,
 672         0x9270, 0xffffffff, 0x00010006,
 673         0x9274, 0xffffffff, 0x00090008,
 674         0x9278, 0xffffffff, 0x00070000,
 675         0x927c, 0xffffffff, 0x00030002,
 676         0x9280, 0xffffffff, 0x00050004,
 677         0x928c, 0xffffffff, 0x00010006,
 678         0x9290, 0xffffffff, 0x00090008,
 679         0x9294, 0xffffffff, 0x00000000,
 680         0x929c, 0xffffffff, 0x00000001,
 681         0x802c, 0xffffffff, 0xc0000000,
 682         0x977c, 0xffffffff, 0x00000100,
 683         0x3f80, 0xffffffff, 0x00000100,
 684         0xa210, 0xffffffff, 0x00000100,
 685         0xa214, 0xffffffff, 0x00000100,
 686         0x4d8, 0xffffffff, 0x00000100,
 687         0x9784, 0xffffffff, 0x00000100,
 688         0x9698, 0xffffffff, 0x00000100,
 689         0x4d4, 0xffffffff, 0x00000200,
 690         0x30cc, 0xffffffff, 0x00000100,
 691         0x802c, 0xffffffff, 0xc0000000
 692 };
 693
 694 static const u32 supersumo_golden_registers[] =
 695 {
 696         0x5eb4, 0xffffffff, 0x00000002,
 697         0x5c4, 0xffffffff, 0x00000001,
 698         0x7030, 0xffffffff, 0x00000011,
 699         0x7c30, 0xffffffff, 0x00000011,
 700         0x6104, 0x01000300, 0x00000000,
 701         0x5bc0, 0x00300000, 0x00000000,
 702         0x8c04, 0xffffffff, 0x40600060,
 703         0x8c08, 0xffffffff, 0x001c001c,
 704         0x8c20, 0xffffffff, 0x00800080,
 705         0x8c24, 0xffffffff, 0x00800080,
 706         0x8c18, 0xffffffff, 0x20202078,
 707         0x8c1c, 0xffffffff, 0x00001010,
 708         0x918c, 0xffffffff, 0x00010006,
 709         0x91a8, 0xffffffff, 0x00010006,
 710         0x91c4, 0xffffffff, 0x00010006,
 711         0x91e0, 0xffffffff, 0x00010006,
 712         0x9200, 0xffffffff, 0x00010006,
 713         0x9150, 0xffffffff, 0x6e944040,
 714         0x917c, 0xffffffff, 0x00030002,
 715         0x9180, 0xffffffff, 0x00050004,
 716         0x9198, 0xffffffff, 0x00030002,
 717         0x919c, 0xffffffff, 0x00050004,
 718         0x91b4, 0xffffffff, 0x00030002,
 719         0x91b8, 0xffffffff, 0x00050004,
 720         0x91d0, 0xffffffff, 0x00030002,
 721         0x91d4, 0xffffffff, 0x00050004,
 722         0x91f0, 0xffffffff, 0x00030002,
 723         0x91f4, 0xffffffff, 0x00050004,
 724         0x915c, 0xffffffff, 0x00010000,
 725         0x9160, 0xffffffff, 0x00030002,
 726         0x3f90, 0xffff0000, 0xff000000,
 727         0x9178, 0xffffffff, 0x00070000,
 728         0x9194, 0xffffffff, 0x00070000,
 729         0x91b0, 0xffffffff, 0x00070000,
 730         0x91cc, 0xffffffff, 0x00070000,
 731         0x91ec, 0xffffffff, 0x00070000,
 732         0x9148, 0xffff0000, 0xff000000,
 733         0x9190, 0xffffffff, 0x00090008,
 734         0x91ac, 0xffffffff, 0x00090008,
 735         0x91c8, 0xffffffff, 0x00090008,
 736         0x91e4, 0xffffffff, 0x00090008,
 737         0x9204, 0xffffffff, 0x00090008,
 738         0x3f94, 0xffff0000, 0xff000000,
 739         0x914c, 0xffff0000, 0xff000000,
 740         0x929c, 0xffffffff, 0x00000001,
 741         0x8a18, 0xffffffff, 0x00000100,
 742         0x8b28, 0xffffffff, 0x00000100,
 743         0x9144, 0xffffffff, 0x00000100,
 744         0x5644, 0xffffffff, 0x00000100,
 745         0x9b7c, 0xffffffff, 0x00000000,
 746         0x8030, 0xffffffff, 0x0000100a,
 747         0x8a14, 0xffffffff, 0x00000007,
 748         0x8b24, 0xffffffff, 0x00ff0fff,
 749         0x8b10, 0xffffffff, 0x00000000,
 750         0x28a4c, 0x06000000, 0x06000000,
 751         0x4d8, 0xffffffff, 0x00000100,
 752         0x913c, 0xffff000f, 0x0100000a,
 753         0x960c, 0xffffffff, 0x54763210,
 754         0x88c4, 0xffffffff, 0x000000c2,
 755         0x88d4, 0xffffffff, 0x00000010,
 756         0x8974, 0xffffffff, 0x00000000,
 757         0xc78, 0x00000080, 0x00000080,
 758         0x5e78, 0xffffffff, 0x001000f0,
 759         0xd02c, 0xffffffff, 0x08421000,
 760         0xa008, 0xffffffff, 0x00010000,
 761         0x8d00, 0xffffffff, 0x100e4848,
 762         0x8d04, 0xffffffff, 0x00164745,
 763         0x8c00, 0xffffffff, 0xe4000003,
 764         0x8cf0, 0x1fffffff, 0x08e00620,
 765         0x28350, 0xffffffff, 0x00000000,
 766         0x9508, 0xffffffff, 0x00000002
 767 };
 768
 769 static const u32 sumo_golden_registers[] =
 770 {
 771         0x900c, 0x00ffffff, 0x0017071f,
 772         0x8c18, 0xffffffff, 0x10101060,
 773         0x8c1c, 0xffffffff, 0x00001010,
 774         0x8c30, 0x0000000f, 0x00000005,
 775         0x9688, 0x0000000f, 0x00000007
 776 };
 777
 778 static const u32 wrestler_golden_registers[] =
 779 {
 780         0x5eb4, 0xffffffff, 0x00000002,
 781         0x5c4, 0xffffffff, 0x00000001,
 782         0x7030, 0xffffffff, 0x00000011,
 783         0x7c30, 0xffffffff, 0x00000011,
 784         0x6104, 0x01000300, 0x00000000,
 785         0x5bc0, 0x00300000, 0x00000000,
 786         0x918c, 0xffffffff, 0x00010006,
 787         0x91a8, 0xffffffff, 0x00010006,
 788         0x9150, 0xffffffff, 0x6e944040,
 789         0x917c, 0xffffffff, 0x00030002,
 790         0x9198, 0xffffffff, 0x00030002,
 791         0x915c, 0xffffffff, 0x00010000,
 792         0x3f90, 0xffff0000, 0xff000000,
 793         0x9178, 0xffffffff, 0x00070000,
 794         0x9194, 0xffffffff, 0x00070000,
 795         0x9148, 0xffff0000, 0xff000000,
 796         0x9190, 0xffffffff, 0x00090008,
 797         0x91ac, 0xffffffff, 0x00090008,
 798         0x3f94, 0xffff0000, 0xff000000,
 799         0x914c, 0xffff0000, 0xff000000,
 800         0x929c, 0xffffffff, 0x00000001,
 801         0x8a18, 0xffffffff, 0x00000100,
 802         0x8b28, 0xffffffff, 0x00000100,
 803         0x9144, 0xffffffff, 0x00000100,
 804         0x9b7c, 0xffffffff, 0x00000000,
 805         0x8030, 0xffffffff, 0x0000100a,
 806         0x8a14, 0xffffffff, 0x00000001,
 807         0x8b24, 0xffffffff, 0x00ff0fff,
 808         0x8b10, 0xffffffff, 0x00000000,
 809         0x28a4c, 0x06000000, 0x06000000,
 810         0x4d8, 0xffffffff, 0x00000100,
 811         0x913c, 0xffff000f, 0x0100000a,
 812         0x960c, 0xffffffff, 0x54763210,
 813         0x88c4, 0xffffffff, 0x000000c2,
 814         0x88d4, 0xffffffff, 0x00000010,
 815         0x8974, 0xffffffff, 0x00000000,
 816         0xc78, 0x00000080, 0x00000080,
 817         0x5e78, 0xffffffff, 0x001000f0,
 818         0xd02c, 0xffffffff, 0x08421000,
 819         0xa008, 0xffffffff, 0x00010000,
 820         0x8d00, 0xffffffff, 0x100e4848,
 821         0x8d04, 0xffffffff, 0x00164745,
 822         0x8c00, 0xffffffff, 0xe4000003,
 823         0x8cf0, 0x1fffffff, 0x08e00410,
 824         0x28350, 0xffffffff, 0x00000000,
 825         0x9508, 0xffffffff, 0x00000002,
 826         0x900c, 0xffffffff, 0x0017071f,
 827         0x8c18, 0xffffffff, 0x10101060,
 828         0x8c1c, 0xffffffff, 0x00001010
 829 };
 830
 831 static const u32 barts_golden_registers[] =
 832 {
 833         0x5eb4, 0xffffffff, 0x00000002,
 834         0x5e78, 0x8f311ff1, 0x001000f0,
 835         0x3f90, 0xffff0000, 0xff000000,
 836         0x9148, 0xffff0000, 0xff000000,
 837         0x3f94, 0xffff0000, 0xff000000,
 838         0x914c, 0xffff0000, 0xff000000,
 839         0xc78, 0x00000080, 0x00000080,
 840         0xbd4, 0x70073777, 0x00010001,
 841         0xd02c, 0xbfffff1f, 0x08421000,
 842         0xd0b8, 0x03773777, 0x02011003,
 843         0x5bc0, 0x00200000, 0x50100000,
 844         0x98f8, 0x33773777, 0x02011003,
 845         0x98fc, 0xffffffff, 0x76543210,
 846         0x7030, 0x31000311, 0x00000011,
 847         0x2f48, 0x00000007, 0x02011003,
 848         0x6b28, 0x00000010, 0x00000012,
 849         0x7728, 0x00000010, 0x00000012,
 850         0x10328, 0x00000010, 0x00000012,
 851         0x10f28, 0x00000010, 0x00000012,
 852         0x11b28, 0x00000010, 0x00000012,
 853         0x12728, 0x00000010, 0x00000012,
 854         0x240c, 0x000007ff, 0x00000380,
 855         0x8a14, 0xf000001f, 0x00000007,
 856         0x8b24, 0x3fff3fff, 0x00ff0fff,
 857         0x8b10, 0x0000ff0f, 0x00000000,
 858         0x28a4c, 0x07ffffff, 0x06000000,
 859         0x10c, 0x00000001, 0x00010003,
 860         0xa02c, 0xffffffff, 0x0000009b,
 861         0x913c, 0x0000000f, 0x0100000a,
 862         0x8d00, 0xffff7f7f, 0x100e4848,
 863         0x8d04, 0x00ffffff, 0x00164745,
 864         0x8c00, 0xfffc0003, 0xe4000003,
 865         0x8c04, 0xf8ff00ff, 0x40600060,
 866         0x8c08, 0x00ff00ff, 0x001c001c,
 867         0x8cf0, 0x1fff1fff, 0x08e00620,
 868         0x8c20, 0x0fff0fff, 0x00800080,
 869         0x8c24, 0x0fff0fff, 0x00800080,
 870         0x8c18, 0xffffffff, 0x20202078,
 871         0x8c1c, 0x0000ffff, 0x00001010,
 872         0x28350, 0x00000f01, 0x00000000,
 873         0x9508, 0x3700001f, 0x00000002,
 874         0x960c, 0xffffffff, 0x54763210,
 875         0x88c4, 0x001f3ae3, 0x000000c2,
 876         0x88d4, 0x0000001f, 0x00000010,
 877         0x8974, 0xffffffff, 0x00000000
 878 };
 879
 880 static const u32 turks_golden_registers[] =
 881 {
 882         0x5eb4, 0xffffffff, 0x00000002,
 883         0x5e78, 0x8f311ff1, 0x001000f0,
 884         0x8c8, 0x00003000, 0x00001070,
 885         0x8cc, 0x000fffff, 0x00040035,
 886         0x3f90, 0xffff0000, 0xfff00000,
 887         0x9148, 0xffff0000, 0xfff00000,
 888         0x3f94, 0xffff0000, 0xfff00000,
 889         0x914c, 0xffff0000, 0xfff00000,
 890         0xc78, 0x00000080, 0x00000080,
 891         0xbd4, 0x00073007, 0x00010002,
 892         0xd02c, 0xbfffff1f, 0x08421000,
 893         0xd0b8, 0x03773777, 0x02010002,
 894         0x5bc0, 0x00200000, 0x50100000,
 895         0x98f8, 0x33773777, 0x00010002,
 896         0x98fc, 0xffffffff, 0x33221100,
 897         0x7030, 0x31000311, 0x00000011,
 898         0x2f48, 0x33773777, 0x00010002,
 899         0x6b28, 0x00000010, 0x00000012,
 900         0x7728, 0x00000010, 0x00000012,
 901         0x10328, 0x00000010, 0x00000012,
 902         0x10f28, 0x00000010, 0x00000012,
 903         0x11b28, 0x00000010, 0x00000012,
 904         0x12728, 0x00000010, 0x00000012,
 905         0x240c, 0x000007ff, 0x00000380,
 906         0x8a14, 0xf000001f, 0x00000007,
 907         0x8b24, 0x3fff3fff, 0x00ff0fff,
 908         0x8b10, 0x0000ff0f, 0x00000000,
 909         0x28a4c, 0x07ffffff, 0x06000000,
 910         0x10c, 0x00000001, 0x00010003,
 911         0xa02c, 0xffffffff, 0x0000009b,
 912         0x913c, 0x0000000f, 0x0100000a,
 913         0x8d00, 0xffff7f7f, 0x100e4848,
 914         0x8d04, 0x00ffffff, 0x00164745,
 915         0x8c00, 0xfffc0003, 0xe4000003,
 916         0x8c04, 0xf8ff00ff, 0x40600060,
 917         0x8c08, 0x00ff00ff, 0x001c001c,
 918         0x8cf0, 0x1fff1fff, 0x08e00410,
 919         0x8c20, 0x0fff0fff, 0x00800080,
 920         0x8c24, 0x0fff0fff, 0x00800080,
 921         0x8c18, 0xffffffff, 0x20202078,
 922         0x8c1c, 0x0000ffff, 0x00001010,
 923         0x28350, 0x00000f01, 0x00000000,
 924         0x9508, 0x3700001f, 0x00000002,
 925         0x960c, 0xffffffff, 0x54763210,
 926         0x88c4, 0x001f3ae3, 0x000000c2,
 927         0x88d4, 0x0000001f, 0x00000010,
 928         0x8974, 0xffffffff, 0x00000000
 929 };
 930
 931 static const u32 caicos_golden_registers[] =
 932 {
 933         0x5eb4, 0xffffffff, 0x00000002,
 934         0x5e78, 0x8f311ff1, 0x001000f0,
 935         0x8c8, 0x00003420, 0x00001450,
 936         0x8cc, 0x000fffff, 0x00040035,
 937         0x3f90, 0xffff0000, 0xfffc0000,
 938         0x9148, 0xffff0000, 0xfffc0000,
 939         0x3f94, 0xffff0000, 0xfffc0000,
 940         0x914c, 0xffff0000, 0xfffc0000,
 941         0xc78, 0x00000080, 0x00000080,
 942         0xbd4, 0x00073007, 0x00010001,
 943         0xd02c, 0xbfffff1f, 0x08421000,
 944         0xd0b8, 0x03773777, 0x02010001,
 945         0x5bc0, 0x00200000, 0x50100000,
 946         0x98f8, 0x33773777, 0x02010001,
 947         0x98fc, 0xffffffff, 0x33221100,
 948         0x7030, 0x31000311, 0x00000011,
 949         0x2f48, 0x33773777, 0x02010001,
 950         0x6b28, 0x00000010, 0x00000012,
 951         0x7728, 0x00000010, 0x00000012,
 952         0x10328, 0x00000010, 0x00000012,
 953         0x10f28, 0x00000010, 0x00000012,
 954         0x11b28, 0x00000010, 0x00000012,
 955         0x12728, 0x00000010, 0x00000012,
 956         0x240c, 0x000007ff, 0x00000380,
 957         0x8a14, 0xf000001f, 0x00000001,
 958         0x8b24, 0x3fff3fff, 0x00ff0fff,
 959         0x8b10, 0x0000ff0f, 0x00000000,
 960         0x28a4c, 0x07ffffff, 0x06000000,
 961         0x10c, 0x00000001, 0x00010003,
 962         0xa02c, 0xffffffff, 0x0000009b,
 963         0x913c, 0x0000000f, 0x0100000a,
 964         0x8d00, 0xffff7f7f, 0x100e4848,
 965         0x8d04, 0x00ffffff, 0x00164745,
 966         0x8c00, 0xfffc0003, 0xe4000003,
 967         0x8c04, 0xf8ff00ff, 0x40600060,
 968         0x8c08, 0x00ff00ff, 0x001c001c,
 969         0x8cf0, 0x1fff1fff, 0x08e00410,
 970         0x8c20, 0x0fff0fff, 0x00800080,
 971         0x8c24, 0x0fff0fff, 0x00800080,
 972         0x8c18, 0xffffffff, 0x20202078,
 973         0x8c1c, 0x0000ffff, 0x00001010,
 974         0x28350, 0x00000f01, 0x00000000,
 975         0x9508, 0x3700001f, 0x00000002,
 976         0x960c, 0xffffffff, 0x54763210,
 977         0x88c4, 0x001f3ae3, 0x000000c2,
 978         0x88d4, 0x0000001f, 0x00000010,
 979         0x8974, 0xffffffff, 0x00000000
 980 };
 981
 982 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 983 {
 984         switch (rdev->family) {
 985         case CHIP_CYPRESS:
 986         case CHIP_HEMLOCK:
 987                 radeon_program_register_sequence(rdev,
 988                                                  evergreen_golden_registers,
 989                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 990                 radeon_program_register_sequence(rdev,
 991                                                  evergreen_golden_registers2,
 992                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 993                 radeon_program_register_sequence(rdev,
 994                                                  cypress_mgcg_init,
 995                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
 996                 break;
 997         case CHIP_JUNIPER:
 998                 radeon_program_register_sequence(rdev,
 999                                                  evergreen_golden_registers,
1000                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1001                 radeon_program_register_sequence(rdev,
1002                                                  evergreen_golden_registers2,
1003                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1004                 radeon_program_register_sequence(rdev,
1005                                                  juniper_mgcg_init,
1006                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
1007                 break;
1008         case CHIP_REDWOOD:
1009                 radeon_program_register_sequence(rdev,
1010                                                  evergreen_golden_registers,
1011                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
1012                 radeon_program_register_sequence(rdev,
1013                                                  evergreen_golden_registers2,
1014                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1015                 radeon_program_register_sequence(rdev,
1016                                                  redwood_mgcg_init,
1017                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
1018                 break;
1019         case CHIP_CEDAR:
1020                 radeon_program_register_sequence(rdev,
1021                                                  cedar_golden_registers,
1022                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
1023                 radeon_program_register_sequence(rdev,
1024                                                  evergreen_golden_registers2,
1025                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1026                 radeon_program_register_sequence(rdev,
1027                                                  cedar_mgcg_init,
1028                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
1029                 break;
1030         case CHIP_PALM:
1031                 radeon_program_register_sequence(rdev,
1032                                                  wrestler_golden_registers,
1033                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
1034                 break;
1035         case CHIP_SUMO:
1036                 radeon_program_register_sequence(rdev,
1037                                                  supersumo_golden_registers,
1038                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1039                 break;
1040         case CHIP_SUMO2:
1041                 radeon_program_register_sequence(rdev,
1042                                                  supersumo_golden_registers,
1043                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
1044                 radeon_program_register_sequence(rdev,
1045                                                  sumo_golden_registers,
1046                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
1047                 break;
1048         case CHIP_BARTS:
1049                 radeon_program_register_sequence(rdev,
1050                                                  barts_golden_registers,
1051                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
1052                 break;
1053         case CHIP_TURKS:
1054                 radeon_program_register_sequence(rdev,
1055                                                  turks_golden_registers,
1056                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
1057                 break;
1058         case CHIP_CAICOS:
1059                 radeon_program_register_sequence(rdev,
1060                                                  caicos_golden_registers,
1061                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1062                 break;
1063         default:
1064                 break;
1065         }
1066 }
1067
1068 /**
1069  * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1070  *
1071  * @rdev: radeon_device pointer
1072  * @reg: register offset in bytes
1073  * @val: register value
1074  *
1075  * Returns 0 for success or -EINVAL for an invalid register
1076  *
1077  */
1078 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1079                                         u32 reg, u32 *val)
1080 {
1081         switch (reg) {
1082         case GRBM_STATUS:
1083         case GRBM_STATUS_SE0:
1084         case GRBM_STATUS_SE1:
1085         case SRBM_STATUS:
1086         case SRBM_STATUS2:
1087         case DMA_STATUS_REG:
1088         case UVD_STATUS:
1089                 *val = RREG32(reg);
1090                 return 0;
1091         default:
1092                 return -EINVAL;
1093         }
1094 }
1095
1096 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1097                              unsigned *bankh, unsigned *mtaspect,
1098                              unsigned *tile_split)
1099 {
1100         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1101         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1102         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1103         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1104         switch (*bankw) {
1105         default:
1106         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1107         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1108         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1109         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1110         }
1111         switch (*bankh) {
1112         default:
1113         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1114         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1115         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1116         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1117         }
1118         switch (*mtaspect) {
1119         default:
1120         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1121         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1122         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1123         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1124         }
1125 }
1126
1127 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1128                               u32 cntl_reg, u32 status_reg)
1129 {
1130         int r, i;
1131         struct atom_clock_dividers dividers;
1132
1133         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1134                                            clock, false, &dividers);
1135         if (r)
1136                 return r;
1137
1138         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1139
1140         for (i = 0; i < 100; i++) {
1141                 if (RREG32(status_reg) & DCLK_STATUS)
1142                         break;
1143                 mdelay(10);
1144         }
1145         if (i == 100)
1146                 return -ETIMEDOUT;
1147
1148         return 0;
1149 }
1150
1151 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1152 {
1153         int r = 0;
1154         u32 cg_scratch = RREG32(CG_SCRATCH1);
1155
1156         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1157         if (r)
1158                 goto done;
1159         cg_scratch &= 0xffff0000;
1160         cg_scratch |= vclk / 100; /* Mhz */
1161
1162         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1163         if (r)
1164                 goto done;
1165         cg_scratch &= 0x0000ffff;
1166         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1167
1168 done:
1169         WREG32(CG_SCRATCH1, cg_scratch);
1170
1171         return r;
1172 }
1173
1174 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1175 {
1176         /* start off with something large */
1177         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1178         int r;
1179
1180         /* bypass vclk and dclk with bclk */
1181         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1182                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1183                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1184
1185         /* put PLL in bypass mode */
1186         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1187
1188         if (!vclk || !dclk) {
1189                 /* keep the Bypass mode, put PLL to sleep */
1190                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1191                 return 0;
1192         }
1193
1194         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1195                                           16384, 0x03FFFFFF, 0, 128, 5,
1196                                           &fb_div, &vclk_div, &dclk_div);
1197         if (r)
1198                 return r;
1199
1200         /* set VCO_MODE to 1 */
1201         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1202
1203         /* toggle UPLL_SLEEP to 1 then back to 0 */
1204         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1205         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1206
1207         /* deassert UPLL_RESET */
1208         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1209
1210         mdelay(1);
1211
1212         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1213         if (r)
1214                 return r;
1215
1216         /* assert UPLL_RESET again */
1217         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1218
1219         /* disable spread spectrum. */
1220         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1221
1222         /* set feedback divider */
1223         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1224
1225         /* set ref divider to 0 */
1226         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1227
1228         if (fb_div < 307200)
1229                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1230         else
1231                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1232
1233         /* set PDIV_A and PDIV_B */
1234         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1235                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1236                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1237
1238         /* give the PLL some time to settle */
1239         mdelay(15);
1240
1241         /* deassert PLL_RESET */
1242         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1243
1244         mdelay(15);
1245
1246         /* switch from bypass mode to normal mode */
1247         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1248
1249         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1250         if (r)
1251                 return r;
1252
1253         /* switch VCLK and DCLK selection */
1254         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1255                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1256                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1257
1258         mdelay(100);
1259
1260         return 0;
1261 }
1262
1263 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1264 {
1265         int readrq;
1266         u16 v;
1267
1268         readrq = pcie_get_readrq(rdev->pdev);
1269         v = ffs(readrq) - 8;
1270         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1271          * to avoid hangs or perfomance issues
1272          */
1273         if ((v == 0) || (v == 6) || (v == 7))
1274                 pcie_set_readrq(rdev->pdev, 512);
1275 }
1276
1277 void dce4_program_fmt(struct drm_encoder *encoder)
1278 {
1279         struct drm_device *dev = encoder->dev;
1280         struct radeon_device *rdev = dev->dev_private;
1281         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1282         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1283         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1284         int bpc = 0;
1285         u32 tmp = 0;
1286         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1287
1288         if (connector) {
1289                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1290                 bpc = radeon_get_monitor_bpc(connector);
1291                 dither = radeon_connector->dither;
1292         }
1293
1294         /* LVDS/eDP FMT is set up by atom */
1295         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1296                 return;
1297
1298         /* not needed for analog */
1299         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1300             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1301                 return;
1302
1303         if (bpc == 0)
1304                 return;
1305
1306         switch (bpc) {
1307         case 6:
1308                 if (dither == RADEON_FMT_DITHER_ENABLE)
1309                         /* XXX sort out optimal dither settings */
1310                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1311                                 FMT_SPATIAL_DITHER_EN);
1312                 else
1313                         tmp |= FMT_TRUNCATE_EN;
1314                 break;
1315         case 8:
1316                 if (dither == RADEON_FMT_DITHER_ENABLE)
1317                         /* XXX sort out optimal dither settings */
1318                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1319                                 FMT_RGB_RANDOM_ENABLE |
1320                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1321                 else
1322                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1323                 break;
1324         case 10:
1325         default:
1326                 /* not needed */
1327                 break;
1328         }
1329
1330         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1331 }
1332
1333 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1334 {
1335         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1336                 return true;
1337         else
1338                 return false;
1339 }
1340
1341 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1342 {
1343         u32 pos1, pos2;
1344
1345         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1346         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1347
1348         if (pos1 != pos2)
1349                 return true;
1350         else
1351                 return false;
1352 }
1353
1354 /**
1355  * dce4_wait_for_vblank - vblank wait asic callback.
1356  *
1357  * @rdev: radeon_device pointer
1358  * @crtc: crtc to wait for vblank on
1359  *
1360  * Wait for vblank on the requested crtc (evergreen+).
1361  */
1362 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1363 {
1364         unsigned i = 0;
1365
1366         if (crtc >= rdev->num_crtc)
1367                 return;
1368
1369         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1370                 return;
1371
1372         /* depending on when we hit vblank, we may be close to active; if so,
1373          * wait for another frame.
1374          */
1375         while (dce4_is_in_vblank(rdev, crtc)) {
1376                 if (i++ % 100 == 0) {
1377                         if (!dce4_is_counter_moving(rdev, crtc))
1378                                 break;
1379                 }
1380         }
1381
1382         while (!dce4_is_in_vblank(rdev, crtc)) {
1383                 if (i++ % 100 == 0) {
1384                         if (!dce4_is_counter_moving(rdev, crtc))
1385                                 break;
1386                 }
1387         }
1388 }
1389
1390 /**
1391  * evergreen_page_flip - pageflip callback.
1392  *
1393  * @rdev: radeon_device pointer
1394  * @crtc_id: crtc to cleanup pageflip on
1395  * @crtc_base: new address of the crtc (GPU MC address)
1396  *
1397  * Triggers the actual pageflip by updating the primary
1398  * surface base address (evergreen+).
1399  */
1400 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1401                          bool async)
1402 {
1403         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1404
1405         /* update the scanout addresses */
1406         WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1407                async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1408         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1409                upper_32_bits(crtc_base));
1410         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1411                (u32)crtc_base);
1412         /* post the write */
1413         RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1414 }
1415
1416 /**
1417  * evergreen_page_flip_pending - check if page flip is still pending
1418  *
1419  * @rdev: radeon_device pointer
1420  * @crtc_id: crtc to check
1421  *
1422  * Returns the current update pending status.
1423  */
1424 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1425 {
1426         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1427
1428         /* Return current update_pending status: */
1429         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1430                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1431 }
1432
1433 /* get temperature in millidegrees */
1434 int evergreen_get_temp(struct radeon_device *rdev)
1435 {
1436         u32 temp, toffset;
1437         int actual_temp = 0;
1438
1439         if (rdev->family == CHIP_JUNIPER) {
1440                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1441                         TOFFSET_SHIFT;
1442                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1443                         TS0_ADC_DOUT_SHIFT;
1444
1445                 if (toffset & 0x100)
1446                         actual_temp = temp / 2 - (0x200 - toffset);
1447                 else
1448                         actual_temp = temp / 2 + toffset;
1449
1450                 actual_temp = actual_temp * 1000;
1451
1452         } else {
1453                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1454                         ASIC_T_SHIFT;
1455
1456                 if (temp & 0x400)
1457                         actual_temp = -256;
1458                 else if (temp & 0x200)
1459                         actual_temp = 255;
1460                 else if (temp & 0x100) {
1461                         actual_temp = temp & 0x1ff;
1462                         actual_temp |= ~0x1ff;
1463                 } else
1464                         actual_temp = temp & 0xff;
1465
1466                 actual_temp = (actual_temp * 1000) / 2;
1467         }
1468
1469         return actual_temp;
1470 }
1471
1472 int sumo_get_temp(struct radeon_device *rdev)
1473 {
1474         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1475         int actual_temp = temp - 49;
1476
1477         return actual_temp * 1000;
1478 }
1479
1480 /**
1481  * sumo_pm_init_profile - Initialize power profiles callback.
1482  *
1483  * @rdev: radeon_device pointer
1484  *
1485  * Initialize the power states used in profile mode
1486  * (sumo, trinity, SI).
1487  * Used for profile mode only.
1488  */
1489 void sumo_pm_init_profile(struct radeon_device *rdev)
1490 {
1491         int idx;
1492
1493         /* default */
1494         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1495         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1496         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1497         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1498
1499         /* low,mid sh/mh */
1500         if (rdev->flags & RADEON_IS_MOBILITY)
1501                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1502         else
1503                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1504
1505         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1506         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1507         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1508         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1509
1510         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1511         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1512         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1513         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1514
1515         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1516         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1517         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1518         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1519
1520         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1521         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1522         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1523         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1524
1525         /* high sh/mh */
1526         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1527         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1528         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1529         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1530         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1531                 rdev->pm.power_state[idx].num_clock_modes - 1;
1532
1533         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1534         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1535         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1536         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1537                 rdev->pm.power_state[idx].num_clock_modes - 1;
1538 }
1539
1540 /**
1541  * btc_pm_init_profile - Initialize power profiles callback.
1542  *
1543  * @rdev: radeon_device pointer
1544  *
1545  * Initialize the power states used in profile mode
1546  * (BTC, cayman).
1547  * Used for profile mode only.
1548  */
1549 void btc_pm_init_profile(struct radeon_device *rdev)
1550 {
1551         int idx;
1552
1553         /* default */
1554         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1555         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1556         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1557         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1558         /* starting with BTC, there is one state that is used for both
1559          * MH and SH.  Difference is that we always use the high clock index for
1560          * mclk.
1561          */
1562         if (rdev->flags & RADEON_IS_MOBILITY)
1563                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1564         else
1565                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1566         /* low sh */
1567         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1568         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1569         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1570         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1571         /* mid sh */
1572         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1573         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1574         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1575         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1576         /* high sh */
1577         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1578         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1579         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1580         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1581         /* low mh */
1582         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1583         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1584         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1585         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1586         /* mid mh */
1587         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1588         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1589         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1590         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1591         /* high mh */
1592         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1593         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1594         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1595         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1596 }
1597
1598 /**
1599  * evergreen_pm_misc - set additional pm hw parameters callback.
1600  *
1601  * @rdev: radeon_device pointer
1602  *
1603  * Set non-clock parameters associated with a power state
1604  * (voltage, etc.) (evergreen+).
1605  */
1606 void evergreen_pm_misc(struct radeon_device *rdev)
1607 {
1608         int req_ps_idx = rdev->pm.requested_power_state_index;
1609         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1610         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1611         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1612
1613         if (voltage->type == VOLTAGE_SW) {
1614                 /* 0xff0x are flags rather then an actual voltage */
1615                 if ((voltage->voltage & 0xff00) == 0xff00)
1616                         return;
1617                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1618                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1619                         rdev->pm.current_vddc = voltage->voltage;
1620                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1621                 }
1622
1623                 /* starting with BTC, there is one state that is used for both
1624                  * MH and SH.  Difference is that we always use the high clock index for
1625                  * mclk and vddci.
1626                  */
1627                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1628                     (rdev->family >= CHIP_BARTS) &&
1629                     rdev->pm.active_crtc_count &&
1630                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1631                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1632                         voltage = &rdev->pm.power_state[req_ps_idx].
1633                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1634
1635                 /* 0xff0x are flags rather then an actual voltage */
1636                 if ((voltage->vddci & 0xff00) == 0xff00)
1637                         return;
1638                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1639                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1640                         rdev->pm.current_vddci = voltage->vddci;
1641                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1642                 }
1643         }
1644 }
1645
1646 /**
1647  * evergreen_pm_prepare - pre-power state change callback.
1648  *
1649  * @rdev: radeon_device pointer
1650  *
1651  * Prepare for a power state change (evergreen+).
1652  */
1653 void evergreen_pm_prepare(struct radeon_device *rdev)
1654 {
1655         struct drm_device *ddev = rdev->ddev;
1656         struct drm_crtc *crtc;
1657         struct radeon_crtc *radeon_crtc;
1658         u32 tmp;
1659
1660         /* disable any active CRTCs */
1661         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1662                 radeon_crtc = to_radeon_crtc(crtc);
1663                 if (radeon_crtc->enabled) {
1664                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1665                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1666                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1667                 }
1668         }
1669 }
1670
1671 /**
1672  * evergreen_pm_finish - post-power state change callback.
1673  *
1674  * @rdev: radeon_device pointer
1675  *
1676  * Clean up after a power state change (evergreen+).
1677  */
1678 void evergreen_pm_finish(struct radeon_device *rdev)
1679 {
1680         struct drm_device *ddev = rdev->ddev;
1681         struct drm_crtc *crtc;
1682         struct radeon_crtc *radeon_crtc;
1683         u32 tmp;
1684
1685         /* enable any active CRTCs */
1686         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1687                 radeon_crtc = to_radeon_crtc(crtc);
1688                 if (radeon_crtc->enabled) {
1689                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1690                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1691                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1692                 }
1693         }
1694 }
1695
1696 /**
1697  * evergreen_hpd_sense - hpd sense callback.
1698  *
1699  * @rdev: radeon_device pointer
1700  * @hpd: hpd (hotplug detect) pin
1701  *
1702  * Checks if a digital monitor is connected (evergreen+).
1703  * Returns true if connected, false if not connected.
1704  */
1705 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1706 {
1707         if (hpd == RADEON_HPD_NONE)
1708                 return false;
1709
1710         return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1711 }
1712
1713 /**
1714  * evergreen_hpd_set_polarity - hpd set polarity callback.
1715  *
1716  * @rdev: radeon_device pointer
1717  * @hpd: hpd (hotplug detect) pin
1718  *
1719  * Set the polarity of the hpd pin (evergreen+).
1720  */
1721 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1722                                 enum radeon_hpd_id hpd)
1723 {
1724         bool connected = evergreen_hpd_sense(rdev, hpd);
1725
1726         if (hpd == RADEON_HPD_NONE)
1727                 return;
1728
1729         if (connected)
1730                 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1731         else
1732                 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1733 }
1734
1735 /**
1736  * evergreen_hpd_init - hpd setup callback.
1737  *
1738  * @rdev: radeon_device pointer
1739  *
1740  * Setup the hpd pins used by the card (evergreen+).
1741  * Enable the pin, set the polarity, and enable the hpd interrupts.
1742  */
1743 void evergreen_hpd_init(struct radeon_device *rdev)
1744 {
1745         struct drm_device *dev = rdev->ddev;
1746         struct drm_connector *connector;
1747         unsigned enabled = 0;
1748         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1749                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1750
1751         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1752                 enum radeon_hpd_id hpd =
1753                         to_radeon_connector(connector)->hpd.hpd;
1754
1755                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1756                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1757                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1758                          * aux dp channel on imac and help (but not completely fix)
1759                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1760                          * also avoid interrupt storms during dpms.
1761                          */
1762                         continue;
1763                 }
1764
1765                 if (hpd == RADEON_HPD_NONE)
1766                         continue;
1767
1768                 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1769                 enabled |= 1 << hpd;
1770
1771                 radeon_hpd_set_polarity(rdev, hpd);
1772         }
1773         radeon_irq_kms_enable_hpd(rdev, enabled);
1774 }
1775
1776 /**
1777  * evergreen_hpd_fini - hpd tear down callback.
1778  *
1779  * @rdev: radeon_device pointer
1780  *
1781  * Tear down the hpd pins used by the card (evergreen+).
1782  * Disable the hpd interrupts.
1783  */
1784 void evergreen_hpd_fini(struct radeon_device *rdev)
1785 {
1786         struct drm_device *dev = rdev->ddev;
1787         struct drm_connector *connector;
1788         unsigned disabled = 0;
1789
1790         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1791                 enum radeon_hpd_id hpd =
1792                         to_radeon_connector(connector)->hpd.hpd;
1793
1794                 if (hpd == RADEON_HPD_NONE)
1795                         continue;
1796
1797                 WREG32(DC_HPDx_CONTROL(hpd), 0);
1798                 disabled |= 1 << hpd;
1799         }
1800         radeon_irq_kms_disable_hpd(rdev, disabled);
1801 }
1802
1803 /* watermark setup */
1804
1805 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1806                                         struct radeon_crtc *radeon_crtc,
1807                                         struct drm_display_mode *mode,
1808                                         struct drm_display_mode *other_mode)
1809 {
1810         u32 tmp, buffer_alloc, i;
1811         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1812         /*
1813          * Line Buffer Setup
1814          * There are 3 line buffers, each one shared by 2 display controllers.
1815          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1816          * the display controllers.  The paritioning is done via one of four
1817          * preset allocations specified in bits 2:0:
1818          * first display controller
1819          *  0 - first half of lb (3840 * 2)
1820          *  1 - first 3/4 of lb (5760 * 2)
1821          *  2 - whole lb (7680 * 2), other crtc must be disabled
1822          *  3 - first 1/4 of lb (1920 * 2)
1823          * second display controller
1824          *  4 - second half of lb (3840 * 2)
1825          *  5 - second 3/4 of lb (5760 * 2)
1826          *  6 - whole lb (7680 * 2), other crtc must be disabled
1827          *  7 - last 1/4 of lb (1920 * 2)
1828          */
1829         /* this can get tricky if we have two large displays on a paired group
1830          * of crtcs.  Ideally for multiple large displays we'd assign them to
1831          * non-linked crtcs for maximum line buffer allocation.
1832          */
1833         if (radeon_crtc->base.enabled && mode) {
1834                 if (other_mode) {
1835                         tmp = 0; /* 1/2 */
1836                         buffer_alloc = 1;
1837                 } else {
1838                         tmp = 2; /* whole */
1839                         buffer_alloc = 2;
1840                 }
1841         } else {
1842                 tmp = 0;
1843                 buffer_alloc = 0;
1844         }
1845
1846         /* second controller of the pair uses second half of the lb */
1847         if (radeon_crtc->crtc_id % 2)
1848                 tmp += 4;
1849         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1850
1851         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1852                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1853                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1854                 for (i = 0; i < rdev->usec_timeout; i++) {
1855                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1856                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1857                                 break;
1858                         udelay(1);
1859                 }
1860         }
1861
1862         if (radeon_crtc->base.enabled && mode) {
1863                 switch (tmp) {
1864                 case 0:
1865                 case 4:
1866                 default:
1867                         if (ASIC_IS_DCE5(rdev))
1868                                 return 4096 * 2;
1869                         else
1870                                 return 3840 * 2;
1871                 case 1:
1872                 case 5:
1873                         if (ASIC_IS_DCE5(rdev))
1874                                 return 6144 * 2;
1875                         else
1876                                 return 5760 * 2;
1877                 case 2:
1878                 case 6:
1879                         if (ASIC_IS_DCE5(rdev))
1880                                 return 8192 * 2;
1881                         else
1882                                 return 7680 * 2;
1883                 case 3:
1884                 case 7:
1885                         if (ASIC_IS_DCE5(rdev))
1886                                 return 2048 * 2;
1887                         else
1888                                 return 1920 * 2;
1889                 }
1890         }
1891
1892         /* controller not enabled, so no lb used */
1893         return 0;
1894 }
1895
1896 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1897 {
1898         u32 tmp = RREG32(MC_SHARED_CHMAP);
1899
1900         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1901         case 0:
1902         default:
1903                 return 1;
1904         case 1:
1905                 return 2;
1906         case 2:
1907                 return 4;
1908         case 3:
1909                 return 8;
1910         }
1911 }
1912
1913 struct evergreen_wm_params {
1914         u32 dram_channels; /* number of dram channels */
1915         u32 yclk;          /* bandwidth per dram data pin in kHz */
1916         u32 sclk;          /* engine clock in kHz */
1917         u32 disp_clk;      /* display clock in kHz */
1918         u32 src_width;     /* viewport width */
1919         u32 active_time;   /* active display time in ns */
1920         u32 blank_time;    /* blank time in ns */
1921         bool interlaced;    /* mode is interlaced */
1922         fixed20_12 vsc;    /* vertical scale ratio */
1923         u32 num_heads;     /* number of active crtcs */
1924         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1925         u32 lb_size;       /* line buffer allocated to pipe */
1926         u32 vtaps;         /* vertical scaler taps */
1927 };
1928
1929 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1930 {
1931         /* Calculate DRAM Bandwidth and the part allocated to display. */
1932         fixed20_12 dram_efficiency; /* 0.7 */
1933         fixed20_12 yclk, dram_channels, bandwidth;
1934         fixed20_12 a;
1935
1936         a.full = dfixed_const(1000);
1937         yclk.full = dfixed_const(wm->yclk);
1938         yclk.full = dfixed_div(yclk, a);
1939         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1940         a.full = dfixed_const(10);
1941         dram_efficiency.full = dfixed_const(7);
1942         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1943         bandwidth.full = dfixed_mul(dram_channels, yclk);
1944         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1945
1946         return dfixed_trunc(bandwidth);
1947 }
1948
1949 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1950 {
1951         /* Calculate DRAM Bandwidth and the part allocated to display. */
1952         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1953         fixed20_12 yclk, dram_channels, bandwidth;
1954         fixed20_12 a;
1955
1956         a.full = dfixed_const(1000);
1957         yclk.full = dfixed_const(wm->yclk);
1958         yclk.full = dfixed_div(yclk, a);
1959         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1960         a.full = dfixed_const(10);
1961         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1962         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1963         bandwidth.full = dfixed_mul(dram_channels, yclk);
1964         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1965
1966         return dfixed_trunc(bandwidth);
1967 }
1968
1969 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1970 {
1971         /* Calculate the display Data return Bandwidth */
1972         fixed20_12 return_efficiency; /* 0.8 */
1973         fixed20_12 sclk, bandwidth;
1974         fixed20_12 a;
1975
1976         a.full = dfixed_const(1000);
1977         sclk.full = dfixed_const(wm->sclk);
1978         sclk.full = dfixed_div(sclk, a);
1979         a.full = dfixed_const(10);
1980         return_efficiency.full = dfixed_const(8);
1981         return_efficiency.full = dfixed_div(return_efficiency, a);
1982         a.full = dfixed_const(32);
1983         bandwidth.full = dfixed_mul(a, sclk);
1984         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1985
1986         return dfixed_trunc(bandwidth);
1987 }
1988
1989 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1990 {
1991         /* Calculate the DMIF Request Bandwidth */
1992         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1993         fixed20_12 disp_clk, bandwidth;
1994         fixed20_12 a;
1995
1996         a.full = dfixed_const(1000);
1997         disp_clk.full = dfixed_const(wm->disp_clk);
1998         disp_clk.full = dfixed_div(disp_clk, a);
1999         a.full = dfixed_const(10);
2000         disp_clk_request_efficiency.full = dfixed_const(8);
2001         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2002         a.full = dfixed_const(32);
2003         bandwidth.full = dfixed_mul(a, disp_clk);
2004         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2005
2006         return dfixed_trunc(bandwidth);
2007 }
2008
2009 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2010 {
2011         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2012         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2013         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2014         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2015
2016         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2017 }
2018
2019 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2020 {
2021         /* Calculate the display mode Average Bandwidth
2022          * DisplayMode should contain the source and destination dimensions,
2023          * timing, etc.
2024          */
2025         fixed20_12 bpp;
2026         fixed20_12 line_time;
2027         fixed20_12 src_width;
2028         fixed20_12 bandwidth;
2029         fixed20_12 a;
2030
2031         a.full = dfixed_const(1000);
2032         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2033         line_time.full = dfixed_div(line_time, a);
2034         bpp.full = dfixed_const(wm->bytes_per_pixel);
2035         src_width.full = dfixed_const(wm->src_width);
2036         bandwidth.full = dfixed_mul(src_width, bpp);
2037         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2038         bandwidth.full = dfixed_div(bandwidth, line_time);
2039
2040         return dfixed_trunc(bandwidth);
2041 }
2042
2043 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2044 {
2045         /* First calcualte the latency in ns */
2046         u32 mc_latency = 2000; /* 2000 ns. */
2047         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2048         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2049         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2050         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2051         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2052                 (wm->num_heads * cursor_line_pair_return_time);
2053         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2054         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2055         fixed20_12 a, b, c;
2056
2057         if (wm->num_heads == 0)
2058                 return 0;
2059
2060         a.full = dfixed_const(2);
2061         b.full = dfixed_const(1);
2062         if ((wm->vsc.full > a.full) ||
2063             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2064             (wm->vtaps >= 5) ||
2065             ((wm->vsc.full >= a.full) && wm->interlaced))
2066                 max_src_lines_per_dst_line = 4;
2067         else
2068                 max_src_lines_per_dst_line = 2;
2069
2070         a.full = dfixed_const(available_bandwidth);
2071         b.full = dfixed_const(wm->num_heads);
2072         a.full = dfixed_div(a, b);
2073
2074         lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2075
2076         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2077         b.full = dfixed_const(1000);
2078         c.full = dfixed_const(lb_fill_bw);
2079         b.full = dfixed_div(c, b);
2080         a.full = dfixed_div(a, b);
2081         line_fill_time = dfixed_trunc(a);
2082
2083         if (line_fill_time < wm->active_time)
2084                 return latency;
2085         else
2086                 return latency + (line_fill_time - wm->active_time);
2087
2088 }
2089
2090 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2091 {
2092         if (evergreen_average_bandwidth(wm) <=
2093             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2094                 return true;
2095         else
2096                 return false;
2097 };
2098
2099 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2100 {
2101         if (evergreen_average_bandwidth(wm) <=
2102             (evergreen_available_bandwidth(wm) / wm->num_heads))
2103                 return true;
2104         else
2105                 return false;
2106 };
2107
2108 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2109 {
2110         u32 lb_partitions = wm->lb_size / wm->src_width;
2111         u32 line_time = wm->active_time + wm->blank_time;
2112         u32 latency_tolerant_lines;
2113         u32 latency_hiding;
2114         fixed20_12 a;
2115
2116         a.full = dfixed_const(1);
2117         if (wm->vsc.full > a.full)
2118                 latency_tolerant_lines = 1;
2119         else {
2120                 if (lb_partitions <= (wm->vtaps + 1))
2121                         latency_tolerant_lines = 1;
2122                 else
2123                         latency_tolerant_lines = 2;
2124         }
2125
2126         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2127
2128         if (evergreen_latency_watermark(wm) <= latency_hiding)
2129                 return true;
2130         else
2131                 return false;
2132 }
2133
2134 static void evergreen_program_watermarks(struct radeon_device *rdev,
2135                                          struct radeon_crtc *radeon_crtc,
2136                                          u32 lb_size, u32 num_heads)
2137 {
2138         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2139         struct evergreen_wm_params wm_low, wm_high;
2140         u32 dram_channels;
2141         u32 active_time;
2142         u32 line_time = 0;
2143         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2144         u32 priority_a_mark = 0, priority_b_mark = 0;
2145         u32 priority_a_cnt = PRIORITY_OFF;
2146         u32 priority_b_cnt = PRIORITY_OFF;
2147         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2148         u32 tmp, arb_control3;
2149         fixed20_12 a, b, c;
2150
2151         if (radeon_crtc->base.enabled && num_heads && mode) {
2152                 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2153                                             (u32)mode->clock);
2154                 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2155                                           (u32)mode->clock);
2156                 line_time = min(line_time, (u32)65535);
2157                 priority_a_cnt = 0;
2158                 priority_b_cnt = 0;
2159                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2160
2161                 /* watermark for high clocks */
2162                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2163                         wm_high.yclk =
2164                                 radeon_dpm_get_mclk(rdev, false) * 10;
2165                         wm_high.sclk =
2166                                 radeon_dpm_get_sclk(rdev, false) * 10;
2167                 } else {
2168                         wm_high.yclk = rdev->pm.current_mclk * 10;
2169                         wm_high.sclk = rdev->pm.current_sclk * 10;
2170                 }
2171
2172                 wm_high.disp_clk = mode->clock;
2173                 wm_high.src_width = mode->crtc_hdisplay;
2174                 wm_high.active_time = active_time;
2175                 wm_high.blank_time = line_time - wm_high.active_time;
2176                 wm_high.interlaced = false;
2177                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2178                         wm_high.interlaced = true;
2179                 wm_high.vsc = radeon_crtc->vsc;
2180                 wm_high.vtaps = 1;
2181                 if (radeon_crtc->rmx_type != RMX_OFF)
2182                         wm_high.vtaps = 2;
2183                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2184                 wm_high.lb_size = lb_size;
2185                 wm_high.dram_channels = dram_channels;
2186                 wm_high.num_heads = num_heads;
2187
2188                 /* watermark for low clocks */
2189                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2190                         wm_low.yclk =
2191                                 radeon_dpm_get_mclk(rdev, true) * 10;
2192                         wm_low.sclk =
2193                                 radeon_dpm_get_sclk(rdev, true) * 10;
2194                 } else {
2195                         wm_low.yclk = rdev->pm.current_mclk * 10;
2196                         wm_low.sclk = rdev->pm.current_sclk * 10;
2197                 }
2198
2199                 wm_low.disp_clk = mode->clock;
2200                 wm_low.src_width = mode->crtc_hdisplay;
2201                 wm_low.active_time = active_time;
2202                 wm_low.blank_time = line_time - wm_low.active_time;
2203                 wm_low.interlaced = false;
2204                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2205                         wm_low.interlaced = true;
2206                 wm_low.vsc = radeon_crtc->vsc;
2207                 wm_low.vtaps = 1;
2208                 if (radeon_crtc->rmx_type != RMX_OFF)
2209                         wm_low.vtaps = 2;
2210                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2211                 wm_low.lb_size = lb_size;
2212                 wm_low.dram_channels = dram_channels;
2213                 wm_low.num_heads = num_heads;
2214
2215                 /* set for high clocks */
2216                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2217                 /* set for low clocks */
2218                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2219
2220                 /* possibly force display priority to high */
2221                 /* should really do this at mode validation time... */
2222                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2223                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2224                     !evergreen_check_latency_hiding(&wm_high) ||
2225                     (rdev->disp_priority == 2)) {
2226                         DRM_DEBUG_KMS("force priority a to high\n");
2227                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2228                 }
2229                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2230                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2231                     !evergreen_check_latency_hiding(&wm_low) ||
2232                     (rdev->disp_priority == 2)) {
2233                         DRM_DEBUG_KMS("force priority b to high\n");
2234                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2235                 }
2236
2237                 a.full = dfixed_const(1000);
2238                 b.full = dfixed_const(mode->clock);
2239                 b.full = dfixed_div(b, a);
2240                 c.full = dfixed_const(latency_watermark_a);
2241                 c.full = dfixed_mul(c, b);
2242                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2243                 c.full = dfixed_div(c, a);
2244                 a.full = dfixed_const(16);
2245                 c.full = dfixed_div(c, a);
2246                 priority_a_mark = dfixed_trunc(c);
2247                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2248
2249                 a.full = dfixed_const(1000);
2250                 b.full = dfixed_const(mode->clock);
2251                 b.full = dfixed_div(b, a);
2252                 c.full = dfixed_const(latency_watermark_b);
2253                 c.full = dfixed_mul(c, b);
2254                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2255                 c.full = dfixed_div(c, a);
2256                 a.full = dfixed_const(16);
2257                 c.full = dfixed_div(c, a);
2258                 priority_b_mark = dfixed_trunc(c);
2259                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2260
2261                 /* Save number of lines the linebuffer leads before the scanout */
2262                 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2263         }
2264
2265         /* select wm A */
2266         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2267         tmp = arb_control3;
2268         tmp &= ~LATENCY_WATERMARK_MASK(3);
2269         tmp |= LATENCY_WATERMARK_MASK(1);
2270         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2271         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2272                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2273                 LATENCY_HIGH_WATERMARK(line_time)));
2274         /* select wm B */
2275         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2276         tmp &= ~LATENCY_WATERMARK_MASK(3);
2277         tmp |= LATENCY_WATERMARK_MASK(2);
2278         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2279         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2280                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2281                 LATENCY_HIGH_WATERMARK(line_time)));
2282         /* restore original selection */
2283         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2284
2285         /* write the priority marks */
2286         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2287         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2288
2289         /* save values for DPM */
2290         radeon_crtc->line_time = line_time;
2291         radeon_crtc->wm_high = latency_watermark_a;
2292         radeon_crtc->wm_low = latency_watermark_b;
2293 }
2294
2295 /**
2296  * evergreen_bandwidth_update - update display watermarks callback.
2297  *
2298  * @rdev: radeon_device pointer
2299  *
2300  * Update the display watermarks based on the requested mode(s)
2301  * (evergreen+).
2302  */
2303 void evergreen_bandwidth_update(struct radeon_device *rdev)
2304 {
2305         struct drm_display_mode *mode0 = NULL;
2306         struct drm_display_mode *mode1 = NULL;
2307         u32 num_heads = 0, lb_size;
2308         int i;
2309
2310         if (!rdev->mode_info.mode_config_initialized)
2311                 return;
2312
2313         radeon_update_display_priority(rdev);
2314
2315         for (i = 0; i < rdev->num_crtc; i++) {
2316                 if (rdev->mode_info.crtcs[i]->base.enabled)
2317                         num_heads++;
2318         }
2319         for (i = 0; i < rdev->num_crtc; i += 2) {
2320                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2321                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2322                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2323                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2324                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2325                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2326         }
2327 }
2328
2329 /**
2330  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2331  *
2332  * @rdev: radeon_device pointer
2333  *
2334  * Wait for the MC (memory controller) to be idle.
2335  * (evergreen+).
2336  * Returns 0 if the MC is idle, -1 if not.
2337  */
2338 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2339 {
2340         unsigned i;
2341         u32 tmp;
2342
2343         for (i = 0; i < rdev->usec_timeout; i++) {
2344                 /* read MC_STATUS */
2345                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2346                 if (!tmp)
2347                         return 0;
2348                 udelay(1);
2349         }
2350         return -1;
2351 }
2352
2353 /*
2354  * GART
2355  */
2356 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2357 {
2358         unsigned i;
2359         u32 tmp;
2360
2361         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2362
2363         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2364         for (i = 0; i < rdev->usec_timeout; i++) {
2365                 /* read MC_STATUS */
2366                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2367                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2368                 if (tmp == 2) {
2369                         pr_warn("[drm] r600 flush TLB failed\n");
2370                         return;
2371                 }
2372                 if (tmp) {
2373                         return;
2374                 }
2375                 udelay(1);
2376         }
2377 }
2378
2379 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2380 {
2381         u32 tmp;
2382         int r;
2383
2384         if (rdev->gart.robj == NULL) {
2385                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2386                 return -EINVAL;
2387         }
2388         r = radeon_gart_table_vram_pin(rdev);
2389         if (r)
2390                 return r;
2391         /* Setup L2 cache */
2392         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2393                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2394                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2395         WREG32(VM_L2_CNTL2, 0);
2396         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2397         /* Setup TLB control */
2398         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2399                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2400                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2401                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2402         if (rdev->flags & RADEON_IS_IGP) {
2403                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2404                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2405                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2406         } else {
2407                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2408                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2409                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2410                 if ((rdev->family == CHIP_JUNIPER) ||
2411                     (rdev->family == CHIP_CYPRESS) ||
2412                     (rdev->family == CHIP_HEMLOCK) ||
2413                     (rdev->family == CHIP_BARTS))
2414                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2415         }
2416         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2417         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2418         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2419         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2420         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2421         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2422         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2423         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2424                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2425         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2426                         (u32)(rdev->dummy_page.addr >> 12));
2427         WREG32(VM_CONTEXT1_CNTL, 0);
2428
2429         evergreen_pcie_gart_tlb_flush(rdev);
2430         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2431                  (unsigned)(rdev->mc.gtt_size >> 20),
2432                  (unsigned long long)rdev->gart.table_addr);
2433         rdev->gart.ready = true;
2434         return 0;
2435 }
2436
2437 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2438 {
2439         u32 tmp;
2440
2441         /* Disable all tables */
2442         WREG32(VM_CONTEXT0_CNTL, 0);
2443         WREG32(VM_CONTEXT1_CNTL, 0);
2444
2445         /* Setup L2 cache */
2446         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2447                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2448         WREG32(VM_L2_CNTL2, 0);
2449         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2450         /* Setup TLB control */
2451         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2452         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2453         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2454         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2455         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2456         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2457         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2458         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2459         radeon_gart_table_vram_unpin(rdev);
2460 }
2461
2462 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2463 {
2464         evergreen_pcie_gart_disable(rdev);
2465         radeon_gart_table_vram_free(rdev);
2466         radeon_gart_fini(rdev);
2467 }
2468
2469
2470 static void evergreen_agp_enable(struct radeon_device *rdev)
2471 {
2472         u32 tmp;
2473
2474         /* Setup L2 cache */
2475         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2476                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2477                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2478         WREG32(VM_L2_CNTL2, 0);
2479         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2480         /* Setup TLB control */
2481         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2482                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2483                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2484                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2485         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2486         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2487         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2488         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2489         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2490         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2491         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2492         WREG32(VM_CONTEXT0_CNTL, 0);
2493         WREG32(VM_CONTEXT1_CNTL, 0);
2494 }
2495
2496 static const unsigned ni_dig_offsets[] =
2497 {
2498         NI_DIG0_REGISTER_OFFSET,
2499         NI_DIG1_REGISTER_OFFSET,
2500         NI_DIG2_REGISTER_OFFSET,
2501         NI_DIG3_REGISTER_OFFSET,
2502         NI_DIG4_REGISTER_OFFSET,
2503         NI_DIG5_REGISTER_OFFSET
2504 };
2505
2506 static const unsigned ni_tx_offsets[] =
2507 {
2508         NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2509         NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2510         NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2511         NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2512         NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2513         NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2514 };
2515
2516 static const unsigned evergreen_dp_offsets[] =
2517 {
2518         EVERGREEN_DP0_REGISTER_OFFSET,
2519         EVERGREEN_DP1_REGISTER_OFFSET,
2520         EVERGREEN_DP2_REGISTER_OFFSET,
2521         EVERGREEN_DP3_REGISTER_OFFSET,
2522         EVERGREEN_DP4_REGISTER_OFFSET,
2523         EVERGREEN_DP5_REGISTER_OFFSET
2524 };
2525
2526 static const unsigned evergreen_disp_int_status[] =
2527 {
2528         DISP_INTERRUPT_STATUS,
2529         DISP_INTERRUPT_STATUS_CONTINUE,
2530         DISP_INTERRUPT_STATUS_CONTINUE2,
2531         DISP_INTERRUPT_STATUS_CONTINUE3,
2532         DISP_INTERRUPT_STATUS_CONTINUE4,
2533         DISP_INTERRUPT_STATUS_CONTINUE5
2534 };
2535
2536 /*
2537  * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2538  * We go from crtc to connector and it is not relible  since it
2539  * should be an opposite direction .If crtc is enable then
2540  * find the dig_fe which selects this crtc and insure that it enable.
2541  * if such dig_fe is found then find dig_be which selects found dig_be and
2542  * insure that it enable and in DP_SST mode.
2543  * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2544  * from dp symbols clocks .
2545  */
2546 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2547                                                unsigned crtc_id, unsigned *ret_dig_fe)
2548 {
2549         unsigned i;
2550         unsigned dig_fe;
2551         unsigned dig_be;
2552         unsigned dig_en_be;
2553         unsigned uniphy_pll;
2554         unsigned digs_fe_selected;
2555         unsigned dig_be_mode;
2556         unsigned dig_fe_mask;
2557         bool is_enabled = false;
2558         bool found_crtc = false;
2559
2560         /* loop through all running dig_fe to find selected crtc */
2561         for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2562                 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2563                 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2564                     crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2565                         /* found running pipe */
2566                         found_crtc = true;
2567                         dig_fe_mask = 1 << i;
2568                         dig_fe = i;
2569                         break;
2570                 }
2571         }
2572
2573         if (found_crtc) {
2574                 /* loop through all running dig_be to find selected dig_fe */
2575                 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2576                         dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2577                         /* if dig_fe_selected by dig_be? */
2578                         digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2579                         dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2580                         if (dig_fe_mask &  digs_fe_selected &&
2581                             /* if dig_be in sst mode? */
2582                             dig_be_mode == NI_DIG_BE_DPSST) {
2583                                 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2584                                                    ni_dig_offsets[i]);
2585                                 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2586                                                     ni_tx_offsets[i]);
2587                                 /* dig_be enable and tx is running */
2588                                 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2589                                     dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2590                                     uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2591                                         is_enabled = true;
2592                                         *ret_dig_fe = dig_fe;
2593                                         break;
2594                                 }
2595                         }
2596                 }
2597         }
2598
2599         return is_enabled;
2600 }
2601
2602 /*
2603  * Blank dig when in dp sst mode
2604  * Dig ignores crtc timing
2605  */
2606 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2607                                       unsigned dig_fe)
2608 {
2609         unsigned stream_ctrl;
2610         unsigned fifo_ctrl;
2611         unsigned counter = 0;
2612
2613         if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2614                 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2615                 return;
2616         }
2617
2618         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2619                              evergreen_dp_offsets[dig_fe]);
2620         if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2621                 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2622                 return;
2623         }
2624
2625         stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2626         WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2627                evergreen_dp_offsets[dig_fe], stream_ctrl);
2628
2629         stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2630                              evergreen_dp_offsets[dig_fe]);
2631         while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2632                 msleep(1);
2633                 counter++;
2634                 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2635                                      evergreen_dp_offsets[dig_fe]);
2636         }
2637         if (counter >= 32 )
2638                 DRM_ERROR("counter exceeds %d\n", counter);
2639
2640         fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2641         fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2642         WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2643
2644 }
2645
2646 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2647 {
2648         u32 crtc_enabled, tmp, frame_count, blackout;
2649         int i, j;
2650         unsigned dig_fe;
2651
2652         if (!ASIC_IS_NODCE(rdev)) {
2653                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2654                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2655
2656                 /* disable VGA render */
2657                 WREG32(VGA_RENDER_CONTROL, 0);
2658         }
2659         /* blank the display controllers */
2660         for (i = 0; i < rdev->num_crtc; i++) {
2661                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2662                 if (crtc_enabled) {
2663                         save->crtc_enabled[i] = true;
2664                         if (ASIC_IS_DCE6(rdev)) {
2665                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2666                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2667                                         radeon_wait_for_vblank(rdev, i);
2668                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2669                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2670                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2671                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2672                                 }
2673                         } else {
2674                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2675                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2676                                         radeon_wait_for_vblank(rdev, i);
2677                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2678                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2679                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2680                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2681                                 }
2682                         }
2683                         /* wait for the next frame */
2684                         frame_count = radeon_get_vblank_counter(rdev, i);
2685                         for (j = 0; j < rdev->usec_timeout; j++) {
2686                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2687                                         break;
2688                                 udelay(1);
2689                         }
2690                         /*we should disable dig if it drives dp sst*/
2691                         /*but we are in radeon_device_init and the topology is unknown*/
2692                         /*and it is available after radeon_modeset_init*/
2693                         /*the following method radeon_atom_encoder_dpms_dig*/
2694                         /*does the job if we initialize it properly*/
2695                         /*for now we do it this manually*/
2696                         /**/
2697                         if (ASIC_IS_DCE5(rdev) &&
2698                             evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2699                                 evergreen_blank_dp_output(rdev, dig_fe);
2700                         /*we could remove 6 lines below*/
2701                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2702                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2703                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2704                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2705                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2706                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2707                         save->crtc_enabled[i] = false;
2708                         /* ***** */
2709                 } else {
2710                         save->crtc_enabled[i] = false;
2711                 }
2712         }
2713
2714         radeon_mc_wait_for_idle(rdev);
2715
2716         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2717         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2718                 /* Block CPU access */
2719                 WREG32(BIF_FB_EN, 0);
2720                 /* blackout the MC */
2721                 blackout &= ~BLACKOUT_MODE_MASK;
2722                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2723         }
2724         /* wait for the MC to settle */
2725         udelay(100);
2726
2727         /* lock double buffered regs */
2728         for (i = 0; i < rdev->num_crtc; i++) {
2729                 if (save->crtc_enabled[i]) {
2730                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2731                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2732                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2733                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2734                         }
2735                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2736                         if (!(tmp & 1)) {
2737                                 tmp |= 1;
2738                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2739                         }
2740                 }
2741         }
2742 }
2743
2744 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2745 {
2746         u32 tmp, frame_count;
2747         int i, j;
2748
2749         /* update crtc base addresses */
2750         for (i = 0; i < rdev->num_crtc; i++) {
2751                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2752                        upper_32_bits(rdev->mc.vram_start));
2753                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2754                        upper_32_bits(rdev->mc.vram_start));
2755                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2756                        (u32)rdev->mc.vram_start);
2757                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2758                        (u32)rdev->mc.vram_start);
2759         }
2760
2761         if (!ASIC_IS_NODCE(rdev)) {
2762                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2763                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2764         }
2765
2766         /* unlock regs and wait for update */
2767         for (i = 0; i < rdev->num_crtc; i++) {
2768                 if (save->crtc_enabled[i]) {
2769                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2770                         if ((tmp & 0x7) != 0) {
2771                                 tmp &= ~0x7;
2772                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2773                         }
2774                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2775                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2776                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2777                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2778                         }
2779                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2780                         if (tmp & 1) {
2781                                 tmp &= ~1;
2782                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2783                         }
2784                         for (j = 0; j < rdev->usec_timeout; j++) {
2785                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2786                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2787                                         break;
2788                                 udelay(1);
2789                         }
2790                 }
2791         }
2792
2793         /* unblackout the MC */
2794         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2795         tmp &= ~BLACKOUT_MODE_MASK;
2796         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2797         /* allow CPU access */
2798         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2799
2800         for (i = 0; i < rdev->num_crtc; i++) {
2801                 if (save->crtc_enabled[i]) {
2802                         if (ASIC_IS_DCE6(rdev)) {
2803                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2804                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2805                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2806                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2807                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2808                         } else {
2809                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2810                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2811                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2812                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2813                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2814                         }
2815                         /* wait for the next frame */
2816                         frame_count = radeon_get_vblank_counter(rdev, i);
2817                         for (j = 0; j < rdev->usec_timeout; j++) {
2818                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2819                                         break;
2820                                 udelay(1);
2821                         }
2822                 }
2823         }
2824         if (!ASIC_IS_NODCE(rdev)) {
2825                 /* Unlock vga access */
2826                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2827                 mdelay(1);
2828                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2829         }
2830 }
2831
2832 void evergreen_mc_program(struct radeon_device *rdev)
2833 {
2834         struct evergreen_mc_save save;
2835         u32 tmp;
2836         int i, j;
2837
2838         /* Initialize HDP */
2839         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2840                 WREG32((0x2c14 + j), 0x00000000);
2841                 WREG32((0x2c18 + j), 0x00000000);
2842                 WREG32((0x2c1c + j), 0x00000000);
2843                 WREG32((0x2c20 + j), 0x00000000);
2844                 WREG32((0x2c24 + j), 0x00000000);
2845         }
2846         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2847
2848         evergreen_mc_stop(rdev, &save);
2849         if (evergreen_mc_wait_for_idle(rdev)) {
2850                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2851         }
2852         /* Lockout access through VGA aperture*/
2853         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2854         /* Update configuration */
2855         if (rdev->flags & RADEON_IS_AGP) {
2856                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2857                         /* VRAM before AGP */
2858                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2859                                 rdev->mc.vram_start >> 12);
2860                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2861                                 rdev->mc.gtt_end >> 12);
2862                 } else {
2863                         /* VRAM after AGP */
2864                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2865                                 rdev->mc.gtt_start >> 12);
2866                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2867                                 rdev->mc.vram_end >> 12);
2868                 }
2869         } else {
2870                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2871                         rdev->mc.vram_start >> 12);
2872                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2873                         rdev->mc.vram_end >> 12);
2874         }
2875         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2876         /* llano/ontario only */
2877         if ((rdev->family == CHIP_PALM) ||
2878             (rdev->family == CHIP_SUMO) ||
2879             (rdev->family == CHIP_SUMO2)) {
2880                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2881                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2882                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2883                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2884         }
2885         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2886         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2887         WREG32(MC_VM_FB_LOCATION, tmp);
2888         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2889         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2890         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2891         if (rdev->flags & RADEON_IS_AGP) {
2892                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2893                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2894                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2895         } else {
2896                 WREG32(MC_VM_AGP_BASE, 0);
2897                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2898                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2899         }
2900         if (evergreen_mc_wait_for_idle(rdev)) {
2901                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2902         }
2903         evergreen_mc_resume(rdev, &save);
2904         /* we need to own VRAM, so turn off the VGA renderer here
2905          * to stop it overwriting our objects */
2906         rv515_vga_render_disable(rdev);
2907 }
2908
2909 /*
2910  * CP.
2911  */
2912 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2913 {
2914         struct radeon_ring *ring = &rdev->ring[ib->ring];
2915         u32 next_rptr;
2916
2917         /* set to DX10/11 mode */
2918         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2919         radeon_ring_write(ring, 1);
2920
2921         if (ring->rptr_save_reg) {
2922                 next_rptr = ring->wptr + 3 + 4;
2923                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2924                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2925                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2926                 radeon_ring_write(ring, next_rptr);
2927         } else if (rdev->wb.enabled) {
2928                 next_rptr = ring->wptr + 5 + 4;
2929                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2930                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2931                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2932                 radeon_ring_write(ring, next_rptr);
2933                 radeon_ring_write(ring, 0);
2934         }
2935
2936         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2937         radeon_ring_write(ring,
2938 #ifdef __BIG_ENDIAN
2939                           (2 << 0) |
2940 #endif
2941                           (ib->gpu_addr & 0xFFFFFFFC));
2942         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2943         radeon_ring_write(ring, ib->length_dw);
2944 }
2945
2946
2947 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2948 {
2949         const __be32 *fw_data;
2950         int i;
2951
2952         if (!rdev->me_fw || !rdev->pfp_fw)
2953                 return -EINVAL;
2954
2955         r700_cp_stop(rdev);
2956         WREG32(CP_RB_CNTL,
2957 #ifdef __BIG_ENDIAN
2958                BUF_SWAP_32BIT |
2959 #endif
2960                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2961
2962         fw_data = (const __be32 *)rdev->pfp_fw->data;
2963         WREG32(CP_PFP_UCODE_ADDR, 0);
2964         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2965                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2966         WREG32(CP_PFP_UCODE_ADDR, 0);
2967
2968         fw_data = (const __be32 *)rdev->me_fw->data;
2969         WREG32(CP_ME_RAM_WADDR, 0);
2970         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2971                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2972
2973         WREG32(CP_PFP_UCODE_ADDR, 0);
2974         WREG32(CP_ME_RAM_WADDR, 0);
2975         WREG32(CP_ME_RAM_RADDR, 0);
2976         return 0;
2977 }
2978
2979 static int evergreen_cp_start(struct radeon_device *rdev)
2980 {
2981         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2982         int r, i;
2983         uint32_t cp_me;
2984
2985         r = radeon_ring_lock(rdev, ring, 7);
2986         if (r) {
2987                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2988                 return r;
2989         }
2990         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2991         radeon_ring_write(ring, 0x1);
2992         radeon_ring_write(ring, 0x0);
2993         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2994         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2995         radeon_ring_write(ring, 0);
2996         radeon_ring_write(ring, 0);
2997         radeon_ring_unlock_commit(rdev, ring, false);
2998
2999         cp_me = 0xff;
3000         WREG32(CP_ME_CNTL, cp_me);
3001
3002         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3003         if (r) {
3004                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3005                 return r;
3006         }
3007
3008         /* setup clear context state */
3009         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3010         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3011
3012         for (i = 0; i < evergreen_default_size; i++)
3013                 radeon_ring_write(ring, evergreen_default_state[i]);
3014
3015         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3016         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3017
3018         /* set clear context state */
3019         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3020         radeon_ring_write(ring, 0);
3021
3022         /* SQ_VTX_BASE_VTX_LOC */
3023         radeon_ring_write(ring, 0xc0026f00);
3024         radeon_ring_write(ring, 0x00000000);
3025         radeon_ring_write(ring, 0x00000000);
3026         radeon_ring_write(ring, 0x00000000);
3027
3028         /* Clear consts */
3029         radeon_ring_write(ring, 0xc0036f00);
3030         radeon_ring_write(ring, 0x00000bc4);
3031         radeon_ring_write(ring, 0xffffffff);
3032         radeon_ring_write(ring, 0xffffffff);
3033         radeon_ring_write(ring, 0xffffffff);
3034
3035         radeon_ring_write(ring, 0xc0026900);
3036         radeon_ring_write(ring, 0x00000316);
3037         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3038         radeon_ring_write(ring, 0x00000010); /*  */
3039
3040         radeon_ring_unlock_commit(rdev, ring, false);
3041
3042         return 0;
3043 }
3044
3045 static int evergreen_cp_resume(struct radeon_device *rdev)
3046 {
3047         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3048         u32 tmp;
3049         u32 rb_bufsz;
3050         int r;
3051
3052         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3053         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3054                                  SOFT_RESET_PA |
3055                                  SOFT_RESET_SH |
3056                                  SOFT_RESET_VGT |
3057                                  SOFT_RESET_SPI |
3058                                  SOFT_RESET_SX));
3059         RREG32(GRBM_SOFT_RESET);
3060         mdelay(15);
3061         WREG32(GRBM_SOFT_RESET, 0);
3062         RREG32(GRBM_SOFT_RESET);
3063
3064         /* Set ring buffer size */
3065         rb_bufsz = order_base_2(ring->ring_size / 8);
3066         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3067 #ifdef __BIG_ENDIAN
3068         tmp |= BUF_SWAP_32BIT;
3069 #endif
3070         WREG32(CP_RB_CNTL, tmp);
3071         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3072         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3073
3074         /* Set the write pointer delay */
3075         WREG32(CP_RB_WPTR_DELAY, 0);
3076
3077         /* Initialize the ring buffer's read and write pointers */
3078         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3079         WREG32(CP_RB_RPTR_WR, 0);
3080         ring->wptr = 0;
3081         WREG32(CP_RB_WPTR, ring->wptr);
3082
3083         /* set the wb address whether it's enabled or not */
3084         WREG32(CP_RB_RPTR_ADDR,
3085                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3086         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3087         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3088
3089         if (rdev->wb.enabled)
3090                 WREG32(SCRATCH_UMSK, 0xff);
3091         else {
3092                 tmp |= RB_NO_UPDATE;
3093                 WREG32(SCRATCH_UMSK, 0);
3094         }
3095
3096         mdelay(1);
3097         WREG32(CP_RB_CNTL, tmp);
3098
3099         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3100         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3101
3102         evergreen_cp_start(rdev);
3103         ring->ready = true;
3104         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3105         if (r) {
3106                 ring->ready = false;
3107                 return r;
3108         }
3109         return 0;
3110 }
3111
3112 /*
3113  * Core functions
3114  */
3115 static void evergreen_gpu_init(struct radeon_device *rdev)
3116 {
3117         u32 gb_addr_config;
3118         u32 mc_shared_chmap, mc_arb_ramcfg;
3119         u32 sx_debug_1;
3120         u32 smx_dc_ctl0;
3121         u32 sq_config;
3122         u32 sq_lds_resource_mgmt;
3123         u32 sq_gpr_resource_mgmt_1;
3124         u32 sq_gpr_resource_mgmt_2;
3125         u32 sq_gpr_resource_mgmt_3;
3126         u32 sq_thread_resource_mgmt;
3127         u32 sq_thread_resource_mgmt_2;
3128         u32 sq_stack_resource_mgmt_1;
3129         u32 sq_stack_resource_mgmt_2;
3130         u32 sq_stack_resource_mgmt_3;
3131         u32 vgt_cache_invalidation;
3132         u32 hdp_host_path_cntl, tmp;
3133         u32 disabled_rb_mask;
3134         int i, j, ps_thread_count;
3135
3136         switch (rdev->family) {
3137         case CHIP_CYPRESS:
3138         case CHIP_HEMLOCK:
3139                 rdev->config.evergreen.num_ses = 2;
3140                 rdev->config.evergreen.max_pipes = 4;
3141                 rdev->config.evergreen.max_tile_pipes = 8;
3142                 rdev->config.evergreen.max_simds = 10;
3143                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3144                 rdev->config.evergreen.max_gprs = 256;
3145                 rdev->config.evergreen.max_threads = 248;
3146                 rdev->config.evergreen.max_gs_threads = 32;
3147                 rdev->config.evergreen.max_stack_entries = 512;
3148                 rdev->config.evergreen.sx_num_of_sets = 4;
3149                 rdev->config.evergreen.sx_max_export_size = 256;
3150                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3151                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3152                 rdev->config.evergreen.max_hw_contexts = 8;
3153                 rdev->config.evergreen.sq_num_cf_insts = 2;
3154
3155                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3156                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3157                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3158                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3159                 break;
3160         case CHIP_JUNIPER:
3161                 rdev->config.evergreen.num_ses = 1;
3162                 rdev->config.evergreen.max_pipes = 4;
3163                 rdev->config.evergreen.max_tile_pipes = 4;
3164                 rdev->config.evergreen.max_simds = 10;
3165                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3166                 rdev->config.evergreen.max_gprs = 256;
3167                 rdev->config.evergreen.max_threads = 248;
3168                 rdev->config.evergreen.max_gs_threads = 32;
3169                 rdev->config.evergreen.max_stack_entries = 512;
3170                 rdev->config.evergreen.sx_num_of_sets = 4;
3171                 rdev->config.evergreen.sx_max_export_size = 256;
3172                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3173                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3174                 rdev->config.evergreen.max_hw_contexts = 8;
3175                 rdev->config.evergreen.sq_num_cf_insts = 2;
3176
3177                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3178                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3179                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3180                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3181                 break;
3182         case CHIP_REDWOOD:
3183                 rdev->config.evergreen.num_ses = 1;
3184                 rdev->config.evergreen.max_pipes = 4;
3185                 rdev->config.evergreen.max_tile_pipes = 4;
3186                 rdev->config.evergreen.max_simds = 5;
3187                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3188                 rdev->config.evergreen.max_gprs = 256;
3189                 rdev->config.evergreen.max_threads = 248;
3190                 rdev->config.evergreen.max_gs_threads = 32;
3191                 rdev->config.evergreen.max_stack_entries = 256;
3192                 rdev->config.evergreen.sx_num_of_sets = 4;
3193                 rdev->config.evergreen.sx_max_export_size = 256;
3194                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3195                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3196                 rdev->config.evergreen.max_hw_contexts = 8;
3197                 rdev->config.evergreen.sq_num_cf_insts = 2;
3198
3199                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3200                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3201                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3202                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3203                 break;
3204         case CHIP_CEDAR:
3205         default:
3206                 rdev->config.evergreen.num_ses = 1;
3207                 rdev->config.evergreen.max_pipes = 2;
3208                 rdev->config.evergreen.max_tile_pipes = 2;
3209                 rdev->config.evergreen.max_simds = 2;
3210                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3211                 rdev->config.evergreen.max_gprs = 256;
3212                 rdev->config.evergreen.max_threads = 192;
3213                 rdev->config.evergreen.max_gs_threads = 16;
3214                 rdev->config.evergreen.max_stack_entries = 256;
3215                 rdev->config.evergreen.sx_num_of_sets = 4;
3216                 rdev->config.evergreen.sx_max_export_size = 128;
3217                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3218                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3219                 rdev->config.evergreen.max_hw_contexts = 4;
3220                 rdev->config.evergreen.sq_num_cf_insts = 1;
3221
3222                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3223                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3224                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3225                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3226                 break;
3227         case CHIP_PALM:
3228                 rdev->config.evergreen.num_ses = 1;
3229                 rdev->config.evergreen.max_pipes = 2;
3230                 rdev->config.evergreen.max_tile_pipes = 2;
3231                 rdev->config.evergreen.max_simds = 2;
3232                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3233                 rdev->config.evergreen.max_gprs = 256;
3234                 rdev->config.evergreen.max_threads = 192;
3235                 rdev->config.evergreen.max_gs_threads = 16;
3236                 rdev->config.evergreen.max_stack_entries = 256;
3237                 rdev->config.evergreen.sx_num_of_sets = 4;
3238                 rdev->config.evergreen.sx_max_export_size = 128;
3239                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3240                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3241                 rdev->config.evergreen.max_hw_contexts = 4;
3242                 rdev->config.evergreen.sq_num_cf_insts = 1;
3243
3244                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3245                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3246                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3247                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3248                 break;
3249         case CHIP_SUMO:
3250                 rdev->config.evergreen.num_ses = 1;
3251                 rdev->config.evergreen.max_pipes = 4;
3252                 rdev->config.evergreen.max_tile_pipes = 4;
3253                 if (rdev->pdev->device == 0x9648)
3254                         rdev->config.evergreen.max_simds = 3;
3255                 else if ((rdev->pdev->device == 0x9647) ||
3256                          (rdev->pdev->device == 0x964a))
3257                         rdev->config.evergreen.max_simds = 4;
3258                 else
3259                         rdev->config.evergreen.max_simds = 5;
3260                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3261                 rdev->config.evergreen.max_gprs = 256;
3262                 rdev->config.evergreen.max_threads = 248;
3263                 rdev->config.evergreen.max_gs_threads = 32;
3264                 rdev->config.evergreen.max_stack_entries = 256;
3265                 rdev->config.evergreen.sx_num_of_sets = 4;
3266                 rdev->config.evergreen.sx_max_export_size = 256;
3267                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3268                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3269                 rdev->config.evergreen.max_hw_contexts = 8;
3270                 rdev->config.evergreen.sq_num_cf_insts = 2;
3271
3272                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3273                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3274                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3275                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3276                 break;
3277         case CHIP_SUMO2:
3278                 rdev->config.evergreen.num_ses = 1;
3279                 rdev->config.evergreen.max_pipes = 4;
3280                 rdev->config.evergreen.max_tile_pipes = 4;
3281                 rdev->config.evergreen.max_simds = 2;
3282                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3283                 rdev->config.evergreen.max_gprs = 256;
3284                 rdev->config.evergreen.max_threads = 248;
3285                 rdev->config.evergreen.max_gs_threads = 32;
3286                 rdev->config.evergreen.max_stack_entries = 512;
3287                 rdev->config.evergreen.sx_num_of_sets = 4;
3288                 rdev->config.evergreen.sx_max_export_size = 256;
3289                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3290                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3291                 rdev->config.evergreen.max_hw_contexts = 4;
3292                 rdev->config.evergreen.sq_num_cf_insts = 2;
3293
3294                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3295                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3296                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3297                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3298                 break;
3299         case CHIP_BARTS:
3300                 rdev->config.evergreen.num_ses = 2;
3301                 rdev->config.evergreen.max_pipes = 4;
3302                 rdev->config.evergreen.max_tile_pipes = 8;
3303                 rdev->config.evergreen.max_simds = 7;
3304                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3305                 rdev->config.evergreen.max_gprs = 256;
3306                 rdev->config.evergreen.max_threads = 248;
3307                 rdev->config.evergreen.max_gs_threads = 32;
3308                 rdev->config.evergreen.max_stack_entries = 512;
3309                 rdev->config.evergreen.sx_num_of_sets = 4;
3310                 rdev->config.evergreen.sx_max_export_size = 256;
3311                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3312                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3313                 rdev->config.evergreen.max_hw_contexts = 8;
3314                 rdev->config.evergreen.sq_num_cf_insts = 2;
3315
3316                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3317                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3318                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3319                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3320                 break;
3321         case CHIP_TURKS:
3322                 rdev->config.evergreen.num_ses = 1;
3323                 rdev->config.evergreen.max_pipes = 4;
3324                 rdev->config.evergreen.max_tile_pipes = 4;
3325                 rdev->config.evergreen.max_simds = 6;
3326                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3327                 rdev->config.evergreen.max_gprs = 256;
3328                 rdev->config.evergreen.max_threads = 248;
3329                 rdev->config.evergreen.max_gs_threads = 32;
3330                 rdev->config.evergreen.max_stack_entries = 256;
3331                 rdev->config.evergreen.sx_num_of_sets = 4;
3332                 rdev->config.evergreen.sx_max_export_size = 256;
3333                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3334                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3335                 rdev->config.evergreen.max_hw_contexts = 8;
3336                 rdev->config.evergreen.sq_num_cf_insts = 2;
3337
3338                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3339                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3340                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3341                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3342                 break;
3343         case CHIP_CAICOS:
3344                 rdev->config.evergreen.num_ses = 1;
3345                 rdev->config.evergreen.max_pipes = 2;
3346                 rdev->config.evergreen.max_tile_pipes = 2;
3347                 rdev->config.evergreen.max_simds = 2;
3348                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3349                 rdev->config.evergreen.max_gprs = 256;
3350                 rdev->config.evergreen.max_threads = 192;
3351                 rdev->config.evergreen.max_gs_threads = 16;
3352                 rdev->config.evergreen.max_stack_entries = 256;
3353                 rdev->config.evergreen.sx_num_of_sets = 4;
3354                 rdev->config.evergreen.sx_max_export_size = 128;
3355                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3356                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3357                 rdev->config.evergreen.max_hw_contexts = 4;
3358                 rdev->config.evergreen.sq_num_cf_insts = 1;
3359
3360                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3361                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3362                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3363                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3364                 break;
3365         }
3366
3367         /* Initialize HDP */
3368         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3369                 WREG32((0x2c14 + j), 0x00000000);
3370                 WREG32((0x2c18 + j), 0x00000000);
3371                 WREG32((0x2c1c + j), 0x00000000);
3372                 WREG32((0x2c20 + j), 0x00000000);
3373                 WREG32((0x2c24 + j), 0x00000000);
3374         }
3375
3376         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3377         WREG32(SRBM_INT_CNTL, 0x1);
3378         WREG32(SRBM_INT_ACK, 0x1);
3379
3380         evergreen_fix_pci_max_read_req_size(rdev);
3381
3382         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3383         if ((rdev->family == CHIP_PALM) ||
3384             (rdev->family == CHIP_SUMO) ||
3385             (rdev->family == CHIP_SUMO2))
3386                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3387         else
3388                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3389
3390         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3391          * not have bank info, so create a custom tiling dword.
3392          * bits 3:0   num_pipes
3393          * bits 7:4   num_banks
3394          * bits 11:8  group_size
3395          * bits 15:12 row_size
3396          */
3397         rdev->config.evergreen.tile_config = 0;
3398         switch (rdev->config.evergreen.max_tile_pipes) {
3399         case 1:
3400         default:
3401                 rdev->config.evergreen.tile_config |= (0 << 0);
3402                 break;
3403         case 2:
3404                 rdev->config.evergreen.tile_config |= (1 << 0);
3405                 break;
3406         case 4:
3407                 rdev->config.evergreen.tile_config |= (2 << 0);
3408                 break;
3409         case 8:
3410                 rdev->config.evergreen.tile_config |= (3 << 0);
3411                 break;
3412         }
3413         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3414         if (rdev->flags & RADEON_IS_IGP)
3415                 rdev->config.evergreen.tile_config |= 1 << 4;
3416         else {
3417                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3418                 case 0: /* four banks */
3419                         rdev->config.evergreen.tile_config |= 0 << 4;
3420                         break;
3421                 case 1: /* eight banks */
3422                         rdev->config.evergreen.tile_config |= 1 << 4;
3423                         break;
3424                 case 2: /* sixteen banks */
3425                 default:
3426                         rdev->config.evergreen.tile_config |= 2 << 4;
3427                         break;
3428                 }
3429         }
3430         rdev->config.evergreen.tile_config |= 0 << 8;
3431         rdev->config.evergreen.tile_config |=
3432                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3433
3434         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3435                 u32 efuse_straps_4;
3436                 u32 efuse_straps_3;
3437
3438                 efuse_straps_4 = RREG32_RCU(0x204);
3439                 efuse_straps_3 = RREG32_RCU(0x203);
3440                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3441                       ((efuse_straps_3 & 0xf0000000) >> 28));
3442         } else {
3443                 tmp = 0;
3444                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3445                         u32 rb_disable_bitmap;
3446
3447                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3448                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3449                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3450                         tmp <<= 4;
3451                         tmp |= rb_disable_bitmap;
3452                 }
3453         }
3454         /* enabled rb are just the one not disabled :) */
3455         disabled_rb_mask = tmp;
3456         tmp = 0;
3457         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3458                 tmp |= (1 << i);
3459         /* if all the backends are disabled, fix it up here */
3460         if ((disabled_rb_mask & tmp) == tmp) {
3461                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3462                         disabled_rb_mask &= ~(1 << i);
3463         }
3464
3465         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3466                 u32 simd_disable_bitmap;
3467
3468                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3469                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3470                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3471                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3472                 tmp <<= 16;
3473                 tmp |= simd_disable_bitmap;
3474         }
3475         rdev->config.evergreen.active_simds = hweight32(~tmp);
3476
3477         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3478         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3479
3480         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3481         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3482         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3483         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3484         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3485         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3486         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3487
3488         if ((rdev->config.evergreen.max_backends == 1) &&
3489             (rdev->flags & RADEON_IS_IGP)) {
3490                 if ((disabled_rb_mask & 3) == 1) {
3491                         /* RB0 disabled, RB1 enabled */
3492                         tmp = 0x11111111;
3493                 } else {
3494                         /* RB1 disabled, RB0 enabled */
3495                         tmp = 0x00000000;
3496                 }
3497         } else {
3498                 tmp = gb_addr_config & NUM_PIPES_MASK;
3499                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3500                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3501         }
3502         WREG32(GB_BACKEND_MAP, tmp);
3503
3504         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3505         WREG32(CGTS_TCC_DISABLE, 0);
3506         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3507         WREG32(CGTS_USER_TCC_DISABLE, 0);
3508
3509         /* set HW defaults for 3D engine */
3510         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3511                                      ROQ_IB2_START(0x2b)));
3512
3513         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3514
3515         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3516                              SYNC_GRADIENT |
3517                              SYNC_WALKER |
3518                              SYNC_ALIGNER));
3519
3520         sx_debug_1 = RREG32(SX_DEBUG_1);
3521         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3522         WREG32(SX_DEBUG_1, sx_debug_1);
3523
3524
3525         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3526         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3527         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3528         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3529
3530         if (rdev->family <= CHIP_SUMO2)
3531                 WREG32(SMX_SAR_CTL0, 0x00010000);
3532
3533         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3534                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3535                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3536
3537         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3538                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3539                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3540
3541         WREG32(VGT_NUM_INSTANCES, 1);
3542         WREG32(SPI_CONFIG_CNTL, 0);
3543         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3544         WREG32(CP_PERFMON_CNTL, 0);
3545
3546         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3547                                   FETCH_FIFO_HIWATER(0x4) |
3548                                   DONE_FIFO_HIWATER(0xe0) |
3549                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3550
3551         sq_config = RREG32(SQ_CONFIG);
3552         sq_config &= ~(PS_PRIO(3) |
3553                        VS_PRIO(3) |
3554                        GS_PRIO(3) |
3555                        ES_PRIO(3));
3556         sq_config |= (VC_ENABLE |
3557                       EXPORT_SRC_C |
3558                       PS_PRIO(0) |
3559                       VS_PRIO(1) |
3560                       GS_PRIO(2) |
3561                       ES_PRIO(3));
3562
3563         switch (rdev->family) {
3564         case CHIP_CEDAR:
3565         case CHIP_PALM:
3566         case CHIP_SUMO:
3567         case CHIP_SUMO2:
3568         case CHIP_CAICOS:
3569                 /* no vertex cache */
3570                 sq_config &= ~VC_ENABLE;
3571                 break;
3572         default:
3573                 break;
3574         }
3575
3576         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3577
3578         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3579         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3580         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3581         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3582         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3583         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3584         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3585
3586         switch (rdev->family) {
3587         case CHIP_CEDAR:
3588         case CHIP_PALM:
3589         case CHIP_SUMO:
3590         case CHIP_SUMO2:
3591                 ps_thread_count = 96;
3592                 break;
3593         default:
3594                 ps_thread_count = 128;
3595                 break;
3596         }
3597
3598         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3599         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3600         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3601         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3602         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3603         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3604
3605         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3606         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3607         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3608         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3609         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3610         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3611
3612         WREG32(SQ_CONFIG, sq_config);
3613         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3614         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3615         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3616         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3617         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3618         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3619         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3620         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3621         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3622         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3623
3624         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3625                                           FORCE_EOV_MAX_REZ_CNT(255)));
3626
3627         switch (rdev->family) {
3628         case CHIP_CEDAR:
3629         case CHIP_PALM:
3630         case CHIP_SUMO:
3631         case CHIP_SUMO2:
3632         case CHIP_CAICOS:
3633                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3634                 break;
3635         default:
3636                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3637                 break;
3638         }
3639         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3640         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3641
3642         WREG32(VGT_GS_VERTEX_REUSE, 16);
3643         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3644         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3645
3646         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3647         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3648
3649         WREG32(CB_PERF_CTR0_SEL_0, 0);
3650         WREG32(CB_PERF_CTR0_SEL_1, 0);
3651         WREG32(CB_PERF_CTR1_SEL_0, 0);
3652         WREG32(CB_PERF_CTR1_SEL_1, 0);
3653         WREG32(CB_PERF_CTR2_SEL_0, 0);
3654         WREG32(CB_PERF_CTR2_SEL_1, 0);
3655         WREG32(CB_PERF_CTR3_SEL_0, 0);
3656         WREG32(CB_PERF_CTR3_SEL_1, 0);
3657
3658         /* clear render buffer base addresses */
3659         WREG32(CB_COLOR0_BASE, 0);
3660         WREG32(CB_COLOR1_BASE, 0);
3661         WREG32(CB_COLOR2_BASE, 0);
3662         WREG32(CB_COLOR3_BASE, 0);
3663         WREG32(CB_COLOR4_BASE, 0);
3664         WREG32(CB_COLOR5_BASE, 0);
3665         WREG32(CB_COLOR6_BASE, 0);
3666         WREG32(CB_COLOR7_BASE, 0);
3667         WREG32(CB_COLOR8_BASE, 0);
3668         WREG32(CB_COLOR9_BASE, 0);
3669         WREG32(CB_COLOR10_BASE, 0);
3670         WREG32(CB_COLOR11_BASE, 0);
3671
3672         /* set the shader const cache sizes to 0 */
3673         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3674                 WREG32(i, 0);
3675         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3676                 WREG32(i, 0);
3677
3678         tmp = RREG32(HDP_MISC_CNTL);
3679         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3680         WREG32(HDP_MISC_CNTL, tmp);
3681
3682         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3683         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3684
3685         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3686
3687         udelay(50);
3688
3689 }
3690
3691 int evergreen_mc_init(struct radeon_device *rdev)
3692 {
3693         u32 tmp;
3694         int chansize, numchan;
3695
3696         /* Get VRAM informations */
3697         rdev->mc.vram_is_ddr = true;
3698         if ((rdev->family == CHIP_PALM) ||
3699             (rdev->family == CHIP_SUMO) ||
3700             (rdev->family == CHIP_SUMO2))
3701                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3702         else
3703                 tmp = RREG32(MC_ARB_RAMCFG);
3704         if (tmp & CHANSIZE_OVERRIDE) {
3705                 chansize = 16;
3706         } else if (tmp & CHANSIZE_MASK) {
3707                 chansize = 64;
3708         } else {
3709                 chansize = 32;
3710         }
3711         tmp = RREG32(MC_SHARED_CHMAP);
3712         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3713         case 0:
3714         default:
3715                 numchan = 1;
3716                 break;
3717         case 1:
3718                 numchan = 2;
3719                 break;
3720         case 2:
3721                 numchan = 4;
3722                 break;
3723         case 3:
3724                 numchan = 8;
3725                 break;
3726         }
3727         rdev->mc.vram_width = numchan * chansize;
3728         /* Could aper size report 0 ? */
3729         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3730         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3731         /* Setup GPU memory space */
3732         if ((rdev->family == CHIP_PALM) ||
3733             (rdev->family == CHIP_SUMO) ||
3734             (rdev->family == CHIP_SUMO2)) {
3735                 /* size in bytes on fusion */
3736                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3737                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3738         } else {
3739                 /* size in MB on evergreen/cayman/tn */
3740                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3741                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3742         }
3743         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3744         r700_vram_gtt_location(rdev, &rdev->mc);
3745         radeon_update_bandwidth_info(rdev);
3746
3747         return 0;
3748 }
3749
3750 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3751 {
3752         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3753                 RREG32(GRBM_STATUS));
3754         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3755                 RREG32(GRBM_STATUS_SE0));
3756         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3757                 RREG32(GRBM_STATUS_SE1));
3758         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3759                 RREG32(SRBM_STATUS));
3760         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3761                 RREG32(SRBM_STATUS2));
3762         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3763                 RREG32(CP_STALLED_STAT1));
3764         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3765                 RREG32(CP_STALLED_STAT2));
3766         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3767                 RREG32(CP_BUSY_STAT));
3768         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3769                 RREG32(CP_STAT));
3770         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3771                 RREG32(DMA_STATUS_REG));
3772         if (rdev->family >= CHIP_CAYMAN) {
3773                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3774                          RREG32(DMA_STATUS_REG + 0x800));
3775         }
3776 }
3777
3778 bool evergreen_is_display_hung(struct radeon_device *rdev)
3779 {
3780         u32 crtc_hung = 0;
3781         u32 crtc_status[6];
3782         u32 i, j, tmp;
3783
3784         for (i = 0; i < rdev->num_crtc; i++) {
3785                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3786                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3787                         crtc_hung |= (1 << i);
3788                 }
3789         }
3790
3791         for (j = 0; j < 10; j++) {
3792                 for (i = 0; i < rdev->num_crtc; i++) {
3793                         if (crtc_hung & (1 << i)) {
3794                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3795                                 if (tmp != crtc_status[i])
3796                                         crtc_hung &= ~(1 << i);
3797                         }
3798                 }
3799                 if (crtc_hung == 0)
3800                         return false;
3801                 udelay(100);
3802         }
3803
3804         return true;
3805 }
3806
3807 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3808 {
3809         u32 reset_mask = 0;
3810         u32 tmp;
3811
3812         /* GRBM_STATUS */
3813         tmp = RREG32(GRBM_STATUS);
3814         if (tmp & (PA_BUSY | SC_BUSY |
3815                    SH_BUSY | SX_BUSY |
3816                    TA_BUSY | VGT_BUSY |
3817                    DB_BUSY | CB_BUSY |
3818                    SPI_BUSY | VGT_BUSY_NO_DMA))
3819                 reset_mask |= RADEON_RESET_GFX;
3820
3821         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3822                    CP_BUSY | CP_COHERENCY_BUSY))
3823                 reset_mask |= RADEON_RESET_CP;
3824
3825         if (tmp & GRBM_EE_BUSY)
3826                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3827
3828         /* DMA_STATUS_REG */
3829         tmp = RREG32(DMA_STATUS_REG);
3830         if (!(tmp & DMA_IDLE))
3831                 reset_mask |= RADEON_RESET_DMA;
3832
3833         /* SRBM_STATUS2 */
3834         tmp = RREG32(SRBM_STATUS2);
3835         if (tmp & DMA_BUSY)
3836                 reset_mask |= RADEON_RESET_DMA;
3837
3838         /* SRBM_STATUS */
3839         tmp = RREG32(SRBM_STATUS);
3840         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3841                 reset_mask |= RADEON_RESET_RLC;
3842
3843         if (tmp & IH_BUSY)
3844                 reset_mask |= RADEON_RESET_IH;
3845
3846         if (tmp & SEM_BUSY)
3847                 reset_mask |= RADEON_RESET_SEM;
3848
3849         if (tmp & GRBM_RQ_PENDING)
3850                 reset_mask |= RADEON_RESET_GRBM;
3851
3852         if (tmp & VMC_BUSY)
3853                 reset_mask |= RADEON_RESET_VMC;
3854
3855         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3856                    MCC_BUSY | MCD_BUSY))
3857                 reset_mask |= RADEON_RESET_MC;
3858
3859         if (evergreen_is_display_hung(rdev))
3860                 reset_mask |= RADEON_RESET_DISPLAY;
3861
3862         /* VM_L2_STATUS */
3863         tmp = RREG32(VM_L2_STATUS);
3864         if (tmp & L2_BUSY)
3865                 reset_mask |= RADEON_RESET_VMC;
3866
3867         /* Skip MC reset as it's mostly likely not hung, just busy */
3868         if (reset_mask & RADEON_RESET_MC) {
3869                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3870                 reset_mask &= ~RADEON_RESET_MC;
3871         }
3872
3873         return reset_mask;
3874 }
3875
3876 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3877 {
3878         struct evergreen_mc_save save;
3879         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3880         u32 tmp;
3881
3882         if (reset_mask == 0)
3883                 return;
3884
3885         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3886
3887         evergreen_print_gpu_status_regs(rdev);
3888
3889         /* Disable CP parsing/prefetching */
3890         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3891
3892         if (reset_mask & RADEON_RESET_DMA) {
3893                 /* Disable DMA */
3894                 tmp = RREG32(DMA_RB_CNTL);
3895                 tmp &= ~DMA_RB_ENABLE;
3896                 WREG32(DMA_RB_CNTL, tmp);
3897         }
3898
3899         udelay(50);
3900
3901         evergreen_mc_stop(rdev, &save);
3902         if (evergreen_mc_wait_for_idle(rdev)) {
3903                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3904         }
3905
3906         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3907                 grbm_soft_reset |= SOFT_RESET_DB |
3908                         SOFT_RESET_CB |
3909                         SOFT_RESET_PA |
3910                         SOFT_RESET_SC |
3911                         SOFT_RESET_SPI |
3912                         SOFT_RESET_SX |
3913                         SOFT_RESET_SH |
3914                         SOFT_RESET_TC |
3915                         SOFT_RESET_TA |
3916                         SOFT_RESET_VC |
3917                         SOFT_RESET_VGT;
3918         }
3919
3920         if (reset_mask & RADEON_RESET_CP) {
3921                 grbm_soft_reset |= SOFT_RESET_CP |
3922                         SOFT_RESET_VGT;
3923
3924                 srbm_soft_reset |= SOFT_RESET_GRBM;
3925         }
3926
3927         if (reset_mask & RADEON_RESET_DMA)
3928                 srbm_soft_reset |= SOFT_RESET_DMA;
3929
3930         if (reset_mask & RADEON_RESET_DISPLAY)
3931                 srbm_soft_reset |= SOFT_RESET_DC;
3932
3933         if (reset_mask & RADEON_RESET_RLC)
3934                 srbm_soft_reset |= SOFT_RESET_RLC;
3935
3936         if (reset_mask & RADEON_RESET_SEM)
3937                 srbm_soft_reset |= SOFT_RESET_SEM;
3938
3939         if (reset_mask & RADEON_RESET_IH)
3940                 srbm_soft_reset |= SOFT_RESET_IH;
3941
3942         if (reset_mask & RADEON_RESET_GRBM)
3943                 srbm_soft_reset |= SOFT_RESET_GRBM;
3944
3945         if (reset_mask & RADEON_RESET_VMC)
3946                 srbm_soft_reset |= SOFT_RESET_VMC;
3947
3948         if (!(rdev->flags & RADEON_IS_IGP)) {
3949                 if (reset_mask & RADEON_RESET_MC)
3950                         srbm_soft_reset |= SOFT_RESET_MC;
3951         }
3952
3953         if (grbm_soft_reset) {
3954                 tmp = RREG32(GRBM_SOFT_RESET);
3955                 tmp |= grbm_soft_reset;
3956                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3957                 WREG32(GRBM_SOFT_RESET, tmp);
3958                 tmp = RREG32(GRBM_SOFT_RESET);
3959
3960                 udelay(50);
3961
3962                 tmp &= ~grbm_soft_reset;
3963                 WREG32(GRBM_SOFT_RESET, tmp);
3964                 tmp = RREG32(GRBM_SOFT_RESET);
3965         }
3966
3967         if (srbm_soft_reset) {
3968                 tmp = RREG32(SRBM_SOFT_RESET);
3969                 tmp |= srbm_soft_reset;
3970                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3971                 WREG32(SRBM_SOFT_RESET, tmp);
3972                 tmp = RREG32(SRBM_SOFT_RESET);
3973
3974                 udelay(50);
3975
3976                 tmp &= ~srbm_soft_reset;
3977                 WREG32(SRBM_SOFT_RESET, tmp);
3978                 tmp = RREG32(SRBM_SOFT_RESET);
3979         }
3980
3981         /* Wait a little for things to settle down */
3982         udelay(50);
3983
3984         evergreen_mc_resume(rdev, &save);
3985         udelay(50);
3986
3987         evergreen_print_gpu_status_regs(rdev);
3988 }
3989
3990 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3991 {
3992         struct evergreen_mc_save save;
3993         u32 tmp, i;
3994
3995         dev_info(rdev->dev, "GPU pci config reset\n");
3996
3997         /* disable dpm? */
3998
3999         /* Disable CP parsing/prefetching */
4000         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4001         udelay(50);
4002         /* Disable DMA */
4003         tmp = RREG32(DMA_RB_CNTL);
4004         tmp &= ~DMA_RB_ENABLE;
4005         WREG32(DMA_RB_CNTL, tmp);
4006         /* XXX other engines? */
4007
4008         /* halt the rlc */
4009         r600_rlc_stop(rdev);
4010
4011         udelay(50);
4012
4013         /* set mclk/sclk to bypass */
4014         rv770_set_clk_bypass_mode(rdev);
4015         /* disable BM */
4016         pci_clear_master(rdev->pdev);
4017         /* disable mem access */
4018         evergreen_mc_stop(rdev, &save);
4019         if (evergreen_mc_wait_for_idle(rdev)) {
4020                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4021         }
4022         /* reset */
4023         radeon_pci_config_reset(rdev);
4024         /* wait for asic to come out of reset */
4025         for (i = 0; i < rdev->usec_timeout; i++) {
4026                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4027                         break;
4028                 udelay(1);
4029         }
4030 }
4031
4032 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4033 {
4034         u32 reset_mask;
4035
4036         if (hard) {
4037                 evergreen_gpu_pci_config_reset(rdev);
4038                 return 0;
4039         }
4040
4041         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4042
4043         if (reset_mask)
4044                 r600_set_bios_scratch_engine_hung(rdev, true);
4045
4046         /* try soft reset */
4047         evergreen_gpu_soft_reset(rdev, reset_mask);
4048
4049         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4050
4051         /* try pci config reset */
4052         if (reset_mask && radeon_hard_reset)
4053                 evergreen_gpu_pci_config_reset(rdev);
4054
4055         reset_mask = evergreen_gpu_check_soft_reset(rdev);
4056
4057         if (!reset_mask)
4058                 r600_set_bios_scratch_engine_hung(rdev, false);
4059
4060         return 0;
4061 }
4062
4063 /**
4064  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4065  *
4066  * @rdev: radeon_device pointer
4067  * @ring: radeon_ring structure holding ring information
4068  *
4069  * Check if the GFX engine is locked up.
4070  * Returns true if the engine appears to be locked up, false if not.
4071  */
4072 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4073 {
4074         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4075
4076         if (!(reset_mask & (RADEON_RESET_GFX |
4077                             RADEON_RESET_COMPUTE |
4078                             RADEON_RESET_CP))) {
4079                 radeon_ring_lockup_update(rdev, ring);
4080                 return false;
4081         }
4082         return radeon_ring_test_lockup(rdev, ring);
4083 }
4084
4085 /*
4086  * RLC
4087  */
4088 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
4089 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
4090
4091 void sumo_rlc_fini(struct radeon_device *rdev)
4092 {
4093         int r;
4094
4095         /* save restore block */
4096         if (rdev->rlc.save_restore_obj) {
4097                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4098                 if (unlikely(r != 0))
4099                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4100                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4101                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4102
4103                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4104                 rdev->rlc.save_restore_obj = NULL;
4105         }
4106
4107         /* clear state block */
4108         if (rdev->rlc.clear_state_obj) {
4109                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4110                 if (unlikely(r != 0))
4111                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4112                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4113                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4114
4115                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4116                 rdev->rlc.clear_state_obj = NULL;
4117         }
4118
4119         /* clear state block */
4120         if (rdev->rlc.cp_table_obj) {
4121                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4122                 if (unlikely(r != 0))
4123                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4124                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4125                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4126
4127                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4128                 rdev->rlc.cp_table_obj = NULL;
4129         }
4130 }
4131
4132 #define CP_ME_TABLE_SIZE    96
4133
4134 int sumo_rlc_init(struct radeon_device *rdev)
4135 {
4136         const u32 *src_ptr;
4137         volatile u32 *dst_ptr;
4138         u32 dws, data, i, j, k, reg_num;
4139         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4140         u64 reg_list_mc_addr;
4141         const struct cs_section_def *cs_data;
4142         int r;
4143
4144         src_ptr = rdev->rlc.reg_list;
4145         dws = rdev->rlc.reg_list_size;
4146         if (rdev->family >= CHIP_BONAIRE) {
4147                 dws += (5 * 16) + 48 + 48 + 64;
4148         }
4149         cs_data = rdev->rlc.cs_data;
4150
4151         if (src_ptr) {
4152                 /* save restore block */
4153                 if (rdev->rlc.save_restore_obj == NULL) {
4154                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4155                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4156                                              NULL, &rdev->rlc.save_restore_obj);
4157                         if (r) {
4158                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4159                                 return r;
4160                         }
4161                 }
4162
4163                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4164                 if (unlikely(r != 0)) {
4165                         sumo_rlc_fini(rdev);
4166                         return r;
4167                 }
4168                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4169                                   &rdev->rlc.save_restore_gpu_addr);
4170                 if (r) {
4171                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4172                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4173                         sumo_rlc_fini(rdev);
4174                         return r;
4175                 }
4176
4177                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4178                 if (r) {
4179                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4180                         sumo_rlc_fini(rdev);
4181                         return r;
4182                 }
4183                 /* write the sr buffer */
4184                 dst_ptr = rdev->rlc.sr_ptr;
4185                 if (rdev->family >= CHIP_TAHITI) {
4186                         /* SI */
4187                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4188                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4189                 } else {
4190                         /* ON/LN/TN */
4191                         /* format:
4192                          * dw0: (reg2 << 16) | reg1
4193                          * dw1: reg1 save space
4194                          * dw2: reg2 save space
4195                          */
4196                         for (i = 0; i < dws; i++) {
4197                                 data = src_ptr[i] >> 2;
4198                                 i++;
4199                                 if (i < dws)
4200                                         data |= (src_ptr[i] >> 2) << 16;
4201                                 j = (((i - 1) * 3) / 2);
4202                                 dst_ptr[j] = cpu_to_le32(data);
4203                         }
4204                         j = ((i * 3) / 2);
4205                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4206                 }
4207                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4208                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4209         }
4210
4211         if (cs_data) {
4212                 /* clear state block */
4213                 if (rdev->family >= CHIP_BONAIRE) {
4214                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4215                 } else if (rdev->family >= CHIP_TAHITI) {
4216                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4217                         dws = rdev->rlc.clear_state_size + (256 / 4);
4218                 } else {
4219                         reg_list_num = 0;
4220                         dws = 0;
4221                         for (i = 0; cs_data[i].section != NULL; i++) {
4222                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4223                                         reg_list_num++;
4224                                         dws += cs_data[i].section[j].reg_count;
4225                                 }
4226                         }
4227                         reg_list_blk_index = (3 * reg_list_num + 2);
4228                         dws += reg_list_blk_index;
4229                         rdev->rlc.clear_state_size = dws;
4230                 }
4231
4232                 if (rdev->rlc.clear_state_obj == NULL) {
4233                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4234                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4235                                              NULL, &rdev->rlc.clear_state_obj);
4236                         if (r) {
4237                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4238                                 sumo_rlc_fini(rdev);
4239                                 return r;
4240                         }
4241                 }
4242                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4243                 if (unlikely(r != 0)) {
4244                         sumo_rlc_fini(rdev);
4245                         return r;
4246                 }
4247                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4248                                   &rdev->rlc.clear_state_gpu_addr);
4249                 if (r) {
4250                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4251                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4252                         sumo_rlc_fini(rdev);
4253                         return r;
4254                 }
4255
4256                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4257                 if (r) {
4258                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4259                         sumo_rlc_fini(rdev);
4260                         return r;
4261                 }
4262                 /* set up the cs buffer */
4263                 dst_ptr = rdev->rlc.cs_ptr;
4264                 if (rdev->family >= CHIP_BONAIRE) {
4265                         cik_get_csb_buffer(rdev, dst_ptr);
4266                 } else if (rdev->family >= CHIP_TAHITI) {
4267                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4268                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4269                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4270                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4271                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4272                 } else {
4273                         reg_list_hdr_blk_index = 0;
4274                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4275                         data = upper_32_bits(reg_list_mc_addr);
4276                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4277                         reg_list_hdr_blk_index++;
4278                         for (i = 0; cs_data[i].section != NULL; i++) {
4279                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4280                                         reg_num = cs_data[i].section[j].reg_count;
4281                                         data = reg_list_mc_addr & 0xffffffff;
4282                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4283                                         reg_list_hdr_blk_index++;
4284
4285                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4286                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4287                                         reg_list_hdr_blk_index++;
4288
4289                                         data = 0x08000000 | (reg_num * 4);
4290                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4291                                         reg_list_hdr_blk_index++;
4292
4293                                         for (k = 0; k < reg_num; k++) {
4294                                                 data = cs_data[i].section[j].extent[k];
4295                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4296                                         }
4297                                         reg_list_mc_addr += reg_num * 4;
4298                                         reg_list_blk_index += reg_num;
4299                                 }
4300                         }
4301                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4302                 }
4303                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4304                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4305         }
4306
4307         if (rdev->rlc.cp_table_size) {
4308                 if (rdev->rlc.cp_table_obj == NULL) {
4309                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4310                                              PAGE_SIZE, true,
4311                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4312                                              NULL, &rdev->rlc.cp_table_obj);
4313                         if (r) {
4314                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4315                                 sumo_rlc_fini(rdev);
4316                                 return r;
4317                         }
4318                 }
4319
4320                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4321                 if (unlikely(r != 0)) {
4322                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4323                         sumo_rlc_fini(rdev);
4324                         return r;
4325                 }
4326                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4327                                   &rdev->rlc.cp_table_gpu_addr);
4328                 if (r) {
4329                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4330                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4331                         sumo_rlc_fini(rdev);
4332                         return r;
4333                 }
4334                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4335                 if (r) {
4336                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4337                         sumo_rlc_fini(rdev);
4338                         return r;
4339                 }
4340
4341                 cik_init_cp_pg_table(rdev);
4342
4343                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4344                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4345
4346         }
4347
4348         return 0;
4349 }
4350
4351 static void evergreen_rlc_start(struct radeon_device *rdev)
4352 {
4353         u32 mask = RLC_ENABLE;
4354
4355         if (rdev->flags & RADEON_IS_IGP) {
4356                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4357         }
4358
4359         WREG32(RLC_CNTL, mask);
4360 }
4361
4362 int evergreen_rlc_resume(struct radeon_device *rdev)
4363 {
4364         u32 i;
4365         const __be32 *fw_data;
4366
4367         if (!rdev->rlc_fw)
4368                 return -EINVAL;
4369
4370         r600_rlc_stop(rdev);
4371
4372         WREG32(RLC_HB_CNTL, 0);
4373
4374         if (rdev->flags & RADEON_IS_IGP) {
4375                 if (rdev->family == CHIP_ARUBA) {
4376                         u32 always_on_bitmap =
4377                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4378                         /* find out the number of active simds */
4379                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4380                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4381                         tmp = hweight32(~tmp);
4382                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4383                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4384                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4385                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4386                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4387                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4388                         }
4389                 } else {
4390                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4391                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4392                 }
4393                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4394                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4395         } else {
4396                 WREG32(RLC_HB_BASE, 0);
4397                 WREG32(RLC_HB_RPTR, 0);
4398                 WREG32(RLC_HB_WPTR, 0);
4399                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4400                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4401         }
4402         WREG32(RLC_MC_CNTL, 0);
4403         WREG32(RLC_UCODE_CNTL, 0);
4404
4405         fw_data = (const __be32 *)rdev->rlc_fw->data;
4406         if (rdev->family >= CHIP_ARUBA) {
4407                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4408                         WREG32(RLC_UCODE_ADDR, i);
4409                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4410                 }
4411         } else if (rdev->family >= CHIP_CAYMAN) {
4412                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4413                         WREG32(RLC_UCODE_ADDR, i);
4414                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4415                 }
4416         } else {
4417                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4418                         WREG32(RLC_UCODE_ADDR, i);
4419                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4420                 }
4421         }
4422         WREG32(RLC_UCODE_ADDR, 0);
4423
4424         evergreen_rlc_start(rdev);
4425
4426         return 0;
4427 }
4428
4429 /* Interrupts */
4430
4431 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4432 {
4433         if (crtc >= rdev->num_crtc)
4434                 return 0;
4435         else
4436                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4437 }
4438
4439 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4440 {
4441         int i;
4442         u32 tmp;
4443
4444         if (rdev->family >= CHIP_CAYMAN) {
4445                 cayman_cp_int_cntl_setup(rdev, 0,
4446                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4447                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4448                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4449                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4450                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4451         } else
4452                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4453         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4454         WREG32(DMA_CNTL, tmp);
4455         WREG32(GRBM_INT_CNTL, 0);
4456         WREG32(SRBM_INT_CNTL, 0);
4457         for (i = 0; i < rdev->num_crtc; i++)
4458                 WREG32(INT_MASK + crtc_offsets[i], 0);
4459         for (i = 0; i < rdev->num_crtc; i++)
4460                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4461
4462         /* only one DAC on DCE5 */
4463         if (!ASIC_IS_DCE5(rdev))
4464                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4465         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4466
4467         for (i = 0; i < 6; i++)
4468                 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4469 }
4470
4471 /* Note that the order we write back regs here is important */
4472 int evergreen_irq_set(struct radeon_device *rdev)
4473 {
4474         int i;
4475         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4476         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4477         u32 grbm_int_cntl = 0;
4478         u32 dma_cntl, dma_cntl1 = 0;
4479         u32 thermal_int = 0;
4480
4481         if (!rdev->irq.installed) {
4482                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4483                 return -EINVAL;
4484         }
4485         /* don't enable anything if the ih is disabled */
4486         if (!rdev->ih.enabled) {
4487                 r600_disable_interrupts(rdev);
4488                 /* force the active interrupt state to all disabled */
4489                 evergreen_disable_interrupt_state(rdev);
4490                 return 0;
4491         }
4492
4493         if (rdev->family == CHIP_ARUBA)
4494                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4495                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4496         else
4497                 thermal_int = RREG32(CG_THERMAL_INT) &
4498                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4499
4500         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4501
4502         if (rdev->family >= CHIP_CAYMAN) {
4503                 /* enable CP interrupts on all rings */
4504                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4505                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4506                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4507                 }
4508                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4509                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4510                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4511                 }
4512                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4513                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4514                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4515                 }
4516         } else {
4517                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4518                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4519                         cp_int_cntl |= RB_INT_ENABLE;
4520                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4521                 }
4522         }
4523
4524         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4525                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4526                 dma_cntl |= TRAP_ENABLE;
4527         }
4528
4529         if (rdev->family >= CHIP_CAYMAN) {
4530                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4531                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4532                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4533                         dma_cntl1 |= TRAP_ENABLE;
4534                 }
4535         }
4536
4537         if (rdev->irq.dpm_thermal) {
4538                 DRM_DEBUG("dpm thermal\n");
4539                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4540         }
4541
4542         if (rdev->family >= CHIP_CAYMAN) {
4543                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4544                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4545                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4546         } else
4547                 WREG32(CP_INT_CNTL, cp_int_cntl);
4548
4549         WREG32(DMA_CNTL, dma_cntl);
4550
4551         if (rdev->family >= CHIP_CAYMAN)
4552                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4553
4554         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4555
4556         for (i = 0; i < rdev->num_crtc; i++) {
4557                 radeon_irq_kms_set_irq_n_enabled(
4558                     rdev, INT_MASK + crtc_offsets[i],
4559                     VBLANK_INT_MASK,
4560                     rdev->irq.crtc_vblank_int[i] ||
4561                     atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4562         }
4563
4564         for (i = 0; i < rdev->num_crtc; i++)
4565                 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4566
4567         for (i = 0; i < 6; i++) {
4568                 radeon_irq_kms_set_irq_n_enabled(
4569                     rdev, DC_HPDx_INT_CONTROL(i),
4570                     DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4571                     rdev->irq.hpd[i], "HPD", i);
4572         }
4573
4574         if (rdev->family == CHIP_ARUBA)
4575                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4576         else
4577                 WREG32(CG_THERMAL_INT, thermal_int);
4578
4579         for (i = 0; i < 6; i++) {
4580                 radeon_irq_kms_set_irq_n_enabled(
4581                     rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4582                     AFMT_AZ_FORMAT_WTRIG_MASK,
4583                     rdev->irq.afmt[i], "HDMI", i);
4584         }
4585
4586         /* posting read */
4587         RREG32(SRBM_STATUS);
4588
4589         return 0;
4590 }
4591
4592 /* Note that the order we write back regs here is important */
4593 static void evergreen_irq_ack(struct radeon_device *rdev)
4594 {
4595         int i, j;
4596         u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4597         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4598         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4599
4600         for (i = 0; i < 6; i++) {
4601                 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4602                 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4603                 if (i < rdev->num_crtc)
4604                         grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4605         }
4606
4607         /* We write back each interrupt register in pairs of two */
4608         for (i = 0; i < rdev->num_crtc; i += 2) {
4609                 for (j = i; j < (i + 2); j++) {
4610                         if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4611                                 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4612                                        GRPH_PFLIP_INT_CLEAR);
4613                 }
4614
4615                 for (j = i; j < (i + 2); j++) {
4616                         if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4617                                 WREG32(VBLANK_STATUS + crtc_offsets[j],
4618                                        VBLANK_ACK);
4619                         if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4620                                 WREG32(VLINE_STATUS + crtc_offsets[j],
4621                                        VLINE_ACK);
4622                 }
4623         }
4624
4625         for (i = 0; i < 6; i++) {
4626                 if (disp_int[i] & DC_HPD1_INTERRUPT)
4627                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4628         }
4629
4630         for (i = 0; i < 6; i++) {
4631                 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4632                         WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4633         }
4634
4635         for (i = 0; i < 6; i++) {
4636                 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4637                         WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4638                                   AFMT_AZ_FORMAT_WTRIG_ACK);
4639         }
4640 }
4641
4642 static void evergreen_irq_disable(struct radeon_device *rdev)
4643 {
4644         r600_disable_interrupts(rdev);
4645         /* Wait and acknowledge irq */
4646         mdelay(1);
4647         evergreen_irq_ack(rdev);
4648         evergreen_disable_interrupt_state(rdev);
4649 }
4650
4651 void evergreen_irq_suspend(struct radeon_device *rdev)
4652 {
4653         evergreen_irq_disable(rdev);
4654         r600_rlc_stop(rdev);
4655 }
4656
4657 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4658 {
4659         u32 wptr, tmp;
4660
4661         if (rdev->wb.enabled)
4662                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4663         else
4664                 wptr = RREG32(IH_RB_WPTR);
4665
4666         if (wptr & RB_OVERFLOW) {
4667                 wptr &= ~RB_OVERFLOW;
4668                 /* When a ring buffer overflow happen start parsing interrupt
4669                  * from the last not overwritten vector (wptr + 16). Hopefully
4670                  * this should allow us to catchup.
4671                  */
4672                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4673                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4674                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4675                 tmp = RREG32(IH_RB_CNTL);
4676                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4677                 WREG32(IH_RB_CNTL, tmp);
4678         }
4679         return (wptr & rdev->ih.ptr_mask);
4680 }
4681
4682 irqreturn_t evergreen_irq_process(struct radeon_device *rdev)
4683 {
4684         u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4685         u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4686         u32 crtc_idx, hpd_idx, afmt_idx;
4687         u32 mask;
4688         u32 wptr;
4689         u32 rptr;
4690         u32 src_id, src_data;
4691         u32 ring_index;
4692         bool queue_hotplug = false;
4693         bool queue_hdmi = false;
4694         bool queue_dp = false;
4695         bool queue_thermal = false;
4696         u32 status, addr;
4697         const char *event_name;
4698
4699         if (!rdev->ih.enabled || rdev->shutdown)
4700                 return IRQ_NONE;
4701
4702         wptr = evergreen_get_ih_wptr(rdev);
4703
4704 restart_ih:
4705         /* is somebody else already processing irqs? */
4706         if (atomic_xchg(&rdev->ih.lock, 1))
4707                 return IRQ_NONE;
4708
4709         rptr = rdev->ih.rptr;
4710         DRM_DEBUG_VBLANK("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4711
4712         /* Order reading of wptr vs. reading of IH ring data */
4713         rmb();
4714
4715         /* display interrupts */
4716         evergreen_irq_ack(rdev);
4717
4718         while (rptr != wptr) {
4719                 /* wptr/rptr are in bytes! */
4720                 ring_index = rptr / 4;
4721                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4722                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4723
4724                 switch (src_id) {
4725                 case 1: /* D1 vblank/vline */
4726                 case 2: /* D2 vblank/vline */
4727                 case 3: /* D3 vblank/vline */
4728                 case 4: /* D4 vblank/vline */
4729                 case 5: /* D5 vblank/vline */
4730                 case 6: /* D6 vblank/vline */
4731                         crtc_idx = src_id - 1;
4732
4733                         if (src_data == 0) { /* vblank */
4734                                 mask = LB_D1_VBLANK_INTERRUPT;
4735                                 event_name = "vblank";
4736
4737                                 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4738                                         drm_handle_vblank(rdev->ddev, crtc_idx);
4739                                         rdev->pm.vblank_sync = true;
4740                                         wake_up(&rdev->irq.vblank_queue);
4741                                 }
4742                                 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4743                                         radeon_crtc_handle_vblank(rdev,
4744                                                                   crtc_idx);
4745                                 }
4746
4747                         } else if (src_data == 1) { /* vline */
4748                                 mask = LB_D1_VLINE_INTERRUPT;
4749                                 event_name = "vline";
4750                         } else {
4751                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4752                                           src_id, src_data);
4753                                 break;
4754                         }
4755
4756                         if (!(disp_int[crtc_idx] & mask)) {
4757                                 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4758                                           crtc_idx + 1, event_name);
4759                         }
4760
4761                         disp_int[crtc_idx] &= ~mask;
4762                         DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4763
4764                         break;
4765                 case 8: /* D1 page flip */
4766                 case 10: /* D2 page flip */
4767                 case 12: /* D3 page flip */
4768                 case 14: /* D4 page flip */
4769                 case 16: /* D5 page flip */
4770                 case 18: /* D6 page flip */
4771                         DRM_DEBUG_VBLANK("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4772                         if (radeon_use_pflipirq > 0)
4773                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4774                         break;
4775                 case 42: /* HPD hotplug */
4776                         if (src_data <= 5) {
4777                                 hpd_idx = src_data;
4778                                 mask = DC_HPD1_INTERRUPT;
4779                                 queue_hotplug = true;
4780                                 event_name = "HPD";
4781
4782                         } else if (src_data <= 11) {
4783                                 hpd_idx = src_data - 6;
4784                                 mask = DC_HPD1_RX_INTERRUPT;
4785                                 queue_dp = true;
4786                                 event_name = "HPD_RX";
4787
4788                         } else {
4789                                 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4790                                           src_id, src_data);
4791                                 break;
4792                         }
4793
4794                         if (!(disp_int[hpd_idx] & mask))
4795                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4796
4797                         disp_int[hpd_idx] &= ~mask;
4798                         DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4799
4800                         break;
4801                 case 44: /* hdmi */
4802                         afmt_idx = src_data;
4803                         if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4804                                 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4805
4806                         if (afmt_idx > 5) {
4807                                 DRM_ERROR("Unhandled interrupt: %d %d\n",
4808                                           src_id, src_data);
4809                                 break;
4810                         }
4811                         afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4812                         queue_hdmi = true;
4813                         DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4814                         break;
4815                 case 96:
4816                         DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4817                         WREG32(SRBM_INT_ACK, 0x1);
4818                         break;
4819                 case 124: /* UVD */
4820                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4821                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4822                         break;
4823                 case 146:
4824                 case 147:
4825                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4826                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4827                         /* reset addr and status */
4828                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4829                         if (addr == 0x0 && status == 0x0)
4830                                 break;
4831                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4832                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4833                                 addr);
4834                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4835                                 status);
4836                         cayman_vm_decode_fault(rdev, status, addr);
4837                         break;
4838                 case 176: /* CP_INT in ring buffer */
4839                 case 177: /* CP_INT in IB1 */
4840                 case 178: /* CP_INT in IB2 */
4841                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4842                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4843                         break;
4844                 case 181: /* CP EOP event */
4845                         DRM_DEBUG("IH: CP EOP\n");
4846                         if (rdev->family >= CHIP_CAYMAN) {
4847                                 switch (src_data) {
4848                                 case 0:
4849                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4850                                         break;
4851                                 case 1:
4852                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4853                                         break;
4854                                 case 2:
4855                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4856                                         break;
4857                                 }
4858                         } else
4859                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4860                         break;
4861                 case 224: /* DMA trap event */
4862                         DRM_DEBUG("IH: DMA trap\n");
4863                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4864                         break;
4865                 case 230: /* thermal low to high */
4866                         DRM_DEBUG("IH: thermal low to high\n");
4867                         rdev->pm.dpm.thermal.high_to_low = false;
4868                         queue_thermal = true;
4869                         break;
4870                 case 231: /* thermal high to low */
4871                         DRM_DEBUG("IH: thermal high to low\n");
4872                         rdev->pm.dpm.thermal.high_to_low = true;
4873                         queue_thermal = true;
4874                         break;
4875                 case 233: /* GUI IDLE */
4876                         DRM_DEBUG("IH: GUI idle\n");
4877                         break;
4878                 case 244: /* DMA trap event */
4879                         if (rdev->family >= CHIP_CAYMAN) {
4880                                 DRM_DEBUG("IH: DMA1 trap\n");
4881                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4882                         }
4883                         break;
4884                 default:
4885                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4886                         break;
4887                 }
4888
4889                 /* wptr/rptr are in bytes! */
4890                 rptr += 16;
4891                 rptr &= rdev->ih.ptr_mask;
4892                 WREG32(IH_RB_RPTR, rptr);
4893         }
4894         if (queue_dp)
4895                 schedule_work(&rdev->dp_work);
4896         if (queue_hotplug)
4897                 schedule_delayed_work(&rdev->hotplug_work, 0);
4898         if (queue_hdmi)
4899                 schedule_work(&rdev->audio_work);
4900         if (queue_thermal && rdev->pm.dpm_enabled)
4901                 schedule_work(&rdev->pm.dpm.thermal.work);
4902         rdev->ih.rptr = rptr;
4903         atomic_set(&rdev->ih.lock, 0);
4904
4905         /* make sure wptr hasn't changed while processing */
4906         wptr = evergreen_get_ih_wptr(rdev);
4907         if (wptr != rptr)
4908                 goto restart_ih;
4909
4910         return IRQ_HANDLED;
4911 }
4912
4913 static void evergreen_uvd_init(struct radeon_device *rdev)
4914 {
4915         int r;
4916
4917         if (!rdev->has_uvd)
4918                 return;
4919
4920         r = radeon_uvd_init(rdev);
4921         if (r) {
4922                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4923                 /*
4924                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4925                  * to early fails uvd_v2_2_resume() and thus nothing happens
4926                  * there. So it is pointless to try to go through that code
4927                  * hence why we disable uvd here.
4928                  */
4929                 rdev->has_uvd = 0;
4930                 return;
4931         }
4932         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4933         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4934 }
4935
4936 static void evergreen_uvd_start(struct radeon_device *rdev)
4937 {
4938         int r;
4939
4940         if (!rdev->has_uvd)
4941                 return;
4942
4943         r = uvd_v2_2_resume(rdev);
4944         if (r) {
4945                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4946                 goto error;
4947         }
4948         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4949         if (r) {
4950                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4951                 goto error;
4952         }
4953         return;
4954
4955 error:
4956         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4957 }
4958
4959 static void evergreen_uvd_resume(struct radeon_device *rdev)
4960 {
4961         struct radeon_ring *ring;
4962         int r;
4963
4964         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4965                 return;
4966
4967         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4968         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4969         if (r) {
4970                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4971                 return;
4972         }
4973         r = uvd_v1_0_init(rdev);
4974         if (r) {
4975                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4976                 return;
4977         }
4978 }
4979
4980 static int evergreen_startup(struct radeon_device *rdev)
4981 {
4982         struct radeon_ring *ring;
4983         int r;
4984
4985         /* enable pcie gen2 link */
4986         evergreen_pcie_gen2_enable(rdev);
4987         /* enable aspm */
4988         evergreen_program_aspm(rdev);
4989
4990         /* scratch needs to be initialized before MC */
4991         r = r600_vram_scratch_init(rdev);
4992         if (r)
4993                 return r;
4994
4995         evergreen_mc_program(rdev);
4996
4997         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
4998                 r = ni_mc_load_microcode(rdev);
4999                 if (r) {
5000                         DRM_ERROR("Failed to load MC firmware!\n");
5001                         return r;
5002                 }
5003         }
5004
5005         if (rdev->flags & RADEON_IS_AGP) {
5006                 evergreen_agp_enable(rdev);
5007         } else {
5008                 r = evergreen_pcie_gart_enable(rdev);
5009                 if (r)
5010                         return r;
5011         }
5012         evergreen_gpu_init(rdev);
5013
5014         /* allocate rlc buffers */
5015         if (rdev->flags & RADEON_IS_IGP) {
5016                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5017                 rdev->rlc.reg_list_size =
5018                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5019                 rdev->rlc.cs_data = evergreen_cs_data;
5020                 r = sumo_rlc_init(rdev);
5021                 if (r) {
5022                         DRM_ERROR("Failed to init rlc BOs!\n");
5023                         return r;
5024                 }
5025         }
5026
5027         /* allocate wb buffer */
5028         r = radeon_wb_init(rdev);
5029         if (r)
5030                 return r;
5031
5032         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5033         if (r) {
5034                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5035                 return r;
5036         }
5037
5038         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5039         if (r) {
5040                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5041                 return r;
5042         }
5043
5044         evergreen_uvd_start(rdev);
5045
5046         /* Enable IRQ */
5047         if (!rdev->irq.installed) {
5048                 r = radeon_irq_kms_init(rdev);
5049                 if (r)
5050                         return r;
5051         }
5052
5053         r = r600_irq_init(rdev);
5054         if (r) {
5055                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5056                 radeon_irq_kms_fini(rdev);
5057                 return r;
5058         }
5059         evergreen_irq_set(rdev);
5060
5061         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5062         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5063                              RADEON_CP_PACKET2);
5064         if (r)
5065                 return r;
5066
5067         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5068         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5069                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5070         if (r)
5071                 return r;
5072
5073         r = evergreen_cp_load_microcode(rdev);
5074         if (r)
5075                 return r;
5076         r = evergreen_cp_resume(rdev);
5077         if (r)
5078                 return r;
5079         r = r600_dma_resume(rdev);
5080         if (r)
5081                 return r;
5082
5083         evergreen_uvd_resume(rdev);
5084
5085         r = radeon_ib_pool_init(rdev);
5086         if (r) {
5087                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5088                 return r;
5089         }
5090
5091         r = radeon_audio_init(rdev);
5092         if (r) {
5093                 DRM_ERROR("radeon: audio init failed\n");
5094                 return r;
5095         }
5096
5097         return 0;
5098 }
5099
5100 int evergreen_resume(struct radeon_device *rdev)
5101 {
5102         int r;
5103
5104         /* reset the asic, the gfx blocks are often in a bad state
5105          * after the driver is unloaded or after a resume
5106          */
5107         if (radeon_asic_reset(rdev))
5108                 dev_warn(rdev->dev, "GPU reset failed !\n");
5109         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5110          * posting will perform necessary task to bring back GPU into good
5111          * shape.
5112          */
5113         /* post card */
5114         atom_asic_init(rdev->mode_info.atom_context);
5115
5116         /* init golden registers */
5117         evergreen_init_golden_registers(rdev);
5118
5119         if (rdev->pm.pm_method == PM_METHOD_DPM)
5120                 radeon_pm_resume(rdev);
5121
5122         rdev->accel_working = true;
5123         r = evergreen_startup(rdev);
5124         if (r) {
5125                 DRM_ERROR("evergreen startup failed on resume\n");
5126                 rdev->accel_working = false;
5127                 return r;
5128         }
5129
5130         return r;
5131
5132 }
5133
5134 int evergreen_suspend(struct radeon_device *rdev)
5135 {
5136         radeon_pm_suspend(rdev);
5137         radeon_audio_fini(rdev);
5138         if (rdev->has_uvd) {
5139                 uvd_v1_0_fini(rdev);
5140                 radeon_uvd_suspend(rdev);
5141         }
5142         r700_cp_stop(rdev);
5143         r600_dma_stop(rdev);
5144         evergreen_irq_suspend(rdev);
5145         radeon_wb_disable(rdev);
5146         evergreen_pcie_gart_disable(rdev);
5147
5148         return 0;
5149 }
5150
5151 /* Plan is to move initialization in that function and use
5152  * helper function so that radeon_device_init pretty much
5153  * do nothing more than calling asic specific function. This
5154  * should also allow to remove a bunch of callback function
5155  * like vram_info.
5156  */
5157 int evergreen_init(struct radeon_device *rdev)
5158 {
5159         int r;
5160
5161         /* Read BIOS */
5162         if (!radeon_get_bios(rdev)) {
5163                 if (ASIC_IS_AVIVO(rdev))
5164                         return -EINVAL;
5165         }
5166         /* Must be an ATOMBIOS */
5167         if (!rdev->is_atom_bios) {
5168                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5169                 return -EINVAL;
5170         }
5171         r = radeon_atombios_init(rdev);
5172         if (r)
5173                 return r;
5174         /* reset the asic, the gfx blocks are often in a bad state
5175          * after the driver is unloaded or after a resume
5176          */
5177         if (radeon_asic_reset(rdev))
5178                 dev_warn(rdev->dev, "GPU reset failed !\n");
5179         /* Post card if necessary */
5180         if (!radeon_card_posted(rdev)) {
5181                 if (!rdev->bios) {
5182                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5183                         return -EINVAL;
5184                 }
5185                 DRM_INFO("GPU not posted. posting now...\n");
5186                 atom_asic_init(rdev->mode_info.atom_context);
5187         }
5188         /* init golden registers */
5189         evergreen_init_golden_registers(rdev);
5190         /* Initialize scratch registers */
5191         r600_scratch_init(rdev);
5192         /* Initialize surface registers */
5193         radeon_surface_init(rdev);
5194         /* Initialize clocks */
5195         radeon_get_clock_info(rdev->ddev);
5196         /* Fence driver */
5197         r = radeon_fence_driver_init(rdev);
5198         if (r)
5199                 return r;
5200         /* initialize AGP */
5201         if (rdev->flags & RADEON_IS_AGP) {
5202                 r = radeon_agp_init(rdev);
5203                 if (r)
5204                         radeon_agp_disable(rdev);
5205         }
5206         /* initialize memory controller */
5207         r = evergreen_mc_init(rdev);
5208         if (r)
5209                 return r;
5210         /* Memory manager */
5211         r = radeon_bo_init(rdev);
5212         if (r)
5213                 return r;
5214
5215         if (ASIC_IS_DCE5(rdev)) {
5216                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5217                         r = ni_init_microcode(rdev);
5218                         if (r) {
5219                                 DRM_ERROR("Failed to load firmware!\n");
5220                                 return r;
5221                         }
5222                 }
5223         } else {
5224                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5225                         r = r600_init_microcode(rdev);
5226                         if (r) {
5227                                 DRM_ERROR("Failed to load firmware!\n");
5228                                 return r;
5229                         }
5230                 }
5231         }
5232
5233         /* Initialize power management */
5234         radeon_pm_init(rdev);
5235
5236         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5237         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5238
5239         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5240         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5241
5242         evergreen_uvd_init(rdev);
5243
5244         rdev->ih.ring_obj = NULL;
5245         r600_ih_ring_init(rdev, 64 * 1024);
5246
5247         r = r600_pcie_gart_init(rdev);
5248         if (r)
5249                 return r;
5250
5251 #ifdef __DragonFly__
5252         /*
5253            There are unresolved crashes on evergreen hardware,
5254            tell userland acceleration is not working properly
5255            Bug report: https://bugs.dragonflybsd.org/issues/3198
5256         */
5257         rdev->accel_working = false;
5258         DRM_ERROR("GPU acceleration disabled for now on DragonFly\n");
5259 #else
5260         rdev->accel_working = true;
5261 #endif
5262         r = evergreen_startup(rdev);
5263         if (r) {
5264                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5265                 r700_cp_fini(rdev);
5266                 r600_dma_fini(rdev);
5267                 r600_irq_fini(rdev);
5268                 if (rdev->flags & RADEON_IS_IGP)
5269                         sumo_rlc_fini(rdev);
5270                 radeon_wb_fini(rdev);
5271                 radeon_ib_pool_fini(rdev);
5272                 radeon_irq_kms_fini(rdev);
5273                 evergreen_pcie_gart_fini(rdev);
5274                 rdev->accel_working = false;
5275         }
5276
5277         /* Don't start up if the MC ucode is missing on BTC parts.
5278          * The default clocks and voltages before the MC ucode
5279          * is loaded are not suffient for advanced operations.
5280          */
5281         if (ASIC_IS_DCE5(rdev)) {
5282                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5283                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5284                         return -EINVAL;
5285                 }
5286         }
5287
5288         return 0;
5289 }
5290
5291 void evergreen_fini(struct radeon_device *rdev)
5292 {
5293         radeon_pm_fini(rdev);
5294         radeon_audio_fini(rdev);
5295         r700_cp_fini(rdev);
5296         r600_dma_fini(rdev);
5297         r600_irq_fini(rdev);
5298         if (rdev->flags & RADEON_IS_IGP)
5299                 sumo_rlc_fini(rdev);
5300         radeon_wb_fini(rdev);
5301         radeon_ib_pool_fini(rdev);
5302         radeon_irq_kms_fini(rdev);
5303         uvd_v1_0_fini(rdev);
5304         radeon_uvd_fini(rdev);
5305         evergreen_pcie_gart_fini(rdev);
5306         r600_vram_scratch_fini(rdev);
5307         radeon_gem_fini(rdev);
5308         radeon_fence_driver_fini(rdev);
5309         radeon_agp_fini(rdev);
5310         radeon_bo_fini(rdev);
5311         radeon_atombios_fini(rdev);
5312         kfree(rdev->bios);
5313         rdev->bios = NULL;
5314 }
5315
5316 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5317 {
5318         u32 link_width_cntl, speed_cntl;
5319         u32 mask;
5320
5321         if (radeon_pcie_gen2 == 0)
5322                 return;
5323
5324         if (rdev->flags & RADEON_IS_IGP)
5325                 return;
5326
5327         if (!(rdev->flags & RADEON_IS_PCIE))
5328                 return;
5329
5330         /* x2 cards have a special sequence */
5331         if (ASIC_IS_X2(rdev))
5332                 return;
5333
5334 #ifdef __DragonFly__
5335         if (drm_pcie_get_speed_cap_mask(rdev->ddev, &mask) != 0)
5336                 return;
5337
5338         rdev->pdev->bus->max_bus_speed = (mask &0xff);
5339 #endif
5340
5341         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5342                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5343                 return;
5344
5345         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5346         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5347                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5348                 return;
5349         }
5350
5351         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5352
5353         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5354             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5355
5356                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5357                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5358                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5359
5360                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5361                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5362                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5363
5364                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5365                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5366                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5367
5368                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5369                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5370                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5371
5372                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5373                 speed_cntl |= LC_GEN2_EN_STRAP;
5374                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5375
5376         } else {
5377                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5378                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5379                 if (1)
5380                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5381                 else
5382                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5383                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5384         }
5385 }
5386
5387 void evergreen_program_aspm(struct radeon_device *rdev)
5388 {
5389         u32 data, orig;
5390         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5391         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5392         /* fusion_platform = true
5393          * if the system is a fusion system
5394          * (APU or DGPU in a fusion system).
5395          * todo: check if the system is a fusion platform.
5396          */
5397         bool fusion_platform = false;
5398
5399         if (radeon_aspm == 0)
5400                 return;
5401
5402         if (!(rdev->flags & RADEON_IS_PCIE))
5403                 return;
5404
5405         switch (rdev->family) {
5406         case CHIP_CYPRESS:
5407         case CHIP_HEMLOCK:
5408         case CHIP_JUNIPER:
5409         case CHIP_REDWOOD:
5410         case CHIP_CEDAR:
5411         case CHIP_SUMO:
5412         case CHIP_SUMO2:
5413         case CHIP_PALM:
5414         case CHIP_ARUBA:
5415                 disable_l0s = true;
5416                 break;
5417         default:
5418                 disable_l0s = false;
5419                 break;
5420         }
5421
5422         if (rdev->flags & RADEON_IS_IGP)
5423                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5424
5425         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5426         if (fusion_platform)
5427                 data &= ~MULTI_PIF;
5428         else
5429                 data |= MULTI_PIF;
5430         if (data != orig)
5431                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5432
5433         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5434         if (fusion_platform)
5435                 data &= ~MULTI_PIF;
5436         else
5437                 data |= MULTI_PIF;
5438         if (data != orig)
5439                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5440
5441         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5442         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5443         if (!disable_l0s) {
5444                 if (rdev->family >= CHIP_BARTS)
5445                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5446                 else
5447                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5448         }
5449
5450         if (!disable_l1) {
5451                 if (rdev->family >= CHIP_BARTS)
5452                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5453                 else
5454                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5455
5456                 if (!disable_plloff_in_l1) {
5457                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5458                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5459                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5460                         if (data != orig)
5461                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5462
5463                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5464                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5465                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5466                         if (data != orig)
5467                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5468
5469                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5470                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5471                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5472                         if (data != orig)
5473                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5474
5475                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5476                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5477                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5478                         if (data != orig)
5479                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5480
5481                         if (rdev->family >= CHIP_BARTS) {
5482                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5483                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5484                                 data |= PLL_RAMP_UP_TIME_0(4);
5485                                 if (data != orig)
5486                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5487
5488                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5489                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5490                                 data |= PLL_RAMP_UP_TIME_1(4);
5491                                 if (data != orig)
5492                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5493
5494                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5495                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5496                                 data |= PLL_RAMP_UP_TIME_0(4);
5497                                 if (data != orig)
5498                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5499
5500                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5501                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5502                                 data |= PLL_RAMP_UP_TIME_1(4);
5503                                 if (data != orig)
5504                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5505                         }
5506
5507                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5508                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5509                         data |= LC_DYN_LANES_PWR_STATE(3);
5510                         if (data != orig)
5511                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5512
5513                         if (rdev->family >= CHIP_BARTS) {
5514                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5515                                 data &= ~LS2_EXIT_TIME_MASK;
5516                                 data |= LS2_EXIT_TIME(1);
5517                                 if (data != orig)
5518                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5519
5520                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5521                                 data &= ~LS2_EXIT_TIME_MASK;
5522                                 data |= LS2_EXIT_TIME(1);
5523                                 if (data != orig)
5524                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5525                         }
5526                 }
5527         }
5528
5529         /* evergreen parts only */
5530         if (rdev->family < CHIP_BARTS)
5531                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5532
5533         if (pcie_lc_cntl != pcie_lc_cntl_old)
5534                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5535 }