sys/dev/drm/radeon/rv770.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/firmware.h>
  29 #include <drm/drmP.h>
  30 #include "radeon.h"
  31 #include "radeon_asic.h"
  32 #include <uapi_drm/radeon_drm.h>
  33 #include "rv770d.h"
  34 #include "atom.h"
  35 #include "avivod.h"
  36
  37 #define R700_PFP_UCODE_SIZE 848
  38 #define R700_PM4_UCODE_SIZE 1360
  39
  40 static void rv770_gpu_init(struct radeon_device *rdev);
  41 static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  42
  43 int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  44 {
  45         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  46         int r;
  47
  48         /* RV740 uses evergreen uvd clk programming */
  49         if (rdev->family == CHIP_RV740)
  50                 return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  51
  52         /* bypass vclk and dclk with bclk */
  53         WREG32_P(CG_UPLL_FUNC_CNTL_2,
  54                  VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  55                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  56
  57         if (!vclk || !dclk) {
  58                 /* keep the Bypass mode, put PLL to sleep */
  59                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  60                 return 0;
  61         }
  62
  63         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  64                                           43663, 0x03FFFFFE, 1, 30, ~0,
  65                                           &fb_div, &vclk_div, &dclk_div);
  66         if (r)
  67                 return r;
  68
  69         fb_div |= 1;
  70         vclk_div -= 1;
  71         dclk_div -= 1;
  72
  73         /* set UPLL_FB_DIV to 0x50000 */
  74         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  75
  76         /* deassert UPLL_RESET and UPLL_SLEEP */
  77         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  78
  79         /* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  80         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  81         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  82
  83         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  84         if (r)
  85                 return r;
  86
  87         /* assert PLL_RESET */
  88         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
  89
  90         /* set the required FB_DIV, REF_DIV, Post divder values */
  91         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
  92         WREG32_P(CG_UPLL_FUNC_CNTL_2,
  93                  UPLL_SW_HILEN(vclk_div >> 1) |
  94                  UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
  95                  UPLL_SW_HILEN2(dclk_div >> 1) |
  96                  UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
  97                  ~UPLL_SW_MASK);
  98
  99         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 100                  ~UPLL_FB_DIV_MASK);
 101
 102         /* give the PLL some time to settle */
 103         mdelay(15);
 104
 105         /* deassert PLL_RESET */
 106         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 107
 108         mdelay(15);
 109
 110         /* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 111         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 112         WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 113
 114         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 115         if (r)
 116                 return r;
 117
 118         /* switch VCLK and DCLK selection */
 119         WREG32_P(CG_UPLL_FUNC_CNTL_2,
 120                  VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 121                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 122
 123         mdelay(100);
 124
 125         return 0;
 126 }
 127
 128 static const u32 r7xx_golden_registers[] =
 129 {
 130         0x8d00, 0xffffffff, 0x0e0e0074,
 131         0x8d04, 0xffffffff, 0x013a2b34,
 132         0x9508, 0xffffffff, 0x00000002,
 133         0x8b20, 0xffffffff, 0,
 134         0x88c4, 0xffffffff, 0x000000c2,
 135         0x28350, 0xffffffff, 0,
 136         0x9058, 0xffffffff, 0x0fffc40f,
 137         0x240c, 0xffffffff, 0x00000380,
 138         0x733c, 0xffffffff, 0x00000002,
 139         0x2650, 0x00040000, 0,
 140         0x20bc, 0x00040000, 0,
 141         0x7300, 0xffffffff, 0x001000f0
 142 };
 143
 144 static const u32 r7xx_golden_dyn_gpr_registers[] =
 145 {
 146         0x8db0, 0xffffffff, 0x98989898,
 147         0x8db4, 0xffffffff, 0x98989898,
 148         0x8db8, 0xffffffff, 0x98989898,
 149         0x8dbc, 0xffffffff, 0x98989898,
 150         0x8dc0, 0xffffffff, 0x98989898,
 151         0x8dc4, 0xffffffff, 0x98989898,
 152         0x8dc8, 0xffffffff, 0x98989898,
 153         0x8dcc, 0xffffffff, 0x98989898,
 154         0x88c4, 0xffffffff, 0x00000082
 155 };
 156
 157 static const u32 rv770_golden_registers[] =
 158 {
 159         0x562c, 0xffffffff, 0,
 160         0x3f90, 0xffffffff, 0,
 161         0x9148, 0xffffffff, 0,
 162         0x3f94, 0xffffffff, 0,
 163         0x914c, 0xffffffff, 0,
 164         0x9698, 0x18000000, 0x18000000
 165 };
 166
 167 static const u32 rv770ce_golden_registers[] =
 168 {
 169         0x562c, 0xffffffff, 0,
 170         0x3f90, 0xffffffff, 0x00cc0000,
 171         0x9148, 0xffffffff, 0x00cc0000,
 172         0x3f94, 0xffffffff, 0x00cc0000,
 173         0x914c, 0xffffffff, 0x00cc0000,
 174         0x9b7c, 0xffffffff, 0x00fa0000,
 175         0x3f8c, 0xffffffff, 0x00fa0000,
 176         0x9698, 0x18000000, 0x18000000
 177 };
 178
 179 static const u32 rv770_mgcg_init[] =
 180 {
 181         0x8bcc, 0xffffffff, 0x130300f9,
 182         0x5448, 0xffffffff, 0x100,
 183         0x55e4, 0xffffffff, 0x100,
 184         0x160c, 0xffffffff, 0x100,
 185         0x5644, 0xffffffff, 0x100,
 186         0xc164, 0xffffffff, 0x100,
 187         0x8a18, 0xffffffff, 0x100,
 188         0x897c, 0xffffffff, 0x8000100,
 189         0x8b28, 0xffffffff, 0x3c000100,
 190         0x9144, 0xffffffff, 0x100,
 191         0x9a1c, 0xffffffff, 0x10000,
 192         0x9a50, 0xffffffff, 0x100,
 193         0x9a1c, 0xffffffff, 0x10001,
 194         0x9a50, 0xffffffff, 0x100,
 195         0x9a1c, 0xffffffff, 0x10002,
 196         0x9a50, 0xffffffff, 0x100,
 197         0x9a1c, 0xffffffff, 0x10003,
 198         0x9a50, 0xffffffff, 0x100,
 199         0x9a1c, 0xffffffff, 0x0,
 200         0x9870, 0xffffffff, 0x100,
 201         0x8d58, 0xffffffff, 0x100,
 202         0x9500, 0xffffffff, 0x0,
 203         0x9510, 0xffffffff, 0x100,
 204         0x9500, 0xffffffff, 0x1,
 205         0x9510, 0xffffffff, 0x100,
 206         0x9500, 0xffffffff, 0x2,
 207         0x9510, 0xffffffff, 0x100,
 208         0x9500, 0xffffffff, 0x3,
 209         0x9510, 0xffffffff, 0x100,
 210         0x9500, 0xffffffff, 0x4,
 211         0x9510, 0xffffffff, 0x100,
 212         0x9500, 0xffffffff, 0x5,
 213         0x9510, 0xffffffff, 0x100,
 214         0x9500, 0xffffffff, 0x6,
 215         0x9510, 0xffffffff, 0x100,
 216         0x9500, 0xffffffff, 0x7,
 217         0x9510, 0xffffffff, 0x100,
 218         0x9500, 0xffffffff, 0x8,
 219         0x9510, 0xffffffff, 0x100,
 220         0x9500, 0xffffffff, 0x9,
 221         0x9510, 0xffffffff, 0x100,
 222         0x9500, 0xffffffff, 0x8000,
 223         0x9490, 0xffffffff, 0x0,
 224         0x949c, 0xffffffff, 0x100,
 225         0x9490, 0xffffffff, 0x1,
 226         0x949c, 0xffffffff, 0x100,
 227         0x9490, 0xffffffff, 0x2,
 228         0x949c, 0xffffffff, 0x100,
 229         0x9490, 0xffffffff, 0x3,
 230         0x949c, 0xffffffff, 0x100,
 231         0x9490, 0xffffffff, 0x4,
 232         0x949c, 0xffffffff, 0x100,
 233         0x9490, 0xffffffff, 0x5,
 234         0x949c, 0xffffffff, 0x100,
 235         0x9490, 0xffffffff, 0x6,
 236         0x949c, 0xffffffff, 0x100,
 237         0x9490, 0xffffffff, 0x7,
 238         0x949c, 0xffffffff, 0x100,
 239         0x9490, 0xffffffff, 0x8,
 240         0x949c, 0xffffffff, 0x100,
 241         0x9490, 0xffffffff, 0x9,
 242         0x949c, 0xffffffff, 0x100,
 243         0x9490, 0xffffffff, 0x8000,
 244         0x9604, 0xffffffff, 0x0,
 245         0x9654, 0xffffffff, 0x100,
 246         0x9604, 0xffffffff, 0x1,
 247         0x9654, 0xffffffff, 0x100,
 248         0x9604, 0xffffffff, 0x2,
 249         0x9654, 0xffffffff, 0x100,
 250         0x9604, 0xffffffff, 0x3,
 251         0x9654, 0xffffffff, 0x100,
 252         0x9604, 0xffffffff, 0x4,
 253         0x9654, 0xffffffff, 0x100,
 254         0x9604, 0xffffffff, 0x5,
 255         0x9654, 0xffffffff, 0x100,
 256         0x9604, 0xffffffff, 0x6,
 257         0x9654, 0xffffffff, 0x100,
 258         0x9604, 0xffffffff, 0x7,
 259         0x9654, 0xffffffff, 0x100,
 260         0x9604, 0xffffffff, 0x8,
 261         0x9654, 0xffffffff, 0x100,
 262         0x9604, 0xffffffff, 0x9,
 263         0x9654, 0xffffffff, 0x100,
 264         0x9604, 0xffffffff, 0x80000000,
 265         0x9030, 0xffffffff, 0x100,
 266         0x9034, 0xffffffff, 0x100,
 267         0x9038, 0xffffffff, 0x100,
 268         0x903c, 0xffffffff, 0x100,
 269         0x9040, 0xffffffff, 0x100,
 270         0xa200, 0xffffffff, 0x100,
 271         0xa204, 0xffffffff, 0x100,
 272         0xa208, 0xffffffff, 0x100,
 273         0xa20c, 0xffffffff, 0x100,
 274         0x971c, 0xffffffff, 0x100,
 275         0x915c, 0xffffffff, 0x00020001,
 276         0x9160, 0xffffffff, 0x00040003,
 277         0x916c, 0xffffffff, 0x00060005,
 278         0x9170, 0xffffffff, 0x00080007,
 279         0x9174, 0xffffffff, 0x000a0009,
 280         0x9178, 0xffffffff, 0x000c000b,
 281         0x917c, 0xffffffff, 0x000e000d,
 282         0x9180, 0xffffffff, 0x0010000f,
 283         0x918c, 0xffffffff, 0x00120011,
 284         0x9190, 0xffffffff, 0x00140013,
 285         0x9194, 0xffffffff, 0x00020001,
 286         0x9198, 0xffffffff, 0x00040003,
 287         0x919c, 0xffffffff, 0x00060005,
 288         0x91a8, 0xffffffff, 0x00080007,
 289         0x91ac, 0xffffffff, 0x000a0009,
 290         0x91b0, 0xffffffff, 0x000c000b,
 291         0x91b4, 0xffffffff, 0x000e000d,
 292         0x91b8, 0xffffffff, 0x0010000f,
 293         0x91c4, 0xffffffff, 0x00120011,
 294         0x91c8, 0xffffffff, 0x00140013,
 295         0x91cc, 0xffffffff, 0x00020001,
 296         0x91d0, 0xffffffff, 0x00040003,
 297         0x91d4, 0xffffffff, 0x00060005,
 298         0x91e0, 0xffffffff, 0x00080007,
 299         0x91e4, 0xffffffff, 0x000a0009,
 300         0x91e8, 0xffffffff, 0x000c000b,
 301         0x91ec, 0xffffffff, 0x00020001,
 302         0x91f0, 0xffffffff, 0x00040003,
 303         0x91f4, 0xffffffff, 0x00060005,
 304         0x9200, 0xffffffff, 0x00080007,
 305         0x9204, 0xffffffff, 0x000a0009,
 306         0x9208, 0xffffffff, 0x000c000b,
 307         0x920c, 0xffffffff, 0x000e000d,
 308         0x9210, 0xffffffff, 0x0010000f,
 309         0x921c, 0xffffffff, 0x00120011,
 310         0x9220, 0xffffffff, 0x00140013,
 311         0x9224, 0xffffffff, 0x00020001,
 312         0x9228, 0xffffffff, 0x00040003,
 313         0x922c, 0xffffffff, 0x00060005,
 314         0x9238, 0xffffffff, 0x00080007,
 315         0x923c, 0xffffffff, 0x000a0009,
 316         0x9240, 0xffffffff, 0x000c000b,
 317         0x9244, 0xffffffff, 0x000e000d,
 318         0x9248, 0xffffffff, 0x0010000f,
 319         0x9254, 0xffffffff, 0x00120011,
 320         0x9258, 0xffffffff, 0x00140013,
 321         0x925c, 0xffffffff, 0x00020001,
 322         0x9260, 0xffffffff, 0x00040003,
 323         0x9264, 0xffffffff, 0x00060005,
 324         0x9270, 0xffffffff, 0x00080007,
 325         0x9274, 0xffffffff, 0x000a0009,
 326         0x9278, 0xffffffff, 0x000c000b,
 327         0x927c, 0xffffffff, 0x000e000d,
 328         0x9280, 0xffffffff, 0x0010000f,
 329         0x928c, 0xffffffff, 0x00120011,
 330         0x9290, 0xffffffff, 0x00140013,
 331         0x9294, 0xffffffff, 0x00020001,
 332         0x929c, 0xffffffff, 0x00040003,
 333         0x92a0, 0xffffffff, 0x00060005,
 334         0x92a4, 0xffffffff, 0x00080007
 335 };
 336
 337 static const u32 rv710_golden_registers[] =
 338 {
 339         0x3f90, 0x00ff0000, 0x00fc0000,
 340         0x9148, 0x00ff0000, 0x00fc0000,
 341         0x3f94, 0x00ff0000, 0x00fc0000,
 342         0x914c, 0x00ff0000, 0x00fc0000,
 343         0xb4c, 0x00000020, 0x00000020,
 344         0xa180, 0xffffffff, 0x00003f3f
 345 };
 346
 347 static const u32 rv710_mgcg_init[] =
 348 {
 349         0x8bcc, 0xffffffff, 0x13030040,
 350         0x5448, 0xffffffff, 0x100,
 351         0x55e4, 0xffffffff, 0x100,
 352         0x160c, 0xffffffff, 0x100,
 353         0x5644, 0xffffffff, 0x100,
 354         0xc164, 0xffffffff, 0x100,
 355         0x8a18, 0xffffffff, 0x100,
 356         0x897c, 0xffffffff, 0x8000100,
 357         0x8b28, 0xffffffff, 0x3c000100,
 358         0x9144, 0xffffffff, 0x100,
 359         0x9a1c, 0xffffffff, 0x10000,
 360         0x9a50, 0xffffffff, 0x100,
 361         0x9a1c, 0xffffffff, 0x0,
 362         0x9870, 0xffffffff, 0x100,
 363         0x8d58, 0xffffffff, 0x100,
 364         0x9500, 0xffffffff, 0x0,
 365         0x9510, 0xffffffff, 0x100,
 366         0x9500, 0xffffffff, 0x1,
 367         0x9510, 0xffffffff, 0x100,
 368         0x9500, 0xffffffff, 0x8000,
 369         0x9490, 0xffffffff, 0x0,
 370         0x949c, 0xffffffff, 0x100,
 371         0x9490, 0xffffffff, 0x1,
 372         0x949c, 0xffffffff, 0x100,
 373         0x9490, 0xffffffff, 0x8000,
 374         0x9604, 0xffffffff, 0x0,
 375         0x9654, 0xffffffff, 0x100,
 376         0x9604, 0xffffffff, 0x1,
 377         0x9654, 0xffffffff, 0x100,
 378         0x9604, 0xffffffff, 0x80000000,
 379         0x9030, 0xffffffff, 0x100,
 380         0x9034, 0xffffffff, 0x100,
 381         0x9038, 0xffffffff, 0x100,
 382         0x903c, 0xffffffff, 0x100,
 383         0x9040, 0xffffffff, 0x100,
 384         0xa200, 0xffffffff, 0x100,
 385         0xa204, 0xffffffff, 0x100,
 386         0xa208, 0xffffffff, 0x100,
 387         0xa20c, 0xffffffff, 0x100,
 388         0x971c, 0xffffffff, 0x100,
 389         0x915c, 0xffffffff, 0x00020001,
 390         0x9174, 0xffffffff, 0x00000003,
 391         0x9178, 0xffffffff, 0x00050001,
 392         0x917c, 0xffffffff, 0x00030002,
 393         0x918c, 0xffffffff, 0x00000004,
 394         0x9190, 0xffffffff, 0x00070006,
 395         0x9194, 0xffffffff, 0x00050001,
 396         0x9198, 0xffffffff, 0x00030002,
 397         0x91a8, 0xffffffff, 0x00000004,
 398         0x91ac, 0xffffffff, 0x00070006,
 399         0x91e8, 0xffffffff, 0x00000001,
 400         0x9294, 0xffffffff, 0x00000001,
 401         0x929c, 0xffffffff, 0x00000002,
 402         0x92a0, 0xffffffff, 0x00040003,
 403         0x9150, 0xffffffff, 0x4d940000
 404 };
 405
 406 static const u32 rv730_golden_registers[] =
 407 {
 408         0x3f90, 0x00ff0000, 0x00f00000,
 409         0x9148, 0x00ff0000, 0x00f00000,
 410         0x3f94, 0x00ff0000, 0x00f00000,
 411         0x914c, 0x00ff0000, 0x00f00000,
 412         0x900c, 0xffffffff, 0x003b033f,
 413         0xb4c, 0x00000020, 0x00000020,
 414         0xa180, 0xffffffff, 0x00003f3f
 415 };
 416
 417 static const u32 rv730_mgcg_init[] =
 418 {
 419         0x8bcc, 0xffffffff, 0x130300f9,
 420         0x5448, 0xffffffff, 0x100,
 421         0x55e4, 0xffffffff, 0x100,
 422         0x160c, 0xffffffff, 0x100,
 423         0x5644, 0xffffffff, 0x100,
 424         0xc164, 0xffffffff, 0x100,
 425         0x8a18, 0xffffffff, 0x100,
 426         0x897c, 0xffffffff, 0x8000100,
 427         0x8b28, 0xffffffff, 0x3c000100,
 428         0x9144, 0xffffffff, 0x100,
 429         0x9a1c, 0xffffffff, 0x10000,
 430         0x9a50, 0xffffffff, 0x100,
 431         0x9a1c, 0xffffffff, 0x10001,
 432         0x9a50, 0xffffffff, 0x100,
 433         0x9a1c, 0xffffffff, 0x0,
 434         0x9870, 0xffffffff, 0x100,
 435         0x8d58, 0xffffffff, 0x100,
 436         0x9500, 0xffffffff, 0x0,
 437         0x9510, 0xffffffff, 0x100,
 438         0x9500, 0xffffffff, 0x1,
 439         0x9510, 0xffffffff, 0x100,
 440         0x9500, 0xffffffff, 0x2,
 441         0x9510, 0xffffffff, 0x100,
 442         0x9500, 0xffffffff, 0x3,
 443         0x9510, 0xffffffff, 0x100,
 444         0x9500, 0xffffffff, 0x4,
 445         0x9510, 0xffffffff, 0x100,
 446         0x9500, 0xffffffff, 0x5,
 447         0x9510, 0xffffffff, 0x100,
 448         0x9500, 0xffffffff, 0x6,
 449         0x9510, 0xffffffff, 0x100,
 450         0x9500, 0xffffffff, 0x7,
 451         0x9510, 0xffffffff, 0x100,
 452         0x9500, 0xffffffff, 0x8000,
 453         0x9490, 0xffffffff, 0x0,
 454         0x949c, 0xffffffff, 0x100,
 455         0x9490, 0xffffffff, 0x1,
 456         0x949c, 0xffffffff, 0x100,
 457         0x9490, 0xffffffff, 0x2,
 458         0x949c, 0xffffffff, 0x100,
 459         0x9490, 0xffffffff, 0x3,
 460         0x949c, 0xffffffff, 0x100,
 461         0x9490, 0xffffffff, 0x4,
 462         0x949c, 0xffffffff, 0x100,
 463         0x9490, 0xffffffff, 0x5,
 464         0x949c, 0xffffffff, 0x100,
 465         0x9490, 0xffffffff, 0x6,
 466         0x949c, 0xffffffff, 0x100,
 467         0x9490, 0xffffffff, 0x7,
 468         0x949c, 0xffffffff, 0x100,
 469         0x9490, 0xffffffff, 0x8000,
 470         0x9604, 0xffffffff, 0x0,
 471         0x9654, 0xffffffff, 0x100,
 472         0x9604, 0xffffffff, 0x1,
 473         0x9654, 0xffffffff, 0x100,
 474         0x9604, 0xffffffff, 0x2,
 475         0x9654, 0xffffffff, 0x100,
 476         0x9604, 0xffffffff, 0x3,
 477         0x9654, 0xffffffff, 0x100,
 478         0x9604, 0xffffffff, 0x4,
 479         0x9654, 0xffffffff, 0x100,
 480         0x9604, 0xffffffff, 0x5,
 481         0x9654, 0xffffffff, 0x100,
 482         0x9604, 0xffffffff, 0x6,
 483         0x9654, 0xffffffff, 0x100,
 484         0x9604, 0xffffffff, 0x7,
 485         0x9654, 0xffffffff, 0x100,
 486         0x9604, 0xffffffff, 0x80000000,
 487         0x9030, 0xffffffff, 0x100,
 488         0x9034, 0xffffffff, 0x100,
 489         0x9038, 0xffffffff, 0x100,
 490         0x903c, 0xffffffff, 0x100,
 491         0x9040, 0xffffffff, 0x100,
 492         0xa200, 0xffffffff, 0x100,
 493         0xa204, 0xffffffff, 0x100,
 494         0xa208, 0xffffffff, 0x100,
 495         0xa20c, 0xffffffff, 0x100,
 496         0x971c, 0xffffffff, 0x100,
 497         0x915c, 0xffffffff, 0x00020001,
 498         0x916c, 0xffffffff, 0x00040003,
 499         0x9170, 0xffffffff, 0x00000005,
 500         0x9178, 0xffffffff, 0x00050001,
 501         0x917c, 0xffffffff, 0x00030002,
 502         0x918c, 0xffffffff, 0x00000004,
 503         0x9190, 0xffffffff, 0x00070006,
 504         0x9194, 0xffffffff, 0x00050001,
 505         0x9198, 0xffffffff, 0x00030002,
 506         0x91a8, 0xffffffff, 0x00000004,
 507         0x91ac, 0xffffffff, 0x00070006,
 508         0x91b0, 0xffffffff, 0x00050001,
 509         0x91b4, 0xffffffff, 0x00030002,
 510         0x91c4, 0xffffffff, 0x00000004,
 511         0x91c8, 0xffffffff, 0x00070006,
 512         0x91cc, 0xffffffff, 0x00050001,
 513         0x91d0, 0xffffffff, 0x00030002,
 514         0x91e0, 0xffffffff, 0x00000004,
 515         0x91e4, 0xffffffff, 0x00070006,
 516         0x91e8, 0xffffffff, 0x00000001,
 517         0x91ec, 0xffffffff, 0x00050001,
 518         0x91f0, 0xffffffff, 0x00030002,
 519         0x9200, 0xffffffff, 0x00000004,
 520         0x9204, 0xffffffff, 0x00070006,
 521         0x9208, 0xffffffff, 0x00050001,
 522         0x920c, 0xffffffff, 0x00030002,
 523         0x921c, 0xffffffff, 0x00000004,
 524         0x9220, 0xffffffff, 0x00070006,
 525         0x9224, 0xffffffff, 0x00050001,
 526         0x9228, 0xffffffff, 0x00030002,
 527         0x9238, 0xffffffff, 0x00000004,
 528         0x923c, 0xffffffff, 0x00070006,
 529         0x9240, 0xffffffff, 0x00050001,
 530         0x9244, 0xffffffff, 0x00030002,
 531         0x9254, 0xffffffff, 0x00000004,
 532         0x9258, 0xffffffff, 0x00070006,
 533         0x9294, 0xffffffff, 0x00000001,
 534         0x929c, 0xffffffff, 0x00000002,
 535         0x92a0, 0xffffffff, 0x00040003,
 536         0x92a4, 0xffffffff, 0x00000005
 537 };
 538
 539 static const u32 rv740_golden_registers[] =
 540 {
 541         0x88c4, 0xffffffff, 0x00000082,
 542         0x28a50, 0xfffffffc, 0x00000004,
 543         0x2650, 0x00040000, 0,
 544         0x20bc, 0x00040000, 0,
 545         0x733c, 0xffffffff, 0x00000002,
 546         0x7300, 0xffffffff, 0x001000f0,
 547         0x3f90, 0x00ff0000, 0,
 548         0x9148, 0x00ff0000, 0,
 549         0x3f94, 0x00ff0000, 0,
 550         0x914c, 0x00ff0000, 0,
 551         0x240c, 0xffffffff, 0x00000380,
 552         0x8a14, 0x00000007, 0x00000007,
 553         0x8b24, 0xffffffff, 0x00ff0fff,
 554         0x28a4c, 0xffffffff, 0x00004000,
 555         0xa180, 0xffffffff, 0x00003f3f,
 556         0x8d00, 0xffffffff, 0x0e0e003a,
 557         0x8d04, 0xffffffff, 0x013a0e2a,
 558         0x8c00, 0xffffffff, 0xe400000f,
 559         0x8db0, 0xffffffff, 0x98989898,
 560         0x8db4, 0xffffffff, 0x98989898,
 561         0x8db8, 0xffffffff, 0x98989898,
 562         0x8dbc, 0xffffffff, 0x98989898,
 563         0x8dc0, 0xffffffff, 0x98989898,
 564         0x8dc4, 0xffffffff, 0x98989898,
 565         0x8dc8, 0xffffffff, 0x98989898,
 566         0x8dcc, 0xffffffff, 0x98989898,
 567         0x9058, 0xffffffff, 0x0fffc40f,
 568         0x900c, 0xffffffff, 0x003b033f,
 569         0x28350, 0xffffffff, 0,
 570         0x8cf0, 0x1fffffff, 0x08e00420,
 571         0x9508, 0xffffffff, 0x00000002,
 572         0x88c4, 0xffffffff, 0x000000c2,
 573         0x9698, 0x18000000, 0x18000000
 574 };
 575
 576 static const u32 rv740_mgcg_init[] =
 577 {
 578         0x8bcc, 0xffffffff, 0x13030100,
 579         0x5448, 0xffffffff, 0x100,
 580         0x55e4, 0xffffffff, 0x100,
 581         0x160c, 0xffffffff, 0x100,
 582         0x5644, 0xffffffff, 0x100,
 583         0xc164, 0xffffffff, 0x100,
 584         0x8a18, 0xffffffff, 0x100,
 585         0x897c, 0xffffffff, 0x100,
 586         0x8b28, 0xffffffff, 0x100,
 587         0x9144, 0xffffffff, 0x100,
 588         0x9a1c, 0xffffffff, 0x10000,
 589         0x9a50, 0xffffffff, 0x100,
 590         0x9a1c, 0xffffffff, 0x10001,
 591         0x9a50, 0xffffffff, 0x100,
 592         0x9a1c, 0xffffffff, 0x10002,
 593         0x9a50, 0xffffffff, 0x100,
 594         0x9a1c, 0xffffffff, 0x10003,
 595         0x9a50, 0xffffffff, 0x100,
 596         0x9a1c, 0xffffffff, 0x0,
 597         0x9870, 0xffffffff, 0x100,
 598         0x8d58, 0xffffffff, 0x100,
 599         0x9500, 0xffffffff, 0x0,
 600         0x9510, 0xffffffff, 0x100,
 601         0x9500, 0xffffffff, 0x1,
 602         0x9510, 0xffffffff, 0x100,
 603         0x9500, 0xffffffff, 0x2,
 604         0x9510, 0xffffffff, 0x100,
 605         0x9500, 0xffffffff, 0x3,
 606         0x9510, 0xffffffff, 0x100,
 607         0x9500, 0xffffffff, 0x4,
 608         0x9510, 0xffffffff, 0x100,
 609         0x9500, 0xffffffff, 0x5,
 610         0x9510, 0xffffffff, 0x100,
 611         0x9500, 0xffffffff, 0x6,
 612         0x9510, 0xffffffff, 0x100,
 613         0x9500, 0xffffffff, 0x7,
 614         0x9510, 0xffffffff, 0x100,
 615         0x9500, 0xffffffff, 0x8000,
 616         0x9490, 0xffffffff, 0x0,
 617         0x949c, 0xffffffff, 0x100,
 618         0x9490, 0xffffffff, 0x1,
 619         0x949c, 0xffffffff, 0x100,
 620         0x9490, 0xffffffff, 0x2,
 621         0x949c, 0xffffffff, 0x100,
 622         0x9490, 0xffffffff, 0x3,
 623         0x949c, 0xffffffff, 0x100,
 624         0x9490, 0xffffffff, 0x4,
 625         0x949c, 0xffffffff, 0x100,
 626         0x9490, 0xffffffff, 0x5,
 627         0x949c, 0xffffffff, 0x100,
 628         0x9490, 0xffffffff, 0x6,
 629         0x949c, 0xffffffff, 0x100,
 630         0x9490, 0xffffffff, 0x7,
 631         0x949c, 0xffffffff, 0x100,
 632         0x9490, 0xffffffff, 0x8000,
 633         0x9604, 0xffffffff, 0x0,
 634         0x9654, 0xffffffff, 0x100,
 635         0x9604, 0xffffffff, 0x1,
 636         0x9654, 0xffffffff, 0x100,
 637         0x9604, 0xffffffff, 0x2,
 638         0x9654, 0xffffffff, 0x100,
 639         0x9604, 0xffffffff, 0x3,
 640         0x9654, 0xffffffff, 0x100,
 641         0x9604, 0xffffffff, 0x4,
 642         0x9654, 0xffffffff, 0x100,
 643         0x9604, 0xffffffff, 0x5,
 644         0x9654, 0xffffffff, 0x100,
 645         0x9604, 0xffffffff, 0x6,
 646         0x9654, 0xffffffff, 0x100,
 647         0x9604, 0xffffffff, 0x7,
 648         0x9654, 0xffffffff, 0x100,
 649         0x9604, 0xffffffff, 0x80000000,
 650         0x9030, 0xffffffff, 0x100,
 651         0x9034, 0xffffffff, 0x100,
 652         0x9038, 0xffffffff, 0x100,
 653         0x903c, 0xffffffff, 0x100,
 654         0x9040, 0xffffffff, 0x100,
 655         0xa200, 0xffffffff, 0x100,
 656         0xa204, 0xffffffff, 0x100,
 657         0xa208, 0xffffffff, 0x100,
 658         0xa20c, 0xffffffff, 0x100,
 659         0x971c, 0xffffffff, 0x100,
 660         0x915c, 0xffffffff, 0x00020001,
 661         0x9160, 0xffffffff, 0x00040003,
 662         0x916c, 0xffffffff, 0x00060005,
 663         0x9170, 0xffffffff, 0x00080007,
 664         0x9174, 0xffffffff, 0x000a0009,
 665         0x9178, 0xffffffff, 0x000c000b,
 666         0x917c, 0xffffffff, 0x000e000d,
 667         0x9180, 0xffffffff, 0x0010000f,
 668         0x918c, 0xffffffff, 0x00120011,
 669         0x9190, 0xffffffff, 0x00140013,
 670         0x9194, 0xffffffff, 0x00020001,
 671         0x9198, 0xffffffff, 0x00040003,
 672         0x919c, 0xffffffff, 0x00060005,
 673         0x91a8, 0xffffffff, 0x00080007,
 674         0x91ac, 0xffffffff, 0x000a0009,
 675         0x91b0, 0xffffffff, 0x000c000b,
 676         0x91b4, 0xffffffff, 0x000e000d,
 677         0x91b8, 0xffffffff, 0x0010000f,
 678         0x91c4, 0xffffffff, 0x00120011,
 679         0x91c8, 0xffffffff, 0x00140013,
 680         0x91cc, 0xffffffff, 0x00020001,
 681         0x91d0, 0xffffffff, 0x00040003,
 682         0x91d4, 0xffffffff, 0x00060005,
 683         0x91e0, 0xffffffff, 0x00080007,
 684         0x91e4, 0xffffffff, 0x000a0009,
 685         0x91e8, 0xffffffff, 0x000c000b,
 686         0x91ec, 0xffffffff, 0x00020001,
 687         0x91f0, 0xffffffff, 0x00040003,
 688         0x91f4, 0xffffffff, 0x00060005,
 689         0x9200, 0xffffffff, 0x00080007,
 690         0x9204, 0xffffffff, 0x000a0009,
 691         0x9208, 0xffffffff, 0x000c000b,
 692         0x920c, 0xffffffff, 0x000e000d,
 693         0x9210, 0xffffffff, 0x0010000f,
 694         0x921c, 0xffffffff, 0x00120011,
 695         0x9220, 0xffffffff, 0x00140013,
 696         0x9224, 0xffffffff, 0x00020001,
 697         0x9228, 0xffffffff, 0x00040003,
 698         0x922c, 0xffffffff, 0x00060005,
 699         0x9238, 0xffffffff, 0x00080007,
 700         0x923c, 0xffffffff, 0x000a0009,
 701         0x9240, 0xffffffff, 0x000c000b,
 702         0x9244, 0xffffffff, 0x000e000d,
 703         0x9248, 0xffffffff, 0x0010000f,
 704         0x9254, 0xffffffff, 0x00120011,
 705         0x9258, 0xffffffff, 0x00140013,
 706         0x9294, 0xffffffff, 0x00020001,
 707         0x929c, 0xffffffff, 0x00040003,
 708         0x92a0, 0xffffffff, 0x00060005,
 709         0x92a4, 0xffffffff, 0x00080007
 710 };
 711
 712 static void rv770_init_golden_registers(struct radeon_device *rdev)
 713 {
 714         switch (rdev->family) {
 715         case CHIP_RV770:
 716                 radeon_program_register_sequence(rdev,
 717                                                  r7xx_golden_registers,
 718                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 719                 radeon_program_register_sequence(rdev,
 720                                                  r7xx_golden_dyn_gpr_registers,
 721                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 722                 if (rdev->pdev->device == 0x994e)
 723                         radeon_program_register_sequence(rdev,
 724                                                          rv770ce_golden_registers,
 725                                                          (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 726                 else
 727                         radeon_program_register_sequence(rdev,
 728                                                          rv770_golden_registers,
 729                                                          (const u32)ARRAY_SIZE(rv770_golden_registers));
 730                 radeon_program_register_sequence(rdev,
 731                                                  rv770_mgcg_init,
 732                                                  (const u32)ARRAY_SIZE(rv770_mgcg_init));
 733                 break;
 734         case CHIP_RV730:
 735                 radeon_program_register_sequence(rdev,
 736                                                  r7xx_golden_registers,
 737                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 738                 radeon_program_register_sequence(rdev,
 739                                                  r7xx_golden_dyn_gpr_registers,
 740                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 741                 radeon_program_register_sequence(rdev,
 742                                                  rv730_golden_registers,
 743                                                  (const u32)ARRAY_SIZE(rv730_golden_registers));
 744                 radeon_program_register_sequence(rdev,
 745                                                  rv730_mgcg_init,
 746                                                  (const u32)ARRAY_SIZE(rv730_mgcg_init));
 747                 break;
 748         case CHIP_RV710:
 749                 radeon_program_register_sequence(rdev,
 750                                                  r7xx_golden_registers,
 751                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 752                 radeon_program_register_sequence(rdev,
 753                                                  r7xx_golden_dyn_gpr_registers,
 754                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 755                 radeon_program_register_sequence(rdev,
 756                                                  rv710_golden_registers,
 757                                                  (const u32)ARRAY_SIZE(rv710_golden_registers));
 758                 radeon_program_register_sequence(rdev,
 759                                                  rv710_mgcg_init,
 760                                                  (const u32)ARRAY_SIZE(rv710_mgcg_init));
 761                 break;
 762         case CHIP_RV740:
 763                 radeon_program_register_sequence(rdev,
 764                                                  rv740_golden_registers,
 765                                                  (const u32)ARRAY_SIZE(rv740_golden_registers));
 766                 radeon_program_register_sequence(rdev,
 767                                                  rv740_mgcg_init,
 768                                                  (const u32)ARRAY_SIZE(rv740_mgcg_init));
 769                 break;
 770         default:
 771                 break;
 772         }
 773 }
 774
 775 #define PCIE_BUS_CLK                10000
 776 #define TCLK                        (PCIE_BUS_CLK / 10)
 777
 778 /**
 779  * rv770_get_xclk - get the xclk
 780  *
 781  * @rdev: radeon_device pointer
 782  *
 783  * Returns the reference clock used by the gfx engine
 784  * (r7xx-cayman).
 785  */
 786 u32 rv770_get_xclk(struct radeon_device *rdev)
 787 {
 788         u32 reference_clock = rdev->clock.spll.reference_freq;
 789         u32 tmp = RREG32(CG_CLKPIN_CNTL);
 790
 791         if (tmp & MUX_TCLK_TO_XCLK)
 792                 return TCLK;
 793
 794         if (tmp & XTALIN_DIVIDE)
 795                 return reference_clock / 4;
 796
 797         return reference_clock;
 798 }
 799
 800 void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 801 {
 802         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 803         u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 804         int i;
 805
 806         /* Lock the graphics update lock */
 807         tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 808         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 809
 810         /* update the scanout addresses */
 811         if (radeon_crtc->crtc_id) {
 812                 WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 813                 WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 814         } else {
 815                 WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 816                 WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 817         }
 818         WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 819                (u32)crtc_base);
 820         WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 821                (u32)crtc_base);
 822
 823         /* Wait for update_pending to go high. */
 824         for (i = 0; i < rdev->usec_timeout; i++) {
 825                 if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 826                         break;
 827                 udelay(1);
 828         }
 829         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 830
 831         /* Unlock the lock, so double-buffering can take place inside vblank */
 832         tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 833         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 834 }
 835
 836 bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 837 {
 838         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 839
 840         /* Return current update_pending status: */
 841         return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 842                 AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 843 }
 844
 845 /* get temperature in millidegrees */
 846 int rv770_get_temp(struct radeon_device *rdev)
 847 {
 848         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 849                 ASIC_T_SHIFT;
 850         int actual_temp;
 851
 852         if (temp & 0x400)
 853                 actual_temp = -256;
 854         else if (temp & 0x200)
 855                 actual_temp = 255;
 856         else if (temp & 0x100) {
 857                 actual_temp = temp & 0x1ff;
 858                 actual_temp |= ~0x1ff;
 859         } else
 860                 actual_temp = temp & 0xff;
 861
 862         return (actual_temp * 1000) / 2;
 863 }
 864
 865 void rv770_pm_misc(struct radeon_device *rdev)
 866 {
 867         int req_ps_idx = rdev->pm.requested_power_state_index;
 868         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 869         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 870         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 871
 872         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 873                 /* 0xff01 is a flag rather then an actual voltage */
 874                 if (voltage->voltage == 0xff01)
 875                         return;
 876                 if (voltage->voltage != rdev->pm.current_vddc) {
 877                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 878                         rdev->pm.current_vddc = voltage->voltage;
 879                         DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 880                 }
 881         }
 882 }
 883
 884 /*
 885  * GART
 886  */
 887 static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 888 {
 889         u32 tmp;
 890         int r, i;
 891
 892         if (rdev->gart.robj == NULL) {
 893                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 894                 return -EINVAL;
 895         }
 896         r = radeon_gart_table_vram_pin(rdev);
 897         if (r)
 898                 return r;
 899         /* Setup L2 cache */
 900         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 901                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 902                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 903         WREG32(VM_L2_CNTL2, 0);
 904         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 905         /* Setup TLB control */
 906         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 907                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 908                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 909                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 910         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 911         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 912         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 913         if (rdev->family == CHIP_RV740)
 914                 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 915         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 916         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 917         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 918         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 919         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 920         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 921         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 922         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 923                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 924         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 925                         (u32)(rdev->dummy_page.addr >> 12));
 926         for (i = 1; i < 7; i++)
 927                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 928
 929         r600_pcie_gart_tlb_flush(rdev);
 930         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 931                  (unsigned)(rdev->mc.gtt_size >> 20),
 932                  (unsigned long long)rdev->gart.table_addr);
 933         rdev->gart.ready = true;
 934         return 0;
 935 }
 936
 937 static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 938 {
 939         u32 tmp;
 940         int i;
 941
 942         /* Disable all tables */
 943         for (i = 0; i < 7; i++)
 944                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 945
 946         /* Setup L2 cache */
 947         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 948                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 949         WREG32(VM_L2_CNTL2, 0);
 950         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 951         /* Setup TLB control */
 952         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 953         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 954         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 955         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 956         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 957         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 958         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 959         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 960         radeon_gart_table_vram_unpin(rdev);
 961 }
 962
 963 static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 964 {
 965         radeon_gart_fini(rdev);
 966         rv770_pcie_gart_disable(rdev);
 967         radeon_gart_table_vram_free(rdev);
 968 }
 969
 970
 971 static void rv770_agp_enable(struct radeon_device *rdev)
 972 {
 973         u32 tmp;
 974         int i;
 975
 976         /* Setup L2 cache */
 977         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 978                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 979                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 980         WREG32(VM_L2_CNTL2, 0);
 981         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 982         /* Setup TLB control */
 983         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 984                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 985                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 986                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 987         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 988         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 989         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 990         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 991         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 992         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 993         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 994         for (i = 0; i < 7; i++)
 995                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 996 }
 997
 998 static void rv770_mc_program(struct radeon_device *rdev)
 999 {
1000         struct rv515_mc_save save;
1001         u32 tmp;
1002         int i, j;
1003
1004         /* Initialize HDP */
1005         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1006                 WREG32((0x2c14 + j), 0x00000000);
1007                 WREG32((0x2c18 + j), 0x00000000);
1008                 WREG32((0x2c1c + j), 0x00000000);
1009                 WREG32((0x2c20 + j), 0x00000000);
1010                 WREG32((0x2c24 + j), 0x00000000);
1011         }
1012         /* r7xx hw bug.  Read from HDP_DEBUG1 rather
1013          * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1014          */
1015         tmp = RREG32(HDP_DEBUG1);
1016
1017         rv515_mc_stop(rdev, &save);
1018         if (r600_mc_wait_for_idle(rdev)) {
1019                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1020         }
1021         /* Lockout access through VGA aperture*/
1022         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1023         /* Update configuration */
1024         if (rdev->flags & RADEON_IS_AGP) {
1025                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1026                         /* VRAM before AGP */
1027                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1028                                 rdev->mc.vram_start >> 12);
1029                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1030                                 rdev->mc.gtt_end >> 12);
1031                 } else {
1032                         /* VRAM after AGP */
1033                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1034                                 rdev->mc.gtt_start >> 12);
1035                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1036                                 rdev->mc.vram_end >> 12);
1037                 }
1038         } else {
1039                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1040                         rdev->mc.vram_start >> 12);
1041                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1042                         rdev->mc.vram_end >> 12);
1043         }
1044         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1045         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1046         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1047         WREG32(MC_VM_FB_LOCATION, tmp);
1048         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1049         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1050         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1051         if (rdev->flags & RADEON_IS_AGP) {
1052                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1053                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1054                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1055         } else {
1056                 WREG32(MC_VM_AGP_BASE, 0);
1057                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1058                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1059         }
1060         if (r600_mc_wait_for_idle(rdev)) {
1061                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1062         }
1063         rv515_mc_resume(rdev, &save);
1064         /* we need to own VRAM, so turn off the VGA renderer here
1065          * to stop it overwriting our objects */
1066         rv515_vga_render_disable(rdev);
1067 }
1068
1069
1070 /*
1071  * CP.
1072  */
1073 void r700_cp_stop(struct radeon_device *rdev)
1074 {
1075         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1076                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1077         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1078         WREG32(SCRATCH_UMSK, 0);
1079         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1080 }
1081
1082 static int rv770_cp_load_microcode(struct radeon_device *rdev)
1083 {
1084         const __be32 *fw_data;
1085         int i;
1086
1087         if (!rdev->me_fw || !rdev->pfp_fw)
1088                 return -EINVAL;
1089
1090         r700_cp_stop(rdev);
1091         WREG32(CP_RB_CNTL,
1092 #ifdef __BIG_ENDIAN
1093                BUF_SWAP_32BIT |
1094 #endif
1095                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1096
1097         /* Reset cp */
1098         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1099         RREG32(GRBM_SOFT_RESET);
1100         mdelay(15);
1101         WREG32(GRBM_SOFT_RESET, 0);
1102
1103         fw_data = (const __be32 *)rdev->pfp_fw->data;
1104         WREG32(CP_PFP_UCODE_ADDR, 0);
1105         for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1106                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1107         WREG32(CP_PFP_UCODE_ADDR, 0);
1108
1109         fw_data = (const __be32 *)rdev->me_fw->data;
1110         WREG32(CP_ME_RAM_WADDR, 0);
1111         for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1112                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1113
1114         WREG32(CP_PFP_UCODE_ADDR, 0);
1115         WREG32(CP_ME_RAM_WADDR, 0);
1116         WREG32(CP_ME_RAM_RADDR, 0);
1117         return 0;
1118 }
1119
1120 void r700_cp_fini(struct radeon_device *rdev)
1121 {
1122         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1123         r700_cp_stop(rdev);
1124         radeon_ring_fini(rdev, ring);
1125         radeon_scratch_free(rdev, ring->rptr_save_reg);
1126 }
1127
1128 void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1129 {
1130         u32 tmp, i;
1131
1132         if (rdev->flags & RADEON_IS_IGP)
1133                 return;
1134
1135         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1136         tmp &= SCLK_MUX_SEL_MASK;
1137         tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1138         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1139
1140         for (i = 0; i < rdev->usec_timeout; i++) {
1141                 if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1142                         break;
1143                 udelay(1);
1144         }
1145
1146         tmp &= ~SCLK_MUX_UPDATE;
1147         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1148
1149         tmp = RREG32(MPLL_CNTL_MODE);
1150         if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1151                 tmp &= ~RV730_MPLL_MCLK_SEL;
1152         else
1153                 tmp &= ~MPLL_MCLK_SEL;
1154         WREG32(MPLL_CNTL_MODE, tmp);
1155 }
1156
1157 /*
1158  * Core functions
1159  */
1160 static void rv770_gpu_init(struct radeon_device *rdev)
1161 {
1162         int i, j, num_qd_pipes;
1163         u32 ta_aux_cntl;
1164         u32 sx_debug_1;
1165         u32 smx_dc_ctl0;
1166         u32 db_debug3;
1167         u32 num_gs_verts_per_thread;
1168         u32 vgt_gs_per_es;
1169         u32 gs_prim_buffer_depth = 0;
1170         u32 sq_ms_fifo_sizes;
1171         u32 sq_config;
1172         u32 sq_thread_resource_mgmt;
1173         u32 hdp_host_path_cntl;
1174         u32 sq_dyn_gpr_size_simd_ab_0;
1175         u32 gb_tiling_config = 0;
1176         u32 cc_gc_shader_pipe_config = 0;
1177         u32 mc_arb_ramcfg;
1178         u32 db_debug4, tmp;
1179         u32 inactive_pipes, shader_pipe_config;
1180         u32 disabled_rb_mask;
1181         unsigned active_number;
1182
1183         /* setup chip specs */
1184         rdev->config.rv770.tiling_group_size = 256;
1185         switch (rdev->family) {
1186         case CHIP_RV770:
1187                 rdev->config.rv770.max_pipes = 4;
1188                 rdev->config.rv770.max_tile_pipes = 8;
1189                 rdev->config.rv770.max_simds = 10;
1190                 rdev->config.rv770.max_backends = 4;
1191                 rdev->config.rv770.max_gprs = 256;
1192                 rdev->config.rv770.max_threads = 248;
1193                 rdev->config.rv770.max_stack_entries = 512;
1194                 rdev->config.rv770.max_hw_contexts = 8;
1195                 rdev->config.rv770.max_gs_threads = 16 * 2;
1196                 rdev->config.rv770.sx_max_export_size = 128;
1197                 rdev->config.rv770.sx_max_export_pos_size = 16;
1198                 rdev->config.rv770.sx_max_export_smx_size = 112;
1199                 rdev->config.rv770.sq_num_cf_insts = 2;
1200
1201                 rdev->config.rv770.sx_num_of_sets = 7;
1202                 rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1203                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1204                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1205                 break;
1206         case CHIP_RV730:
1207                 rdev->config.rv770.max_pipes = 2;
1208                 rdev->config.rv770.max_tile_pipes = 4;
1209                 rdev->config.rv770.max_simds = 8;
1210                 rdev->config.rv770.max_backends = 2;
1211                 rdev->config.rv770.max_gprs = 128;
1212                 rdev->config.rv770.max_threads = 248;
1213                 rdev->config.rv770.max_stack_entries = 256;
1214                 rdev->config.rv770.max_hw_contexts = 8;
1215                 rdev->config.rv770.max_gs_threads = 16 * 2;
1216                 rdev->config.rv770.sx_max_export_size = 256;
1217                 rdev->config.rv770.sx_max_export_pos_size = 32;
1218                 rdev->config.rv770.sx_max_export_smx_size = 224;
1219                 rdev->config.rv770.sq_num_cf_insts = 2;
1220
1221                 rdev->config.rv770.sx_num_of_sets = 7;
1222                 rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1223                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1224                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1225                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1226                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1227                         rdev->config.rv770.sx_max_export_smx_size += 16;
1228                 }
1229                 break;
1230         case CHIP_RV710:
1231                 rdev->config.rv770.max_pipes = 2;
1232                 rdev->config.rv770.max_tile_pipes = 2;
1233                 rdev->config.rv770.max_simds = 2;
1234                 rdev->config.rv770.max_backends = 1;
1235                 rdev->config.rv770.max_gprs = 256;
1236                 rdev->config.rv770.max_threads = 192;
1237                 rdev->config.rv770.max_stack_entries = 256;
1238                 rdev->config.rv770.max_hw_contexts = 4;
1239                 rdev->config.rv770.max_gs_threads = 8 * 2;
1240                 rdev->config.rv770.sx_max_export_size = 128;
1241                 rdev->config.rv770.sx_max_export_pos_size = 16;
1242                 rdev->config.rv770.sx_max_export_smx_size = 112;
1243                 rdev->config.rv770.sq_num_cf_insts = 1;
1244
1245                 rdev->config.rv770.sx_num_of_sets = 7;
1246                 rdev->config.rv770.sc_prim_fifo_size = 0x40;
1247                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1248                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1249                 break;
1250         case CHIP_RV740:
1251                 rdev->config.rv770.max_pipes = 4;
1252                 rdev->config.rv770.max_tile_pipes = 4;
1253                 rdev->config.rv770.max_simds = 8;
1254                 rdev->config.rv770.max_backends = 4;
1255                 rdev->config.rv770.max_gprs = 256;
1256                 rdev->config.rv770.max_threads = 248;
1257                 rdev->config.rv770.max_stack_entries = 512;
1258                 rdev->config.rv770.max_hw_contexts = 8;
1259                 rdev->config.rv770.max_gs_threads = 16 * 2;
1260                 rdev->config.rv770.sx_max_export_size = 256;
1261                 rdev->config.rv770.sx_max_export_pos_size = 32;
1262                 rdev->config.rv770.sx_max_export_smx_size = 224;
1263                 rdev->config.rv770.sq_num_cf_insts = 2;
1264
1265                 rdev->config.rv770.sx_num_of_sets = 7;
1266                 rdev->config.rv770.sc_prim_fifo_size = 0x100;
1267                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1268                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1269
1270                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1271                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1272                         rdev->config.rv770.sx_max_export_smx_size += 16;
1273                 }
1274                 break;
1275         default:
1276                 break;
1277         }
1278
1279         /* Initialize HDP */
1280         j = 0;
1281         for (i = 0; i < 32; i++) {
1282                 WREG32((0x2c14 + j), 0x00000000);
1283                 WREG32((0x2c18 + j), 0x00000000);
1284                 WREG32((0x2c1c + j), 0x00000000);
1285                 WREG32((0x2c20 + j), 0x00000000);
1286                 WREG32((0x2c24 + j), 0x00000000);
1287                 j += 0x18;
1288         }
1289
1290         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1291
1292         /* setup tiling, simd, pipe config */
1293         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1294
1295         shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1296         inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1297         for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1298                 if (!(inactive_pipes & tmp)) {
1299                         active_number++;
1300                 }
1301                 tmp <<= 1;
1302         }
1303         if (active_number == 1) {
1304                 WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1305         } else {
1306                 WREG32(SPI_CONFIG_CNTL, 0);
1307         }
1308
1309         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1310         tmp = rdev->config.rv770.max_simds -
1311                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1312         rdev->config.rv770.active_simds = tmp;
1313
1314         switch (rdev->config.rv770.max_tile_pipes) {
1315         case 1:
1316         default:
1317                 gb_tiling_config = PIPE_TILING(0);
1318                 break;
1319         case 2:
1320                 gb_tiling_config = PIPE_TILING(1);
1321                 break;
1322         case 4:
1323                 gb_tiling_config = PIPE_TILING(2);
1324                 break;
1325         case 8:
1326                 gb_tiling_config = PIPE_TILING(3);
1327                 break;
1328         }
1329         rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1330
1331         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1332         tmp = 0;
1333         for (i = 0; i < rdev->config.rv770.max_backends; i++)
1334                 tmp |= (1 << i);
1335         /* if all the backends are disabled, fix it up here */
1336         if ((disabled_rb_mask & tmp) == tmp) {
1337                 for (i = 0; i < rdev->config.rv770.max_backends; i++)
1338                         disabled_rb_mask &= ~(1 << i);
1339         }
1340         tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1341         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1342                                         R7XX_MAX_BACKENDS, disabled_rb_mask);
1343         gb_tiling_config |= tmp << 16;
1344         rdev->config.rv770.backend_map = tmp;
1345
1346         if (rdev->family == CHIP_RV770)
1347                 gb_tiling_config |= BANK_TILING(1);
1348         else {
1349                 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1350                         gb_tiling_config |= BANK_TILING(1);
1351                 else
1352                         gb_tiling_config |= BANK_TILING(0);
1353         }
1354         rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1355         gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1356         if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1357                 gb_tiling_config |= ROW_TILING(3);
1358                 gb_tiling_config |= SAMPLE_SPLIT(3);
1359         } else {
1360                 gb_tiling_config |=
1361                         ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1362                 gb_tiling_config |=
1363                         SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1364         }
1365
1366         gb_tiling_config |= BANK_SWAPS(1);
1367         rdev->config.rv770.tile_config = gb_tiling_config;
1368
1369         WREG32(GB_TILING_CONFIG, gb_tiling_config);
1370         WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1371         WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1372         WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1373         WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1374         if (rdev->family == CHIP_RV730) {
1375                 WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1376                 WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1377                 WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1378         }
1379
1380         WREG32(CGTS_SYS_TCC_DISABLE, 0);
1381         WREG32(CGTS_TCC_DISABLE, 0);
1382         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1383         WREG32(CGTS_USER_TCC_DISABLE, 0);
1384
1385
1386         num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1387         WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1388         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1389
1390         /* set HW defaults for 3D engine */
1391         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1392                                      ROQ_IB2_START(0x2b)));
1393
1394         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1395
1396         ta_aux_cntl = RREG32(TA_CNTL_AUX);
1397         WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1398
1399         sx_debug_1 = RREG32(SX_DEBUG_1);
1400         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1401         WREG32(SX_DEBUG_1, sx_debug_1);
1402
1403         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1404         smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1405         smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1406         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1407
1408         if (rdev->family != CHIP_RV740)
1409                 WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1410                                        GS_FLUSH_CTL(4) |
1411                                        ACK_FLUSH_CTL(3) |
1412                                        SYNC_FLUSH_CTL));
1413
1414         if (rdev->family != CHIP_RV770)
1415                 WREG32(SMX_SAR_CTL0, 0x00003f3f);
1416
1417         db_debug3 = RREG32(DB_DEBUG3);
1418         db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1419         switch (rdev->family) {
1420         case CHIP_RV770:
1421         case CHIP_RV740:
1422                 db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1423                 break;
1424         case CHIP_RV710:
1425         case CHIP_RV730:
1426         default:
1427                 db_debug3 |= DB_CLK_OFF_DELAY(2);
1428                 break;
1429         }
1430         WREG32(DB_DEBUG3, db_debug3);
1431
1432         if (rdev->family != CHIP_RV770) {
1433                 db_debug4 = RREG32(DB_DEBUG4);
1434                 db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1435                 WREG32(DB_DEBUG4, db_debug4);
1436         }
1437
1438         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1439                                         POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1440                                         SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1441
1442         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1443                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1444                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1445
1446         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1447
1448         WREG32(VGT_NUM_INSTANCES, 1);
1449
1450         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1451
1452         WREG32(CP_PERFMON_CNTL, 0);
1453
1454         sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1455                             DONE_FIFO_HIWATER(0xe0) |
1456                             ALU_UPDATE_FIFO_HIWATER(0x8));
1457         switch (rdev->family) {
1458         case CHIP_RV770:
1459         case CHIP_RV730:
1460         case CHIP_RV710:
1461                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1462                 break;
1463         case CHIP_RV740:
1464         default:
1465                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1466                 break;
1467         }
1468         WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1469
1470         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1471          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1472          */
1473         sq_config = RREG32(SQ_CONFIG);
1474         sq_config &= ~(PS_PRIO(3) |
1475                        VS_PRIO(3) |
1476                        GS_PRIO(3) |
1477                        ES_PRIO(3));
1478         sq_config |= (DX9_CONSTS |
1479                       VC_ENABLE |
1480                       EXPORT_SRC_C |
1481                       PS_PRIO(0) |
1482                       VS_PRIO(1) |
1483                       GS_PRIO(2) |
1484                       ES_PRIO(3));
1485         if (rdev->family == CHIP_RV710)
1486                 /* no vertex cache */
1487                 sq_config &= ~VC_ENABLE;
1488
1489         WREG32(SQ_CONFIG, sq_config);
1490
1491         WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1492                                          NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1493                                          NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1494
1495         WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1496                                          NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1497
1498         sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1499                                    NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1500                                    NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1501         if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1502                 sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1503         else
1504                 sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1505         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1506
1507         WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1508                                                      NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1509
1510         WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1511                                                      NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1512
1513         sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1514                                      SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1515                                      SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1516                                      SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1517
1518         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1519         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1520         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1521         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1522         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1523         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1524         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1525         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1526
1527         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1528                                           FORCE_EOV_MAX_REZ_CNT(255)));
1529
1530         if (rdev->family == CHIP_RV710)
1531                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1532                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1533         else
1534                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1535                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1536
1537         switch (rdev->family) {
1538         case CHIP_RV770:
1539         case CHIP_RV730:
1540         case CHIP_RV740:
1541                 gs_prim_buffer_depth = 384;
1542                 break;
1543         case CHIP_RV710:
1544                 gs_prim_buffer_depth = 128;
1545                 break;
1546         default:
1547                 break;
1548         }
1549
1550         num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1551         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1552         /* Max value for this is 256 */
1553         if (vgt_gs_per_es > 256)
1554                 vgt_gs_per_es = 256;
1555
1556         WREG32(VGT_ES_PER_GS, 128);
1557         WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1558         WREG32(VGT_GS_PER_VS, 2);
1559
1560         /* more default values. 2D/3D driver should adjust as needed */
1561         WREG32(VGT_GS_VERTEX_REUSE, 16);
1562         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1563         WREG32(VGT_STRMOUT_EN, 0);
1564         WREG32(SX_MISC, 0);
1565         WREG32(PA_SC_MODE_CNTL, 0);
1566         WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1567         WREG32(PA_SC_AA_CONFIG, 0);
1568         WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1569         WREG32(PA_SC_LINE_STIPPLE, 0);
1570         WREG32(SPI_INPUT_Z, 0);
1571         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1572         WREG32(CB_COLOR7_FRAG, 0);
1573
1574         /* clear render buffer base addresses */
1575         WREG32(CB_COLOR0_BASE, 0);
1576         WREG32(CB_COLOR1_BASE, 0);
1577         WREG32(CB_COLOR2_BASE, 0);
1578         WREG32(CB_COLOR3_BASE, 0);
1579         WREG32(CB_COLOR4_BASE, 0);
1580         WREG32(CB_COLOR5_BASE, 0);
1581         WREG32(CB_COLOR6_BASE, 0);
1582         WREG32(CB_COLOR7_BASE, 0);
1583
1584         WREG32(TCP_CNTL, 0);
1585
1586         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1587         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1588
1589         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1590
1591         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1592                                           NUM_CLIP_SEQ(3)));
1593         WREG32(VC_ENHANCE, 0);
1594 }
1595
1596 void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1597 {
1598         u64 size_bf, size_af;
1599
1600         if (mc->mc_vram_size > 0xE0000000) {
1601                 /* leave room for at least 512M GTT */
1602                 dev_warn(rdev->dev, "limiting VRAM\n");
1603                 mc->real_vram_size = 0xE0000000;
1604                 mc->mc_vram_size = 0xE0000000;
1605         }
1606         if (rdev->flags & RADEON_IS_AGP) {
1607                 size_bf = mc->gtt_start;
1608                 size_af = mc->mc_mask - mc->gtt_end;
1609                 if (size_bf > size_af) {
1610                         if (mc->mc_vram_size > size_bf) {
1611                                 dev_warn(rdev->dev, "limiting VRAM\n");
1612                                 mc->real_vram_size = size_bf;
1613                                 mc->mc_vram_size = size_bf;
1614                         }
1615                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1616                 } else {
1617                         if (mc->mc_vram_size > size_af) {
1618                                 dev_warn(rdev->dev, "limiting VRAM\n");
1619                                 mc->real_vram_size = size_af;
1620                                 mc->mc_vram_size = size_af;
1621                         }
1622                         mc->vram_start = mc->gtt_end + 1;
1623                 }
1624                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1625                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1626                                 mc->mc_vram_size >> 20, mc->vram_start,
1627                                 mc->vram_end, mc->real_vram_size >> 20);
1628         } else {
1629                 radeon_vram_location(rdev, &rdev->mc, 0);
1630                 rdev->mc.gtt_base_align = 0;
1631                 radeon_gtt_location(rdev, mc);
1632         }
1633 }
1634
1635 static int rv770_mc_init(struct radeon_device *rdev)
1636 {
1637         u32 tmp;
1638         int chansize, numchan;
1639
1640         /* Get VRAM informations */
1641         rdev->mc.vram_is_ddr = true;
1642         tmp = RREG32(MC_ARB_RAMCFG);
1643         if (tmp & CHANSIZE_OVERRIDE) {
1644                 chansize = 16;
1645         } else if (tmp & CHANSIZE_MASK) {
1646                 chansize = 64;
1647         } else {
1648                 chansize = 32;
1649         }
1650         tmp = RREG32(MC_SHARED_CHMAP);
1651         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1652         case 0:
1653         default:
1654                 numchan = 1;
1655                 break;
1656         case 1:
1657                 numchan = 2;
1658                 break;
1659         case 2:
1660                 numchan = 4;
1661                 break;
1662         case 3:
1663                 numchan = 8;
1664                 break;
1665         }
1666         rdev->mc.vram_width = numchan * chansize;
1667         /* Could aper size report 0 ? */
1668         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1669         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1670         /* Setup GPU memory space */
1671         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1672         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1673         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1674         r700_vram_gtt_location(rdev, &rdev->mc);
1675         radeon_update_bandwidth_info(rdev);
1676
1677         return 0;
1678 }
1679
1680 static int rv770_startup(struct radeon_device *rdev)
1681 {
1682         struct radeon_ring *ring;
1683         int r;
1684
1685         /* enable pcie gen2 link */
1686         rv770_pcie_gen2_enable(rdev);
1687
1688         /* scratch needs to be initialized before MC */
1689         r = r600_vram_scratch_init(rdev);
1690         if (r)
1691                 return r;
1692
1693         rv770_mc_program(rdev);
1694
1695         if (rdev->flags & RADEON_IS_AGP) {
1696                 rv770_agp_enable(rdev);
1697         } else {
1698                 r = rv770_pcie_gart_enable(rdev);
1699                 if (r)
1700                         return r;
1701         }
1702
1703         rv770_gpu_init(rdev);
1704
1705         /* allocate wb buffer */
1706         r = radeon_wb_init(rdev);
1707         if (r)
1708                 return r;
1709
1710         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1711         if (r) {
1712                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1713                 return r;
1714         }
1715
1716         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1717         if (r) {
1718                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1719                 return r;
1720         }
1721
1722         r = uvd_v2_2_resume(rdev);
1723         if (!r) {
1724                 r = radeon_fence_driver_start_ring(rdev,
1725                                                    R600_RING_TYPE_UVD_INDEX);
1726                 if (r)
1727                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
1728         }
1729
1730         if (r)
1731                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1732
1733         /* Enable IRQ */
1734         if (!rdev->irq.installed) {
1735                 r = radeon_irq_kms_init(rdev);
1736                 if (r)
1737                         return r;
1738         }
1739
1740         r = r600_irq_init(rdev);
1741         if (r) {
1742                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1743                 radeon_irq_kms_fini(rdev);
1744                 return r;
1745         }
1746         r600_irq_set(rdev);
1747
1748         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1749         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1750                              RADEON_CP_PACKET2);
1751         if (r)
1752                 return r;
1753
1754         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1755         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1756                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1757         if (r)
1758                 return r;
1759
1760         r = rv770_cp_load_microcode(rdev);
1761         if (r)
1762                 return r;
1763         r = r600_cp_resume(rdev);
1764         if (r)
1765                 return r;
1766
1767         r = r600_dma_resume(rdev);
1768         if (r)
1769                 return r;
1770
1771         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1772         if (ring->ring_size) {
1773                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1774                                      RADEON_CP_PACKET2);
1775                 if (!r)
1776                         r = uvd_v1_0_init(rdev);
1777
1778                 if (r)
1779                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
1780         }
1781
1782         r = radeon_ib_pool_init(rdev);
1783         if (r) {
1784                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1785                 return r;
1786         }
1787
1788         r = r600_audio_init(rdev);
1789         if (r) {
1790                 DRM_ERROR("radeon: audio init failed\n");
1791                 return r;
1792         }
1793
1794         return 0;
1795 }
1796
1797 int rv770_resume(struct radeon_device *rdev)
1798 {
1799         int r;
1800
1801         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1802          * posting will perform necessary task to bring back GPU into good
1803          * shape.
1804          */
1805         /* post card */
1806         atom_asic_init(rdev->mode_info.atom_context);
1807
1808         /* init golden registers */
1809         rv770_init_golden_registers(rdev);
1810
1811         if (rdev->pm.pm_method == PM_METHOD_DPM)
1812                 radeon_pm_resume(rdev);
1813
1814         rdev->accel_working = true;
1815         r = rv770_startup(rdev);
1816         if (r) {
1817                 DRM_ERROR("r600 startup failed on resume\n");
1818                 rdev->accel_working = false;
1819                 return r;
1820         }
1821
1822         return r;
1823
1824 }
1825
1826 int rv770_suspend(struct radeon_device *rdev)
1827 {
1828         radeon_pm_suspend(rdev);
1829         r600_audio_fini(rdev);
1830         uvd_v1_0_fini(rdev);
1831         radeon_uvd_suspend(rdev);
1832         r700_cp_stop(rdev);
1833         r600_dma_stop(rdev);
1834         r600_irq_suspend(rdev);
1835         radeon_wb_disable(rdev);
1836         rv770_pcie_gart_disable(rdev);
1837
1838         return 0;
1839 }
1840
1841 /* Plan is to move initialization in that function and use
1842  * helper function so that radeon_device_init pretty much
1843  * do nothing more than calling asic specific function. This
1844  * should also allow to remove a bunch of callback function
1845  * like vram_info.
1846  */
1847 int rv770_init(struct radeon_device *rdev)
1848 {
1849         int r;
1850
1851         /* Read BIOS */
1852         if (!radeon_get_bios(rdev)) {
1853                 if (ASIC_IS_AVIVO(rdev))
1854                         return -EINVAL;
1855         }
1856         /* Must be an ATOMBIOS */
1857         if (!rdev->is_atom_bios) {
1858                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1859                 return -EINVAL;
1860         }
1861         r = radeon_atombios_init(rdev);
1862         if (r)
1863                 return r;
1864         /* Post card if necessary */
1865         if (!radeon_card_posted(rdev)) {
1866                 if (!rdev->bios) {
1867                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1868                         return -EINVAL;
1869                 }
1870                 DRM_INFO("GPU not posted. posting now...\n");
1871                 atom_asic_init(rdev->mode_info.atom_context);
1872         }
1873         /* init golden registers */
1874         rv770_init_golden_registers(rdev);
1875         /* Initialize scratch registers */
1876         r600_scratch_init(rdev);
1877         /* Initialize surface registers */
1878         radeon_surface_init(rdev);
1879         /* Initialize clocks */
1880         radeon_get_clock_info(rdev->ddev);
1881         /* Fence driver */
1882         r = radeon_fence_driver_init(rdev);
1883         if (r)
1884                 return r;
1885         /* initialize AGP */
1886         if (rdev->flags & RADEON_IS_AGP) {
1887                 r = radeon_agp_init(rdev);
1888                 if (r)
1889                         radeon_agp_disable(rdev);
1890         }
1891         r = rv770_mc_init(rdev);
1892         if (r)
1893                 return r;
1894         /* Memory manager */
1895         r = radeon_bo_init(rdev);
1896         if (r)
1897                 return r;
1898
1899         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1900                 r = r600_init_microcode(rdev);
1901                 if (r) {
1902                         DRM_ERROR("Failed to load firmware!\n");
1903                         return r;
1904                 }
1905         }
1906
1907         /* Initialize power management */
1908         radeon_pm_init(rdev);
1909
1910         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1911         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1912
1913         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1914         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1915
1916         r = radeon_uvd_init(rdev);
1917         if (!r) {
1918                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1919                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
1920                                4096);
1921         }
1922
1923         rdev->ih.ring_obj = NULL;
1924         r600_ih_ring_init(rdev, 64 * 1024);
1925
1926         r = r600_pcie_gart_init(rdev);
1927         if (r)
1928                 return r;
1929
1930         rdev->accel_working = true;
1931         r = rv770_startup(rdev);
1932         if (r) {
1933                 dev_err(rdev->dev, "disabling GPU acceleration\n");
1934                 r700_cp_fini(rdev);
1935                 r600_dma_fini(rdev);
1936                 r600_irq_fini(rdev);
1937                 radeon_wb_fini(rdev);
1938                 radeon_ib_pool_fini(rdev);
1939                 radeon_irq_kms_fini(rdev);
1940                 rv770_pcie_gart_fini(rdev);
1941                 rdev->accel_working = false;
1942         }
1943
1944         return 0;
1945 }
1946
1947 void rv770_fini(struct radeon_device *rdev)
1948 {
1949         radeon_pm_fini(rdev);
1950         r700_cp_fini(rdev);
1951         r600_dma_fini(rdev);
1952         r600_irq_fini(rdev);
1953         radeon_wb_fini(rdev);
1954         radeon_ib_pool_fini(rdev);
1955         radeon_irq_kms_fini(rdev);
1956         uvd_v1_0_fini(rdev);
1957         radeon_uvd_fini(rdev);
1958         rv770_pcie_gart_fini(rdev);
1959         r600_vram_scratch_fini(rdev);
1960         radeon_gem_fini(rdev);
1961         radeon_fence_driver_fini(rdev);
1962         radeon_agp_fini(rdev);
1963         radeon_bo_fini(rdev);
1964         radeon_atombios_fini(rdev);
1965         r600_fini_microcode(rdev);
1966         kfree(rdev->bios);
1967         rdev->bios = NULL;
1968 }
1969
1970 static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
1971 {
1972         u32 link_width_cntl, lanes, speed_cntl, tmp;
1973         u16 link_cntl2;
1974         u32 mask;
1975         int ret;
1976
1977         if (radeon_pcie_gen2 == 0)
1978                 return;
1979
1980         if (rdev->flags & RADEON_IS_IGP)
1981                 return;
1982
1983         if (!(rdev->flags & RADEON_IS_PCIE))
1984                 return;
1985
1986         /* x2 cards have a special sequence */
1987         if (ASIC_IS_X2(rdev))
1988                 return;
1989
1990         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
1991         if (ret != 0)
1992                 return;
1993
1994         if (!(mask & DRM_PCIE_SPEED_50))
1995                 return;
1996
1997         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
1998
1999         /* advertise upconfig capability */
2000         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2001         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2002         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2003         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2004         if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2005                 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2006                 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2007                                      LC_RECONFIG_ARC_MISSING_ESCAPE);
2008                 link_width_cntl |= lanes | LC_RECONFIG_NOW |
2009                         LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2010                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2011         } else {
2012                 link_width_cntl |= LC_UPCONFIGURE_DIS;
2013                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2014         }
2015
2016         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2017         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2018             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2019
2020                 tmp = RREG32(0x541c);
2021                 WREG32(0x541c, tmp | 0x8);
2022                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2023                 link_cntl2 = RREG16(0x4088);
2024                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2025                 link_cntl2 |= 0x2;
2026                 WREG16(0x4088, link_cntl2);
2027                 WREG32(MM_CFGREGS_CNTL, 0);
2028
2029                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2030                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2031                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2032
2033                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2034                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2035                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2036
2037                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2038                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2039                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2040
2041                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2042                 speed_cntl |= LC_GEN2_EN_STRAP;
2043                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2044
2045         } else {
2046                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2047                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2048                 if (1)
2049                         link_width_cntl |= LC_UPCONFIGURE_DIS;
2050                 else
2051                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2052                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2053         }
2054 }