sys/dev/drm/radeon/rv770_dpm.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <drm/drmP.h>
  26 #include "radeon.h"
  27 #include "radeon_asic.h"
  28 #include "rv770d.h"
  29 #include "r600_dpm.h"
  30 #include "rv770_dpm.h"
  31 #include "cypress_dpm.h"
  32 #include "atom.h"
  33 #include <linux/seq_file.h>
  34
  35 #define MC_CG_ARB_FREQ_F0           0x0a
  36 #define MC_CG_ARB_FREQ_F1           0x0b
  37 #define MC_CG_ARB_FREQ_F2           0x0c
  38 #define MC_CG_ARB_FREQ_F3           0x0d
  39
  40 #define MC_CG_SEQ_DRAMCONF_S0       0x05
  41 #define MC_CG_SEQ_DRAMCONF_S1       0x06
  42
  43 #define PCIE_BUS_CLK                10000
  44 #define TCLK                        (PCIE_BUS_CLK / 10)
  45
  46 #define SMC_RAM_END 0xC000
  47
  48 struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps);
  49 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
  50 void rv770_dpm_reset_asic(struct radeon_device *rdev);
  51 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
  52
  53 struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps)
  54 {
  55         struct rv7xx_ps *ps = rps->ps_priv;
  56
  57         return ps;
  58 }
  59
  60 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev)
  61 {
  62         struct rv7xx_power_info *pi = rdev->pm.dpm.priv;
  63
  64         return pi;
  65 }
  66
  67 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev)
  68 {
  69         struct evergreen_power_info *pi = rdev->pm.dpm.priv;
  70
  71         return pi;
  72 }
  73
  74 static void rv770_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
  75                                                bool enable)
  76 {
  77         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
  78         u32 tmp;
  79
  80         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
  81         if (enable) {
  82                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
  83                 tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
  84                 tmp |= LC_GEN2_EN_STRAP;
  85         } else {
  86                 if (!pi->boot_in_gen2) {
  87                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
  88                         tmp &= ~LC_GEN2_EN_STRAP;
  89                 }
  90         }
  91         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
  92             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
  93                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
  94
  95 }
  96
  97 static void rv770_enable_l0s(struct radeon_device *rdev)
  98 {
  99         u32 tmp;
 100
 101         tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
 102         tmp |= LC_L0S_INACTIVITY(3);
 103         WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
 104 }
 105
 106 static void rv770_enable_l1(struct radeon_device *rdev)
 107 {
 108         u32 tmp;
 109
 110         tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
 111         tmp &= ~LC_L1_INACTIVITY_MASK;
 112         tmp |= LC_L1_INACTIVITY(4);
 113         tmp &= ~LC_PMI_TO_L1_DIS;
 114         tmp &= ~LC_ASPM_TO_L1_DIS;
 115         WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
 116 }
 117
 118 static void rv770_enable_pll_sleep_in_l1(struct radeon_device *rdev)
 119 {
 120         u32 tmp;
 121
 122         tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
 123         tmp |= LC_L1_INACTIVITY(8);
 124         WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
 125
 126         /* NOTE, this is a PCIE indirect reg, not PCIE PORT */
 127         tmp = RREG32_PCIE(PCIE_P_CNTL);
 128         tmp |= P_PLL_PWRDN_IN_L1L23;
 129         tmp &= ~P_PLL_BUF_PDNB;
 130         tmp &= ~P_PLL_PDNB;
 131         tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
 132         WREG32_PCIE(PCIE_P_CNTL, tmp);
 133 }
 134
 135 static void rv770_gfx_clock_gating_enable(struct radeon_device *rdev,
 136                                           bool enable)
 137 {
 138         if (enable)
 139                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 140         else {
 141                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
 142                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
 143                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
 144                 RREG32(GB_TILING_CONFIG);
 145         }
 146 }
 147
 148 static void rv770_mg_clock_gating_enable(struct radeon_device *rdev,
 149                                          bool enable)
 150 {
 151         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 152
 153         if (enable) {
 154                 u32 mgcg_cgtt_local0;
 155
 156                 if (rdev->family == CHIP_RV770)
 157                         mgcg_cgtt_local0 = RV770_MGCGTTLOCAL0_DFLT;
 158                 else
 159                         mgcg_cgtt_local0 = RV7XX_MGCGTTLOCAL0_DFLT;
 160
 161                 WREG32(CG_CGTT_LOCAL_0, mgcg_cgtt_local0);
 162                 WREG32(CG_CGTT_LOCAL_1, (RV770_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF));
 163
 164                 if (pi->mgcgtssm)
 165                         WREG32(CGTS_SM_CTRL_REG, RV770_MGCGCGTSSMCTRL_DFLT);
 166         } else {
 167                 WREG32(CG_CGTT_LOCAL_0, 0xFFFFFFFF);
 168                 WREG32(CG_CGTT_LOCAL_1, 0xFFFFCFFF);
 169         }
 170 }
 171
 172 void rv770_restore_cgcg(struct radeon_device *rdev)
 173 {
 174         bool dpm_en = false, cg_en = false;
 175
 176         if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
 177                 dpm_en = true;
 178         if (RREG32(SCLK_PWRMGT_CNTL) & DYN_GFX_CLK_OFF_EN)
 179                 cg_en = true;
 180
 181         if (dpm_en && !cg_en)
 182                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 183 }
 184
 185 static void rv770_start_dpm(struct radeon_device *rdev)
 186 {
 187         WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
 188
 189         WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
 190
 191         WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
 192 }
 193
 194 void rv770_stop_dpm(struct radeon_device *rdev)
 195 {
 196         PPSMC_Result result;
 197
 198         result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled);
 199
 200         if (result != PPSMC_Result_OK)
 201                 DRM_ERROR("Could not force DPM to low.\n");
 202
 203         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
 204
 205         WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
 206
 207         WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
 208 }
 209
 210 bool rv770_dpm_enabled(struct radeon_device *rdev)
 211 {
 212         if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
 213                 return true;
 214         else
 215                 return false;
 216 }
 217
 218 void rv770_enable_thermal_protection(struct radeon_device *rdev,
 219                                      bool enable)
 220 {
 221         if (enable)
 222                 WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
 223         else
 224                 WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
 225 }
 226
 227 void rv770_enable_acpi_pm(struct radeon_device *rdev)
 228 {
 229         WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
 230 }
 231
 232 u8 rv770_get_seq_value(struct radeon_device *rdev,
 233                        struct rv7xx_pl *pl)
 234 {
 235         return (pl->flags & ATOM_PPLIB_R600_FLAGS_LOWPOWER) ?
 236                 MC_CG_SEQ_DRAMCONF_S0 : MC_CG_SEQ_DRAMCONF_S1;
 237 }
 238
 239 int rv770_read_smc_soft_register(struct radeon_device *rdev,
 240                                  u16 reg_offset, u32 *value)
 241 {
 242         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 243
 244         return rv770_read_smc_sram_dword(rdev,
 245                                          pi->soft_regs_start + reg_offset,
 246                                          value, pi->sram_end);
 247 }
 248
 249 int rv770_write_smc_soft_register(struct radeon_device *rdev,
 250                                   u16 reg_offset, u32 value)
 251 {
 252         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 253
 254         return rv770_write_smc_sram_dword(rdev,
 255                                           pi->soft_regs_start + reg_offset,
 256                                           value, pi->sram_end);
 257 }
 258
 259 int rv770_populate_smc_t(struct radeon_device *rdev,
 260                          struct radeon_ps *radeon_state,
 261                          RV770_SMC_SWSTATE *smc_state)
 262 {
 263         struct rv7xx_ps *state = rv770_get_ps(radeon_state);
 264         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 265         int i;
 266         int a_n;
 267         int a_d;
 268         u8 l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
 269         u8 r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
 270         u32 a_t;
 271
 272         l[0] = 0;
 273         r[2] = 100;
 274
 275         a_n = (int)state->medium.sclk * pi->lmp +
 276                 (int)state->low.sclk * (R600_AH_DFLT - pi->rlp);
 277         a_d = (int)state->low.sclk * (100 - (int)pi->rlp) +
 278                 (int)state->medium.sclk * pi->lmp;
 279
 280         l[1] = (u8)(pi->lmp - (int)pi->lmp * a_n / a_d);
 281         r[0] = (u8)(pi->rlp + (100 - (int)pi->rlp) * a_n / a_d);
 282
 283         a_n = (int)state->high.sclk * pi->lhp + (int)state->medium.sclk *
 284                 (R600_AH_DFLT - pi->rmp);
 285         a_d = (int)state->medium.sclk * (100 - (int)pi->rmp) +
 286                 (int)state->high.sclk * pi->lhp;
 287
 288         l[2] = (u8)(pi->lhp - (int)pi->lhp * a_n / a_d);
 289         r[1] = (u8)(pi->rmp + (100 - (int)pi->rmp) * a_n / a_d);
 290
 291         for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++) {
 292                 a_t = CG_R(r[i] * pi->bsp / 200) | CG_L(l[i] * pi->bsp / 200);
 293                 smc_state->levels[i].aT = cpu_to_be32(a_t);
 294         }
 295
 296         a_t = CG_R(r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200) |
 297                 CG_L(l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200);
 298
 299         smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].aT =
 300                 cpu_to_be32(a_t);
 301
 302         return 0;
 303 }
 304
 305 int rv770_populate_smc_sp(struct radeon_device *rdev,
 306                           struct radeon_ps *radeon_state,
 307                           RV770_SMC_SWSTATE *smc_state)
 308 {
 309         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 310         int i;
 311
 312         for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++)
 313                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
 314
 315         smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].bSP =
 316                 cpu_to_be32(pi->psp);
 317
 318         return 0;
 319 }
 320
 321 static void rv770_calculate_fractional_mpll_feedback_divider(u32 memory_clock,
 322                                                              u32 reference_clock,
 323                                                              bool gddr5,
 324                                                              struct atom_clock_dividers *dividers,
 325                                                              u32 *clkf,
 326                                                              u32 *clkfrac)
 327 {
 328         u32 post_divider, reference_divider, feedback_divider8;
 329         u32 fyclk;
 330
 331         if (gddr5)
 332                 fyclk = (memory_clock * 8) / 2;
 333         else
 334                 fyclk = (memory_clock * 4) / 2;
 335
 336         post_divider = dividers->post_div;
 337         reference_divider = dividers->ref_div;
 338
 339         feedback_divider8 =
 340                 (8 * fyclk * reference_divider * post_divider) / reference_clock;
 341
 342         *clkf = feedback_divider8 / 8;
 343         *clkfrac = feedback_divider8 % 8;
 344 }
 345
 346 static int rv770_encode_yclk_post_div(u32 postdiv, u32 *encoded_postdiv)
 347 {
 348         int ret = 0;
 349
 350         switch (postdiv) {
 351         case 1:
 352                 *encoded_postdiv = 0;
 353                 break;
 354         case 2:
 355                 *encoded_postdiv = 1;
 356                 break;
 357         case 4:
 358                 *encoded_postdiv = 2;
 359                 break;
 360         case 8:
 361                 *encoded_postdiv = 3;
 362                 break;
 363         case 16:
 364                 *encoded_postdiv = 4;
 365                 break;
 366         default:
 367                 ret = -EINVAL;
 368                 break;
 369         }
 370
 371     return ret;
 372 }
 373
 374 u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf)
 375 {
 376         if (clkf <= 0x10)
 377                 return 0x4B;
 378         if (clkf <= 0x19)
 379                 return 0x5B;
 380         if (clkf <= 0x21)
 381                 return 0x2B;
 382         if (clkf <= 0x27)
 383                 return 0x6C;
 384         if (clkf <= 0x31)
 385                 return 0x9D;
 386         return 0xC6;
 387 }
 388
 389 static int rv770_populate_mclk_value(struct radeon_device *rdev,
 390                                      u32 engine_clock, u32 memory_clock,
 391                                      RV7XX_SMC_MCLK_VALUE *mclk)
 392 {
 393         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 394         u8 encoded_reference_dividers[] = { 0, 16, 17, 20, 21 };
 395         u32 mpll_ad_func_cntl =
 396                 pi->clk_regs.rv770.mpll_ad_func_cntl;
 397         u32 mpll_ad_func_cntl_2 =
 398                 pi->clk_regs.rv770.mpll_ad_func_cntl_2;
 399         u32 mpll_dq_func_cntl =
 400                 pi->clk_regs.rv770.mpll_dq_func_cntl;
 401         u32 mpll_dq_func_cntl_2 =
 402                 pi->clk_regs.rv770.mpll_dq_func_cntl_2;
 403         u32 mclk_pwrmgt_cntl =
 404                 pi->clk_regs.rv770.mclk_pwrmgt_cntl;
 405         u32 dll_cntl = pi->clk_regs.rv770.dll_cntl;
 406         struct atom_clock_dividers dividers;
 407         u32 reference_clock = rdev->clock.mpll.reference_freq;
 408         u32 clkf, clkfrac;
 409         u32 postdiv_yclk;
 410         u32 ibias;
 411         int ret;
 412
 413         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
 414                                              memory_clock, false, &dividers);
 415         if (ret)
 416                 return ret;
 417
 418         if ((dividers.ref_div < 1) || (dividers.ref_div > 5))
 419                 return -EINVAL;
 420
 421         rv770_calculate_fractional_mpll_feedback_divider(memory_clock, reference_clock,
 422                                                          pi->mem_gddr5,
 423                                                          &dividers, &clkf, &clkfrac);
 424
 425         ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
 426         if (ret)
 427                 return ret;
 428
 429         ibias = rv770_map_clkf_to_ibias(rdev, clkf);
 430
 431         mpll_ad_func_cntl &= ~(CLKR_MASK |
 432                                YCLK_POST_DIV_MASK |
 433                                CLKF_MASK |
 434                                CLKFRAC_MASK |
 435                                IBIAS_MASK);
 436         mpll_ad_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
 437         mpll_ad_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
 438         mpll_ad_func_cntl |= CLKF(clkf);
 439         mpll_ad_func_cntl |= CLKFRAC(clkfrac);
 440         mpll_ad_func_cntl |= IBIAS(ibias);
 441
 442         if (dividers.vco_mode)
 443                 mpll_ad_func_cntl_2 |= VCO_MODE;
 444         else
 445                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
 446
 447         if (pi->mem_gddr5) {
 448                 rv770_calculate_fractional_mpll_feedback_divider(memory_clock,
 449                                                                  reference_clock,
 450                                                                  pi->mem_gddr5,
 451                                                                  &dividers, &clkf, &clkfrac);
 452
 453                 ibias = rv770_map_clkf_to_ibias(rdev, clkf);
 454
 455                 ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
 456                 if (ret)
 457                         return ret;
 458
 459                 mpll_dq_func_cntl &= ~(CLKR_MASK |
 460                                        YCLK_POST_DIV_MASK |
 461                                        CLKF_MASK |
 462                                        CLKFRAC_MASK |
 463                                        IBIAS_MASK);
 464                 mpll_dq_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
 465                 mpll_dq_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
 466                 mpll_dq_func_cntl |= CLKF(clkf);
 467                 mpll_dq_func_cntl |= CLKFRAC(clkfrac);
 468                 mpll_dq_func_cntl |= IBIAS(ibias);
 469
 470                 if (dividers.vco_mode)
 471                         mpll_dq_func_cntl_2 |= VCO_MODE;
 472                 else
 473                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
 474         }
 475
 476         mclk->mclk770.mclk_value = cpu_to_be32(memory_clock);
 477         mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
 478         mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
 479         mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
 480         mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
 481         mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
 482         mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
 483
 484         return 0;
 485 }
 486
 487 static int rv770_populate_sclk_value(struct radeon_device *rdev,
 488                                      u32 engine_clock,
 489                                      RV770_SMC_SCLK_VALUE *sclk)
 490 {
 491         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 492         struct atom_clock_dividers dividers;
 493         u32 spll_func_cntl =
 494                 pi->clk_regs.rv770.cg_spll_func_cntl;
 495         u32 spll_func_cntl_2 =
 496                 pi->clk_regs.rv770.cg_spll_func_cntl_2;
 497         u32 spll_func_cntl_3 =
 498                 pi->clk_regs.rv770.cg_spll_func_cntl_3;
 499         u32 cg_spll_spread_spectrum =
 500                 pi->clk_regs.rv770.cg_spll_spread_spectrum;
 501         u32 cg_spll_spread_spectrum_2 =
 502                 pi->clk_regs.rv770.cg_spll_spread_spectrum_2;
 503         u64 tmp;
 504         u32 reference_clock = rdev->clock.spll.reference_freq;
 505         u32 reference_divider, post_divider;
 506         u32 fbdiv;
 507         int ret;
 508
 509         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 510                                              engine_clock, false, &dividers);
 511         if (ret)
 512                 return ret;
 513
 514         reference_divider = 1 + dividers.ref_div;
 515
 516         if (dividers.enable_post_div)
 517                 post_divider = (0x0f & (dividers.post_div >> 4)) + (0x0f & dividers.post_div) + 2;
 518         else
 519                 post_divider = 1;
 520
 521         tmp = (u64) engine_clock * reference_divider * post_divider * 16384;
 522         do_div(tmp, reference_clock);
 523         fbdiv = (u32) tmp;
 524
 525         if (dividers.enable_post_div)
 526                 spll_func_cntl |= SPLL_DIVEN;
 527         else
 528                 spll_func_cntl &= ~SPLL_DIVEN;
 529         spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK);
 530         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
 531         spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf);
 532         spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf);
 533
 534         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
 535         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
 536
 537         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
 538         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
 539         spll_func_cntl_3 |= SPLL_DITHEN;
 540
 541         if (pi->sclk_ss) {
 542                 struct radeon_atom_ss ss;
 543                 u32 vco_freq = engine_clock * post_divider;
 544
 545                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
 546                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
 547                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
 548                         u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000);
 549
 550                         cg_spll_spread_spectrum &= ~CLKS_MASK;
 551                         cg_spll_spread_spectrum |= CLKS(clk_s);
 552                         cg_spll_spread_spectrum |= SSEN;
 553
 554                         cg_spll_spread_spectrum_2 &= ~CLKV_MASK;
 555                         cg_spll_spread_spectrum_2 |= CLKV(clk_v);
 556                 }
 557         }
 558
 559         sclk->sclk_value = cpu_to_be32(engine_clock);
 560         sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
 561         sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
 562         sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
 563         sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum);
 564         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2);
 565
 566         return 0;
 567 }
 568
 569 int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc,
 570                               RV770_SMC_VOLTAGE_VALUE *voltage)
 571 {
 572         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 573         int i;
 574
 575         if (!pi->voltage_control) {
 576                 voltage->index = 0;
 577                 voltage->value = 0;
 578                 return 0;
 579         }
 580
 581         for (i = 0; i < pi->valid_vddc_entries; i++) {
 582                 if (vddc <= pi->vddc_table[i].vddc) {
 583                         voltage->index = pi->vddc_table[i].vddc_index;
 584                         voltage->value = cpu_to_be16(vddc);
 585                         break;
 586                 }
 587         }
 588
 589         if (i == pi->valid_vddc_entries)
 590                 return -EINVAL;
 591
 592         return 0;
 593 }
 594
 595 int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
 596                               RV770_SMC_VOLTAGE_VALUE *voltage)
 597 {
 598         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 599
 600         if (!pi->mvdd_control) {
 601                 voltage->index = MVDD_HIGH_INDEX;
 602                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
 603                 return 0;
 604         }
 605
 606         if (mclk <= pi->mvdd_split_frequency) {
 607                 voltage->index = MVDD_LOW_INDEX;
 608                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
 609         } else {
 610                 voltage->index = MVDD_HIGH_INDEX;
 611                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
 612         }
 613
 614         return 0;
 615 }
 616
 617 static int rv770_convert_power_level_to_smc(struct radeon_device *rdev,
 618                                             struct rv7xx_pl *pl,
 619                                             RV770_SMC_HW_PERFORMANCE_LEVEL *level,
 620                                             u8 watermark_level)
 621 {
 622         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 623         int ret;
 624
 625         level->gen2PCIE = pi->pcie_gen2 ?
 626                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
 627         level->gen2XSP  = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0;
 628         level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0;
 629         level->displayWatermark = watermark_level;
 630
 631         if (rdev->family == CHIP_RV740)
 632                 ret = rv740_populate_sclk_value(rdev, pl->sclk,
 633                                                 &level->sclk);
 634         else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
 635                 ret = rv730_populate_sclk_value(rdev, pl->sclk,
 636                                                 &level->sclk);
 637         else
 638                 ret = rv770_populate_sclk_value(rdev, pl->sclk,
 639                                                 &level->sclk);
 640         if (ret)
 641                 return ret;
 642
 643         if (rdev->family == CHIP_RV740) {
 644                 if (pi->mem_gddr5) {
 645                         if (pl->mclk <= pi->mclk_strobe_mode_threshold)
 646                                 level->strobeMode =
 647                                         rv740_get_mclk_frequency_ratio(pl->mclk) | 0x10;
 648                         else
 649                                 level->strobeMode = 0;
 650
 651                         if (pl->mclk > pi->mclk_edc_enable_threshold)
 652                                 level->mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
 653                         else
 654                                 level->mcFlags =  0;
 655                 }
 656                 ret = rv740_populate_mclk_value(rdev, pl->sclk,
 657                                                 pl->mclk, &level->mclk);
 658         } else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
 659                 ret = rv730_populate_mclk_value(rdev, pl->sclk,
 660                                                 pl->mclk, &level->mclk);
 661         else
 662                 ret = rv770_populate_mclk_value(rdev, pl->sclk,
 663                                                 pl->mclk, &level->mclk);
 664         if (ret)
 665                 return ret;
 666
 667         ret = rv770_populate_vddc_value(rdev, pl->vddc,
 668                                         &level->vddc);
 669         if (ret)
 670                 return ret;
 671
 672         ret = rv770_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
 673
 674         return ret;
 675 }
 676
 677 static int rv770_convert_power_state_to_smc(struct radeon_device *rdev,
 678                                             struct radeon_ps *radeon_state,
 679                                             RV770_SMC_SWSTATE *smc_state)
 680 {
 681         struct rv7xx_ps *state = rv770_get_ps(radeon_state);
 682         int ret;
 683
 684         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
 685                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
 686
 687         ret = rv770_convert_power_level_to_smc(rdev,
 688                                                &state->low,
 689                                                &smc_state->levels[0],
 690                                                PPSMC_DISPLAY_WATERMARK_LOW);
 691         if (ret)
 692                 return ret;
 693
 694         ret = rv770_convert_power_level_to_smc(rdev,
 695                                                &state->medium,
 696                                                &smc_state->levels[1],
 697                                                PPSMC_DISPLAY_WATERMARK_LOW);
 698         if (ret)
 699                 return ret;
 700
 701         ret = rv770_convert_power_level_to_smc(rdev,
 702                                                &state->high,
 703                                                &smc_state->levels[2],
 704                                                PPSMC_DISPLAY_WATERMARK_HIGH);
 705         if (ret)
 706                 return ret;
 707
 708         smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1;
 709         smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2;
 710         smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3;
 711
 712         smc_state->levels[0].seqValue = rv770_get_seq_value(rdev,
 713                                                             &state->low);
 714         smc_state->levels[1].seqValue = rv770_get_seq_value(rdev,
 715                                                             &state->medium);
 716         smc_state->levels[2].seqValue = rv770_get_seq_value(rdev,
 717                                                             &state->high);
 718
 719         rv770_populate_smc_sp(rdev, radeon_state, smc_state);
 720
 721         return rv770_populate_smc_t(rdev, radeon_state, smc_state);
 722
 723 }
 724
 725 u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev,
 726                                         u32 engine_clock)
 727 {
 728         u32 dram_rows;
 729         u32 dram_refresh_rate;
 730         u32 mc_arb_rfsh_rate;
 731         u32 tmp;
 732
 733         tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
 734         dram_rows = 1 << (tmp + 10);
 735         tmp = RREG32(MC_SEQ_MISC0) & 3;
 736         dram_refresh_rate = 1 << (tmp + 3);
 737         mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
 738
 739         return mc_arb_rfsh_rate;
 740 }
 741
 742 static void rv770_program_memory_timing_parameters(struct radeon_device *rdev,
 743                                                    struct radeon_ps *radeon_state)
 744 {
 745         struct rv7xx_ps *state = rv770_get_ps(radeon_state);
 746         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 747         u32 sqm_ratio;
 748         u32 arb_refresh_rate;
 749         u32 high_clock;
 750
 751         if (state->high.sclk < (state->low.sclk * 0xFF / 0x40))
 752                 high_clock = state->high.sclk;
 753         else
 754                 high_clock = (state->low.sclk * 0xFF / 0x40);
 755
 756         radeon_atom_set_engine_dram_timings(rdev, high_clock,
 757                                             state->high.mclk);
 758
 759         sqm_ratio =
 760                 STATE0(64 * high_clock / pi->boot_sclk) |
 761                 STATE1(64 * high_clock / state->low.sclk) |
 762                 STATE2(64 * high_clock / state->medium.sclk) |
 763                 STATE3(64 * high_clock / state->high.sclk);
 764         WREG32(MC_ARB_SQM_RATIO, sqm_ratio);
 765
 766         arb_refresh_rate =
 767                 POWERMODE0(rv770_calculate_memory_refresh_rate(rdev, pi->boot_sclk)) |
 768                 POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) |
 769                 POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) |
 770                 POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk));
 771         WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate);
 772 }
 773
 774 void rv770_enable_backbias(struct radeon_device *rdev,
 775                            bool enable)
 776 {
 777         if (enable)
 778                 WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN, ~BACKBIAS_PAD_EN);
 779         else
 780                 WREG32_P(GENERAL_PWRMGT, 0, ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN));
 781 }
 782
 783 static void rv770_enable_spread_spectrum(struct radeon_device *rdev,
 784                                          bool enable)
 785 {
 786         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 787
 788         if (enable) {
 789                 if (pi->sclk_ss)
 790                         WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
 791
 792                 if (pi->mclk_ss) {
 793                         if (rdev->family == CHIP_RV740)
 794                                 rv740_enable_mclk_spread_spectrum(rdev, true);
 795                 }
 796         } else {
 797                 WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
 798
 799                 WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
 800
 801                 WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
 802
 803                 if (rdev->family == CHIP_RV740)
 804                         rv740_enable_mclk_spread_spectrum(rdev, false);
 805         }
 806 }
 807
 808 static void rv770_program_mpll_timing_parameters(struct radeon_device *rdev)
 809 {
 810         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 811
 812         if ((rdev->family == CHIP_RV770) && !pi->mem_gddr5) {
 813                 WREG32(MPLL_TIME,
 814                        (MPLL_LOCK_TIME(R600_MPLLLOCKTIME_DFLT * pi->ref_div) |
 815                         MPLL_RESET_TIME(R600_MPLLRESETTIME_DFLT)));
 816         }
 817 }
 818
 819 void rv770_setup_bsp(struct radeon_device *rdev)
 820 {
 821         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 822         u32 xclk = radeon_get_xclk(rdev);
 823
 824         r600_calculate_u_and_p(pi->asi,
 825                                xclk,
 826                                16,
 827                                &pi->bsp,
 828                                &pi->bsu);
 829
 830         r600_calculate_u_and_p(pi->pasi,
 831                                xclk,
 832                                16,
 833                                &pi->pbsp,
 834                                &pi->pbsu);
 835
 836         pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
 837         pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
 838
 839         WREG32(CG_BSP, pi->dsp);
 840
 841 }
 842
 843 void rv770_program_git(struct radeon_device *rdev)
 844 {
 845         WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK);
 846 }
 847
 848 void rv770_program_tp(struct radeon_device *rdev)
 849 {
 850         int i;
 851         enum r600_td td = R600_TD_DFLT;
 852
 853         for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
 854                 WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i])));
 855
 856         if (td == R600_TD_AUTO)
 857                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
 858         else
 859                 WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
 860         if (td == R600_TD_UP)
 861                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
 862         if (td == R600_TD_DOWN)
 863                 WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
 864 }
 865
 866 void rv770_program_tpp(struct radeon_device *rdev)
 867 {
 868         WREG32(CG_TPC, R600_TPC_DFLT);
 869 }
 870
 871 void rv770_program_sstp(struct radeon_device *rdev)
 872 {
 873         WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
 874 }
 875
 876 void rv770_program_engine_speed_parameters(struct radeon_device *rdev)
 877 {
 878         WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
 879 }
 880
 881 static void rv770_enable_display_gap(struct radeon_device *rdev)
 882 {
 883         u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
 884
 885         tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
 886         tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
 887                 DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
 888         WREG32(CG_DISPLAY_GAP_CNTL, tmp);
 889 }
 890
 891 void rv770_program_vc(struct radeon_device *rdev)
 892 {
 893         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 894
 895         WREG32(CG_FTV, pi->vrc);
 896 }
 897
 898 void rv770_clear_vc(struct radeon_device *rdev)
 899 {
 900         WREG32(CG_FTV, 0);
 901 }
 902
 903 int rv770_upload_firmware(struct radeon_device *rdev)
 904 {
 905         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 906         int ret;
 907
 908         rv770_reset_smc(rdev);
 909         rv770_stop_smc_clock(rdev);
 910
 911         ret = rv770_load_smc_ucode(rdev, pi->sram_end);
 912         if (ret)
 913                 return ret;
 914
 915         return 0;
 916 }
 917
 918 static int rv770_populate_smc_acpi_state(struct radeon_device *rdev,
 919                                          RV770_SMC_STATETABLE *table)
 920 {
 921         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 922
 923         u32 mpll_ad_func_cntl =
 924                 pi->clk_regs.rv770.mpll_ad_func_cntl;
 925         u32 mpll_ad_func_cntl_2 =
 926                 pi->clk_regs.rv770.mpll_ad_func_cntl_2;
 927         u32 mpll_dq_func_cntl =
 928                 pi->clk_regs.rv770.mpll_dq_func_cntl;
 929         u32 mpll_dq_func_cntl_2 =
 930                 pi->clk_regs.rv770.mpll_dq_func_cntl_2;
 931         u32 spll_func_cntl =
 932                 pi->clk_regs.rv770.cg_spll_func_cntl;
 933         u32 spll_func_cntl_2 =
 934                 pi->clk_regs.rv770.cg_spll_func_cntl_2;
 935         u32 spll_func_cntl_3 =
 936                 pi->clk_regs.rv770.cg_spll_func_cntl_3;
 937         u32 mclk_pwrmgt_cntl;
 938         u32 dll_cntl;
 939
 940         table->ACPIState = table->initialState;
 941
 942         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
 943
 944         if (pi->acpi_vddc) {
 945                 rv770_populate_vddc_value(rdev, pi->acpi_vddc,
 946                                           &table->ACPIState.levels[0].vddc);
 947                 if (pi->pcie_gen2) {
 948                         if (pi->acpi_pcie_gen2)
 949                                 table->ACPIState.levels[0].gen2PCIE = 1;
 950                         else
 951                                 table->ACPIState.levels[0].gen2PCIE = 0;
 952                 } else
 953                         table->ACPIState.levels[0].gen2PCIE = 0;
 954                 if (pi->acpi_pcie_gen2)
 955                         table->ACPIState.levels[0].gen2XSP = 1;
 956                 else
 957                         table->ACPIState.levels[0].gen2XSP = 0;
 958         } else {
 959                 rv770_populate_vddc_value(rdev, pi->min_vddc_in_table,
 960                                           &table->ACPIState.levels[0].vddc);
 961                 table->ACPIState.levels[0].gen2PCIE = 0;
 962         }
 963
 964
 965         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
 966
 967         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
 968
 969         mclk_pwrmgt_cntl = (MRDCKA0_RESET |
 970                             MRDCKA1_RESET |
 971                             MRDCKB0_RESET |
 972                             MRDCKB1_RESET |
 973                             MRDCKC0_RESET |
 974                             MRDCKC1_RESET |
 975                             MRDCKD0_RESET |
 976                             MRDCKD1_RESET);
 977
 978         dll_cntl = 0xff000000;
 979
 980         spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN;
 981
 982         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
 983         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
 984
 985         table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
 986         table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
 987         table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
 988         table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
 989
 990         table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
 991         table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
 992
 993         table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0;
 994
 995         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
 996         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
 997         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
 998
 999         table->ACPIState.levels[0].sclk.sclk_value = 0;
1000
1001         rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1002
1003         table->ACPIState.levels[1] = table->ACPIState.levels[0];
1004         table->ACPIState.levels[2] = table->ACPIState.levels[0];
1005
1006         return 0;
1007 }
1008
1009 int rv770_populate_initial_mvdd_value(struct radeon_device *rdev,
1010                                       RV770_SMC_VOLTAGE_VALUE *voltage)
1011 {
1012         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1013
1014         if ((pi->s0_vid_lower_smio_cntl & pi->mvdd_mask_low) ==
1015              (pi->mvdd_low_smio[MVDD_LOW_INDEX] & pi->mvdd_mask_low) ) {
1016                 voltage->index = MVDD_LOW_INDEX;
1017                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1018         } else {
1019                 voltage->index = MVDD_HIGH_INDEX;
1020                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1021         }
1022
1023         return 0;
1024 }
1025
1026 static int rv770_populate_smc_initial_state(struct radeon_device *rdev,
1027                                             struct radeon_ps *radeon_state,
1028                                             RV770_SMC_STATETABLE *table)
1029 {
1030         struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state);
1031         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1032         u32 a_t;
1033
1034         table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL =
1035                 cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl);
1036         table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 =
1037                 cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2);
1038         table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL =
1039                 cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl);
1040         table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 =
1041                 cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2);
1042         table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL =
1043                 cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl);
1044         table->initialState.levels[0].mclk.mclk770.vDLL_CNTL =
1045                 cpu_to_be32(pi->clk_regs.rv770.dll_cntl);
1046
1047         table->initialState.levels[0].mclk.mclk770.vMPLL_SS =
1048                 cpu_to_be32(pi->clk_regs.rv770.mpll_ss1);
1049         table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 =
1050                 cpu_to_be32(pi->clk_regs.rv770.mpll_ss2);
1051
1052         table->initialState.levels[0].mclk.mclk770.mclk_value =
1053                 cpu_to_be32(initial_state->low.mclk);
1054
1055         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1056                 cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl);
1057         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1058                 cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2);
1059         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1060                 cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3);
1061         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1062                 cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum);
1063         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1064                 cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2);
1065
1066         table->initialState.levels[0].sclk.sclk_value =
1067                 cpu_to_be32(initial_state->low.sclk);
1068
1069         table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
1070
1071         table->initialState.levels[0].seqValue =
1072                 rv770_get_seq_value(rdev, &initial_state->low);
1073
1074         rv770_populate_vddc_value(rdev,
1075                                   initial_state->low.vddc,
1076                                   &table->initialState.levels[0].vddc);
1077         rv770_populate_initial_mvdd_value(rdev,
1078                                           &table->initialState.levels[0].mvdd);
1079
1080         a_t = CG_R(0xffff) | CG_L(0);
1081         table->initialState.levels[0].aT = cpu_to_be32(a_t);
1082
1083         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1084
1085         if (pi->boot_in_gen2)
1086                 table->initialState.levels[0].gen2PCIE = 1;
1087         else
1088                 table->initialState.levels[0].gen2PCIE = 0;
1089         if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
1090                 table->initialState.levels[0].gen2XSP = 1;
1091         else
1092                 table->initialState.levels[0].gen2XSP = 0;
1093
1094         if (rdev->family == CHIP_RV740) {
1095                 if (pi->mem_gddr5) {
1096                         if (initial_state->low.mclk <= pi->mclk_strobe_mode_threshold)
1097                                 table->initialState.levels[0].strobeMode =
1098                                         rv740_get_mclk_frequency_ratio(initial_state->low.mclk) | 0x10;
1099                         else
1100                                 table->initialState.levels[0].strobeMode = 0;
1101
1102                         if (initial_state->low.mclk >= pi->mclk_edc_enable_threshold)
1103                                 table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
1104                         else
1105                                 table->initialState.levels[0].mcFlags =  0;
1106                 }
1107         }
1108
1109         table->initialState.levels[1] = table->initialState.levels[0];
1110         table->initialState.levels[2] = table->initialState.levels[0];
1111
1112         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1113
1114         return 0;
1115 }
1116
1117 static int rv770_populate_smc_vddc_table(struct radeon_device *rdev,
1118                                          RV770_SMC_STATETABLE *table)
1119 {
1120         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1121         int i;
1122
1123         for (i = 0; i < pi->valid_vddc_entries; i++) {
1124                 table->highSMIO[pi->vddc_table[i].vddc_index] =
1125                         pi->vddc_table[i].high_smio;
1126                 table->lowSMIO[pi->vddc_table[i].vddc_index] =
1127                         cpu_to_be32(pi->vddc_table[i].low_smio);
1128         }
1129
1130         table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0;
1131         table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] =
1132                 cpu_to_be32(pi->vddc_mask_low);
1133
1134         for (i = 0;
1135              ((i < pi->valid_vddc_entries) &&
1136               (pi->max_vddc_in_table >
1137                pi->vddc_table[i].vddc));
1138              i++);
1139
1140         table->maxVDDCIndexInPPTable =
1141                 pi->vddc_table[i].vddc_index;
1142
1143         return 0;
1144 }
1145
1146 static int rv770_populate_smc_mvdd_table(struct radeon_device *rdev,
1147                                          RV770_SMC_STATETABLE *table)
1148 {
1149         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1150
1151         if (pi->mvdd_control) {
1152                 table->lowSMIO[MVDD_HIGH_INDEX] |=
1153                         cpu_to_be32(pi->mvdd_low_smio[MVDD_HIGH_INDEX]);
1154                 table->lowSMIO[MVDD_LOW_INDEX] |=
1155                         cpu_to_be32(pi->mvdd_low_smio[MVDD_LOW_INDEX]);
1156
1157                 table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_MVDD] = 0;
1158                 table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_MVDD] =
1159                         cpu_to_be32(pi->mvdd_mask_low);
1160         }
1161
1162         return 0;
1163 }
1164
1165 static int rv770_init_smc_table(struct radeon_device *rdev,
1166                                 struct radeon_ps *radeon_boot_state)
1167 {
1168         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1169         struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
1170         RV770_SMC_STATETABLE *table = &pi->smc_statetable;
1171         int ret;
1172
1173         memset(table, 0, sizeof(RV770_SMC_STATETABLE));
1174
1175         pi->boot_sclk = boot_state->low.sclk;
1176
1177         rv770_populate_smc_vddc_table(rdev, table);
1178         rv770_populate_smc_mvdd_table(rdev, table);
1179
1180         switch (rdev->pm.int_thermal_type) {
1181         case THERMAL_TYPE_RV770:
1182         case THERMAL_TYPE_ADT7473_WITH_INTERNAL:
1183                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1184                 break;
1185         case THERMAL_TYPE_NONE:
1186                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1187                 break;
1188         case THERMAL_TYPE_EXTERNAL_GPIO:
1189         default:
1190                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1191                 break;
1192         }
1193
1194         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) {
1195                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1196
1197                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT)
1198                         table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK;
1199
1200                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT)
1201                         table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE;
1202         }
1203
1204         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1205                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1206
1207         if (pi->mem_gddr5)
1208                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1209
1210         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1211                 ret = rv730_populate_smc_initial_state(rdev, radeon_boot_state, table);
1212         else
1213                 ret = rv770_populate_smc_initial_state(rdev, radeon_boot_state, table);
1214         if (ret)
1215                 return ret;
1216
1217         if (rdev->family == CHIP_RV740)
1218                 ret = rv740_populate_smc_acpi_state(rdev, table);
1219         else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1220                 ret = rv730_populate_smc_acpi_state(rdev, table);
1221         else
1222                 ret = rv770_populate_smc_acpi_state(rdev, table);
1223         if (ret)
1224                 return ret;
1225
1226         table->driverState = table->initialState;
1227
1228         return rv770_copy_bytes_to_smc(rdev,
1229                                        pi->state_table_start,
1230                                        (const u8 *)table,
1231                                        sizeof(RV770_SMC_STATETABLE),
1232                                        pi->sram_end);
1233 }
1234
1235 static int rv770_construct_vddc_table(struct radeon_device *rdev)
1236 {
1237         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1238         u16 min, max, step;
1239         u32 steps = 0;
1240         u8 vddc_index = 0;
1241         u32 i;
1242
1243         radeon_atom_get_min_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &min);
1244         radeon_atom_get_max_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &max);
1245         radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &step);
1246
1247         steps = (max - min) / step + 1;
1248
1249         if (steps > MAX_NO_VREG_STEPS)
1250                 return -EINVAL;
1251
1252         for (i = 0; i < steps; i++) {
1253                 u32 gpio_pins, gpio_mask;
1254
1255                 pi->vddc_table[i].vddc = (u16)(min + i * step);
1256                 radeon_atom_get_voltage_gpio_settings(rdev,
1257                                                       pi->vddc_table[i].vddc,
1258                                                       SET_VOLTAGE_TYPE_ASIC_VDDC,
1259                                                       &gpio_pins, &gpio_mask);
1260                 pi->vddc_table[i].low_smio = gpio_pins & gpio_mask;
1261                 pi->vddc_table[i].high_smio = 0;
1262                 pi->vddc_mask_low = gpio_mask;
1263                 if (i > 0) {
1264                         if ((pi->vddc_table[i].low_smio !=
1265                              pi->vddc_table[i - 1].low_smio ) ||
1266                              (pi->vddc_table[i].high_smio !=
1267                               pi->vddc_table[i - 1].high_smio))
1268                                 vddc_index++;
1269                 }
1270                 pi->vddc_table[i].vddc_index = vddc_index;
1271         }
1272
1273         pi->valid_vddc_entries = (u8)steps;
1274
1275         return 0;
1276 }
1277
1278 static u32 rv770_get_mclk_split_point(struct atom_memory_info *memory_info)
1279 {
1280         if (memory_info->mem_type == MEM_TYPE_GDDR3)
1281                 return 30000;
1282
1283         return 0;
1284 }
1285
1286 static int rv770_get_mvdd_pin_configuration(struct radeon_device *rdev)
1287 {
1288         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1289         u32 gpio_pins, gpio_mask;
1290
1291         radeon_atom_get_voltage_gpio_settings(rdev,
1292                                               MVDD_HIGH_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
1293                                               &gpio_pins, &gpio_mask);
1294         pi->mvdd_mask_low = gpio_mask;
1295         pi->mvdd_low_smio[MVDD_HIGH_INDEX] =
1296                 gpio_pins & gpio_mask;
1297
1298         radeon_atom_get_voltage_gpio_settings(rdev,
1299                                               MVDD_LOW_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
1300                                               &gpio_pins, &gpio_mask);
1301         pi->mvdd_low_smio[MVDD_LOW_INDEX] =
1302                 gpio_pins & gpio_mask;
1303
1304         return 0;
1305 }
1306
1307 u8 rv770_get_memory_module_index(struct radeon_device *rdev)
1308 {
1309         return (u8) ((RREG32(BIOS_SCRATCH_4) >> 16) & 0xff);
1310 }
1311
1312 static int rv770_get_mvdd_configuration(struct radeon_device *rdev)
1313 {
1314         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1315         u8 memory_module_index;
1316         struct atom_memory_info memory_info;
1317
1318         memory_module_index = rv770_get_memory_module_index(rdev);
1319
1320         if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info)) {
1321                 pi->mvdd_control = false;
1322                 return 0;
1323         }
1324
1325         pi->mvdd_split_frequency =
1326                 rv770_get_mclk_split_point(&memory_info);
1327
1328         if (pi->mvdd_split_frequency == 0) {
1329                 pi->mvdd_control = false;
1330                 return 0;
1331         }
1332
1333         return rv770_get_mvdd_pin_configuration(rdev);
1334 }
1335
1336 void rv770_enable_voltage_control(struct radeon_device *rdev,
1337                                   bool enable)
1338 {
1339         if (enable)
1340                 WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
1341         else
1342                 WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
1343 }
1344
1345 static void rv770_program_display_gap(struct radeon_device *rdev)
1346 {
1347         u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
1348
1349         tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
1350         if (rdev->pm.dpm.new_active_crtcs & 1) {
1351                 tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1352                 tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1353         } else if (rdev->pm.dpm.new_active_crtcs & 2) {
1354                 tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1355                 tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1356         } else {
1357                 tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1358                 tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1359         }
1360         WREG32(CG_DISPLAY_GAP_CNTL, tmp);
1361 }
1362
1363 static void rv770_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
1364                                            bool enable)
1365 {
1366         rv770_enable_bif_dynamic_pcie_gen2(rdev, enable);
1367
1368         if (enable)
1369                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
1370         else
1371                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
1372 }
1373
1374 static void r7xx_program_memory_timing_parameters(struct radeon_device *rdev,
1375                                                   struct radeon_ps *radeon_new_state)
1376 {
1377         if ((rdev->family == CHIP_RV730) ||
1378             (rdev->family == CHIP_RV710) ||
1379             (rdev->family == CHIP_RV740))
1380                 rv730_program_memory_timing_parameters(rdev, radeon_new_state);
1381         else
1382                 rv770_program_memory_timing_parameters(rdev, radeon_new_state);
1383 }
1384
1385 static int rv770_upload_sw_state(struct radeon_device *rdev,
1386                                  struct radeon_ps *radeon_new_state)
1387 {
1388         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1389         u16 address = pi->state_table_start +
1390                 offsetof(RV770_SMC_STATETABLE, driverState);
1391         RV770_SMC_SWSTATE state = { 0 };
1392         int ret;
1393
1394         ret = rv770_convert_power_state_to_smc(rdev, radeon_new_state, &state);
1395         if (ret)
1396                 return ret;
1397
1398         return rv770_copy_bytes_to_smc(rdev, address, (const u8 *)&state,
1399                                        sizeof(RV770_SMC_SWSTATE),
1400                                        pi->sram_end);
1401 }
1402
1403 int rv770_halt_smc(struct radeon_device *rdev)
1404 {
1405         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK)
1406                 return -EINVAL;
1407
1408         if (rv770_wait_for_smc_inactive(rdev) != PPSMC_Result_OK)
1409                 return -EINVAL;
1410
1411         return 0;
1412 }
1413
1414 int rv770_resume_smc(struct radeon_device *rdev)
1415 {
1416         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Resume) != PPSMC_Result_OK)
1417                 return -EINVAL;
1418         return 0;
1419 }
1420
1421 int rv770_set_sw_state(struct radeon_device *rdev)
1422 {
1423         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) != PPSMC_Result_OK)
1424                 return -EINVAL;
1425         return 0;
1426 }
1427
1428 int rv770_set_boot_state(struct radeon_device *rdev)
1429 {
1430         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) != PPSMC_Result_OK)
1431                 return -EINVAL;
1432         return 0;
1433 }
1434
1435 void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
1436                                               struct radeon_ps *new_ps,
1437                                               struct radeon_ps *old_ps)
1438 {
1439         struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
1440         struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
1441
1442         if ((new_ps->vclk == old_ps->vclk) &&
1443             (new_ps->dclk == old_ps->dclk))
1444                 return;
1445
1446         if (new_state->high.sclk >= current_state->high.sclk)
1447                 return;
1448
1449         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1450 }
1451
1452 void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
1453                                              struct radeon_ps *new_ps,
1454                                              struct radeon_ps *old_ps)
1455 {
1456         struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
1457         struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
1458
1459         if ((new_ps->vclk == old_ps->vclk) &&
1460             (new_ps->dclk == old_ps->dclk))
1461                 return;
1462
1463         if (new_state->high.sclk < current_state->high.sclk)
1464                 return;
1465
1466         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1467 }
1468
1469 int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1470 {
1471         if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_NoForcedLevel)) != PPSMC_Result_OK)
1472                 return -EINVAL;
1473
1474         if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled)) != PPSMC_Result_OK)
1475                 return -EINVAL;
1476
1477         return 0;
1478 }
1479
1480 int rv770_dpm_force_performance_level(struct radeon_device *rdev,
1481                                       enum radeon_dpm_forced_level level)
1482 {
1483         PPSMC_Msg msg;
1484
1485         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1486                 if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_ZeroLevelsDisabled) != PPSMC_Result_OK)
1487                         return -EINVAL;
1488                 msg = PPSMC_MSG_ForceHigh;
1489         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1490                 if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1491                         return -EINVAL;
1492                 msg = (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled);
1493         } else {
1494                 if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1495                         return -EINVAL;
1496                 msg = (PPSMC_Msg)(PPSMC_MSG_ZeroLevelsDisabled);
1497         }
1498
1499         if (rv770_send_msg_to_smc(rdev, msg) != PPSMC_Result_OK)
1500                 return -EINVAL;
1501
1502         rdev->pm.dpm.forced_level = level;
1503
1504         return 0;
1505 }
1506
1507 void r7xx_start_smc(struct radeon_device *rdev)
1508 {
1509         rv770_start_smc(rdev);
1510         rv770_start_smc_clock(rdev);
1511 }
1512
1513
1514 void r7xx_stop_smc(struct radeon_device *rdev)
1515 {
1516         rv770_reset_smc(rdev);
1517         rv770_stop_smc_clock(rdev);
1518 }
1519
1520 static void rv770_read_clock_registers(struct radeon_device *rdev)
1521 {
1522         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1523
1524         pi->clk_regs.rv770.cg_spll_func_cntl =
1525                 RREG32(CG_SPLL_FUNC_CNTL);
1526         pi->clk_regs.rv770.cg_spll_func_cntl_2 =
1527                 RREG32(CG_SPLL_FUNC_CNTL_2);
1528         pi->clk_regs.rv770.cg_spll_func_cntl_3 =
1529                 RREG32(CG_SPLL_FUNC_CNTL_3);
1530         pi->clk_regs.rv770.cg_spll_spread_spectrum =
1531                 RREG32(CG_SPLL_SPREAD_SPECTRUM);
1532         pi->clk_regs.rv770.cg_spll_spread_spectrum_2 =
1533                 RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1534         pi->clk_regs.rv770.mpll_ad_func_cntl =
1535                 RREG32(MPLL_AD_FUNC_CNTL);
1536         pi->clk_regs.rv770.mpll_ad_func_cntl_2 =
1537                 RREG32(MPLL_AD_FUNC_CNTL_2);
1538         pi->clk_regs.rv770.mpll_dq_func_cntl =
1539                 RREG32(MPLL_DQ_FUNC_CNTL);
1540         pi->clk_regs.rv770.mpll_dq_func_cntl_2 =
1541                 RREG32(MPLL_DQ_FUNC_CNTL_2);
1542         pi->clk_regs.rv770.mclk_pwrmgt_cntl =
1543                 RREG32(MCLK_PWRMGT_CNTL);
1544         pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL);
1545 }
1546
1547 static void r7xx_read_clock_registers(struct radeon_device *rdev)
1548 {
1549         if (rdev->family == CHIP_RV740)
1550                 rv740_read_clock_registers(rdev);
1551         else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1552                 rv730_read_clock_registers(rdev);
1553         else
1554                 rv770_read_clock_registers(rdev);
1555 }
1556
1557 void rv770_read_voltage_smio_registers(struct radeon_device *rdev)
1558 {
1559         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1560
1561         pi->s0_vid_lower_smio_cntl =
1562                 RREG32(S0_VID_LOWER_SMIO_CNTL);
1563 }
1564
1565 void rv770_reset_smio_status(struct radeon_device *rdev)
1566 {
1567         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1568         u32 sw_smio_index, vid_smio_cntl;
1569
1570         sw_smio_index =
1571                 (RREG32(GENERAL_PWRMGT) & SW_SMIO_INDEX_MASK) >> SW_SMIO_INDEX_SHIFT;
1572         switch (sw_smio_index) {
1573         case 3:
1574                 vid_smio_cntl = RREG32(S3_VID_LOWER_SMIO_CNTL);
1575                 break;
1576         case 2:
1577                 vid_smio_cntl = RREG32(S2_VID_LOWER_SMIO_CNTL);
1578                 break;
1579         case 1:
1580                 vid_smio_cntl = RREG32(S1_VID_LOWER_SMIO_CNTL);
1581                 break;
1582         case 0:
1583                 return;
1584         default:
1585                 vid_smio_cntl = pi->s0_vid_lower_smio_cntl;
1586                 break;
1587         }
1588
1589         WREG32(S0_VID_LOWER_SMIO_CNTL, vid_smio_cntl);
1590         WREG32_P(GENERAL_PWRMGT, SW_SMIO_INDEX(0), ~SW_SMIO_INDEX_MASK);
1591 }
1592
1593 void rv770_get_memory_type(struct radeon_device *rdev)
1594 {
1595         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1596         u32 tmp;
1597
1598         tmp = RREG32(MC_SEQ_MISC0);
1599
1600         if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) ==
1601             MC_SEQ_MISC0_GDDR5_VALUE)
1602                 pi->mem_gddr5 = true;
1603         else
1604                 pi->mem_gddr5 = false;
1605
1606 }
1607
1608 void rv770_get_pcie_gen2_status(struct radeon_device *rdev)
1609 {
1610         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1611         u32 tmp;
1612
1613         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1614
1615         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
1616             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
1617                 pi->pcie_gen2 = true;
1618         else
1619                 pi->pcie_gen2 = false;
1620
1621         if (pi->pcie_gen2) {
1622                 if (tmp & LC_CURRENT_DATA_RATE)
1623                         pi->boot_in_gen2 = true;
1624                 else
1625                         pi->boot_in_gen2 = false;
1626         } else
1627                 pi->boot_in_gen2 = false;
1628 }
1629
1630 #if 0
1631 static int rv770_enter_ulp_state(struct radeon_device *rdev)
1632 {
1633         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1634
1635         if (pi->gfx_clock_gating) {
1636                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1637                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1638                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1639                 RREG32(GB_TILING_CONFIG);
1640         }
1641
1642         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1643                  ~HOST_SMC_MSG_MASK);
1644
1645         DRM_UDELAY(7000);
1646
1647         return 0;
1648 }
1649
1650 static int rv770_exit_ulp_state(struct radeon_device *rdev)
1651 {
1652         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1653         int i;
1654
1655         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_ResumeFromMinimumPower),
1656                  ~HOST_SMC_MSG_MASK);
1657
1658         DRM_UDELAY(7000);
1659
1660         for (i = 0; i < rdev->usec_timeout; i++) {
1661                 if (((RREG32(SMC_MSG) & HOST_SMC_RESP_MASK) >> HOST_SMC_RESP_SHIFT) == 1)
1662                         break;
1663                 DRM_UDELAY(1000);
1664         }
1665
1666         if (pi->gfx_clock_gating)
1667                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
1668
1669         return 0;
1670 }
1671 #endif
1672
1673 static void rv770_get_mclk_odt_threshold(struct radeon_device *rdev)
1674 {
1675         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1676         u8 memory_module_index;
1677         struct atom_memory_info memory_info;
1678
1679         pi->mclk_odt_threshold = 0;
1680
1681         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) {
1682                 memory_module_index = rv770_get_memory_module_index(rdev);
1683
1684                 if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info))
1685                         return;
1686
1687                 if (memory_info.mem_type == MEM_TYPE_DDR2 ||
1688                     memory_info.mem_type == MEM_TYPE_DDR3)
1689                         pi->mclk_odt_threshold = 30000;
1690         }
1691 }
1692
1693 void rv770_get_max_vddc(struct radeon_device *rdev)
1694 {
1695         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1696         u16 vddc;
1697
1698         if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc))
1699                 pi->max_vddc = 0;
1700         else
1701                 pi->max_vddc = vddc;
1702 }
1703
1704 void rv770_program_response_times(struct radeon_device *rdev)
1705 {
1706         u32 voltage_response_time, backbias_response_time;
1707         u32 acpi_delay_time, vbi_time_out;
1708         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly;
1709         u32 reference_clock;
1710
1711         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1712         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1713
1714         if (voltage_response_time == 0)
1715                 voltage_response_time = 1000;
1716
1717         if (backbias_response_time == 0)
1718                 backbias_response_time = 1000;
1719
1720         acpi_delay_time = 15000;
1721         vbi_time_out = 100000;
1722
1723         reference_clock = radeon_get_xclk(rdev);
1724
1725         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1726         bb_dly = (backbias_response_time * reference_clock) / 1600;
1727         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1728         vbi_dly = (vbi_time_out * reference_clock) / 1600;
1729
1730         rv770_write_smc_soft_register(rdev,
1731                                       RV770_SMC_SOFT_REGISTER_delay_vreg, vddc_dly);
1732         rv770_write_smc_soft_register(rdev,
1733                                       RV770_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1734         rv770_write_smc_soft_register(rdev,
1735                                       RV770_SMC_SOFT_REGISTER_delay_acpi, acpi_dly);
1736         rv770_write_smc_soft_register(rdev,
1737                                       RV770_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1738 #if 0
1739         /* XXX look up hw revision */
1740         if (WEKIVA_A21)
1741                 rv770_write_smc_soft_register(rdev,
1742                                               RV770_SMC_SOFT_REGISTER_baby_step_timer,
1743                                               0x10);
1744 #endif
1745 }
1746
1747 static void rv770_program_dcodt_before_state_switch(struct radeon_device *rdev,
1748                                                     struct radeon_ps *radeon_new_state,
1749                                                     struct radeon_ps *radeon_current_state)
1750 {
1751         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1752         struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
1753         struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
1754         bool current_use_dc = false;
1755         bool new_use_dc = false;
1756
1757         if (pi->mclk_odt_threshold == 0)
1758                 return;
1759
1760         if (current_state->high.mclk <= pi->mclk_odt_threshold)
1761                 current_use_dc = true;
1762
1763         if (new_state->high.mclk <= pi->mclk_odt_threshold)
1764                 new_use_dc = true;
1765
1766         if (current_use_dc == new_use_dc)
1767                 return;
1768
1769         if (!current_use_dc && new_use_dc)
1770                 return;
1771
1772         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1773                 rv730_program_dcodt(rdev, new_use_dc);
1774 }
1775
1776 static void rv770_program_dcodt_after_state_switch(struct radeon_device *rdev,
1777                                                    struct radeon_ps *radeon_new_state,
1778                                                    struct radeon_ps *radeon_current_state)
1779 {
1780         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1781         struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
1782         struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
1783         bool current_use_dc = false;
1784         bool new_use_dc = false;
1785
1786         if (pi->mclk_odt_threshold == 0)
1787                 return;
1788
1789         if (current_state->high.mclk <= pi->mclk_odt_threshold)
1790                 current_use_dc = true;
1791
1792         if (new_state->high.mclk <= pi->mclk_odt_threshold)
1793                 new_use_dc = true;
1794
1795         if (current_use_dc == new_use_dc)
1796                 return;
1797
1798         if (current_use_dc && !new_use_dc)
1799                 return;
1800
1801         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1802                 rv730_program_dcodt(rdev, new_use_dc);
1803 }
1804
1805 static void rv770_retrieve_odt_values(struct radeon_device *rdev)
1806 {
1807         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1808
1809         if (pi->mclk_odt_threshold == 0)
1810                 return;
1811
1812         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1813                 rv730_get_odt_values(rdev);
1814 }
1815
1816 static void rv770_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
1817 {
1818         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1819         bool want_thermal_protection;
1820         enum radeon_dpm_event_src dpm_event_src;
1821
1822         switch (sources) {
1823         case 0:
1824         default:
1825                 want_thermal_protection = false;
1826                 break;
1827         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
1828                 want_thermal_protection = true;
1829                 dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
1830                 break;
1831
1832         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1833                 want_thermal_protection = true;
1834                 dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
1835                 break;
1836
1837         case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1838               (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1839                 want_thermal_protection = true;
1840                 dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
1841                 break;
1842         }
1843
1844         if (want_thermal_protection) {
1845                 WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
1846                 if (pi->thermal_protection)
1847                         WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
1848         } else {
1849                 WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
1850         }
1851 }
1852
1853 void rv770_enable_auto_throttle_source(struct radeon_device *rdev,
1854                                        enum radeon_dpm_auto_throttle_src source,
1855                                        bool enable)
1856 {
1857         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1858
1859         if (enable) {
1860                 if (!(pi->active_auto_throttle_sources & (1 << source))) {
1861                         pi->active_auto_throttle_sources |= 1 << source;
1862                         rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1863                 }
1864         } else {
1865                 if (pi->active_auto_throttle_sources & (1 << source)) {
1866                         pi->active_auto_throttle_sources &= ~(1 << source);
1867                         rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1868                 }
1869         }
1870 }
1871
1872 int rv770_set_thermal_temperature_range(struct radeon_device *rdev,
1873                                         int min_temp, int max_temp)
1874 {
1875         int low_temp = 0 * 1000;
1876         int high_temp = 255 * 1000;
1877
1878         if (low_temp < min_temp)
1879                 low_temp = min_temp;
1880         if (high_temp > max_temp)
1881                 high_temp = max_temp;
1882         if (high_temp < low_temp) {
1883                 DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1884                 return -EINVAL;
1885         }
1886
1887         WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
1888         WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
1889         WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
1890
1891         rdev->pm.dpm.thermal.min_temp = low_temp;
1892         rdev->pm.dpm.thermal.max_temp = high_temp;
1893
1894         return 0;
1895 }
1896
1897 int rv770_dpm_enable(struct radeon_device *rdev)
1898 {
1899         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1900         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
1901         int ret;
1902
1903         if (pi->gfx_clock_gating)
1904                 rv770_restore_cgcg(rdev);
1905
1906         if (rv770_dpm_enabled(rdev))
1907                 return -EINVAL;
1908
1909         if (pi->voltage_control) {
1910                 rv770_enable_voltage_control(rdev, true);
1911                 ret = rv770_construct_vddc_table(rdev);
1912                 if (ret) {
1913                         DRM_ERROR("rv770_construct_vddc_table failed\n");
1914                         return ret;
1915                 }
1916         }
1917
1918         if (pi->dcodt)
1919                 rv770_retrieve_odt_values(rdev);
1920
1921         if (pi->mvdd_control) {
1922                 ret = rv770_get_mvdd_configuration(rdev);
1923                 if (ret) {
1924                         DRM_ERROR("rv770_get_mvdd_configuration failed\n");
1925                         return ret;
1926                 }
1927         }
1928
1929         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1930                 rv770_enable_backbias(rdev, true);
1931
1932         rv770_enable_spread_spectrum(rdev, true);
1933
1934         if (pi->thermal_protection)
1935                 rv770_enable_thermal_protection(rdev, true);
1936
1937         rv770_program_mpll_timing_parameters(rdev);
1938         rv770_setup_bsp(rdev);
1939         rv770_program_git(rdev);
1940         rv770_program_tp(rdev);
1941         rv770_program_tpp(rdev);
1942         rv770_program_sstp(rdev);
1943         rv770_program_engine_speed_parameters(rdev);
1944         rv770_enable_display_gap(rdev);
1945         rv770_program_vc(rdev);
1946
1947         if (pi->dynamic_pcie_gen2)
1948                 rv770_enable_dynamic_pcie_gen2(rdev, true);
1949
1950         ret = rv770_upload_firmware(rdev);
1951         if (ret) {
1952                 DRM_ERROR("rv770_upload_firmware failed\n");
1953                 return ret;
1954         }
1955         ret = rv770_init_smc_table(rdev, boot_ps);
1956         if (ret) {
1957                 DRM_ERROR("rv770_init_smc_table failed\n");
1958                 return ret;
1959         }
1960
1961         rv770_program_response_times(rdev);
1962         r7xx_start_smc(rdev);
1963
1964         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1965                 rv730_start_dpm(rdev);
1966         else
1967                 rv770_start_dpm(rdev);
1968
1969         if (pi->gfx_clock_gating)
1970                 rv770_gfx_clock_gating_enable(rdev, true);
1971
1972         if (pi->mg_clock_gating)
1973                 rv770_mg_clock_gating_enable(rdev, true);
1974
1975         if (rdev->irq.installed &&
1976             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1977                 PPSMC_Result result;
1978
1979                 ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1980                 if (ret)
1981                         return ret;
1982                 rdev->irq.dpm_thermal = true;
1983                 radeon_irq_set(rdev);
1984                 result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
1985
1986                 if (result != PPSMC_Result_OK)
1987                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
1988         }
1989
1990         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
1991
1992         return 0;
1993 }
1994
1995 void rv770_dpm_disable(struct radeon_device *rdev)
1996 {
1997         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1998
1999         if (!rv770_dpm_enabled(rdev))
2000                 return;
2001
2002         rv770_clear_vc(rdev);
2003
2004         if (pi->thermal_protection)
2005                 rv770_enable_thermal_protection(rdev, false);
2006
2007         rv770_enable_spread_spectrum(rdev, false);
2008
2009         if (pi->dynamic_pcie_gen2)
2010                 rv770_enable_dynamic_pcie_gen2(rdev, false);
2011
2012         if (rdev->irq.installed &&
2013             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
2014                 rdev->irq.dpm_thermal = false;
2015                 radeon_irq_set(rdev);
2016         }
2017
2018         if (pi->gfx_clock_gating)
2019                 rv770_gfx_clock_gating_enable(rdev, false);
2020
2021         if (pi->mg_clock_gating)
2022                 rv770_mg_clock_gating_enable(rdev, false);
2023
2024         if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
2025                 rv730_stop_dpm(rdev);
2026         else
2027                 rv770_stop_dpm(rdev);
2028
2029         r7xx_stop_smc(rdev);
2030         rv770_reset_smio_status(rdev);
2031 }
2032
2033 int rv770_dpm_set_power_state(struct radeon_device *rdev)
2034 {
2035         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2036         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
2037         struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
2038         int ret;
2039
2040         ret = rv770_restrict_performance_levels_before_switch(rdev);
2041         if (ret) {
2042                 DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n");
2043                 return ret;
2044         }
2045         rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
2046         ret = rv770_halt_smc(rdev);
2047         if (ret) {
2048                 DRM_ERROR("rv770_halt_smc failed\n");
2049                 return ret;
2050         }
2051         ret = rv770_upload_sw_state(rdev, new_ps);
2052         if (ret) {
2053                 DRM_ERROR("rv770_upload_sw_state failed\n");
2054                 return ret;
2055         }
2056         r7xx_program_memory_timing_parameters(rdev, new_ps);
2057         if (pi->dcodt)
2058                 rv770_program_dcodt_before_state_switch(rdev, new_ps, old_ps);
2059         ret = rv770_resume_smc(rdev);
2060         if (ret) {
2061                 DRM_ERROR("rv770_resume_smc failed\n");
2062                 return ret;
2063         }
2064         ret = rv770_set_sw_state(rdev);
2065         if (ret) {
2066                 DRM_ERROR("rv770_set_sw_state failed\n");
2067                 return ret;
2068         }
2069         if (pi->dcodt)
2070                 rv770_program_dcodt_after_state_switch(rdev, new_ps, old_ps);
2071         rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
2072
2073         ret = rv770_dpm_force_performance_level(rdev, RADEON_DPM_FORCED_LEVEL_AUTO);
2074         if (ret) {
2075                 DRM_ERROR("rv770_dpm_force_performance_level failed\n");
2076                 return ret;
2077         }
2078
2079         return 0;
2080 }
2081
2082 void rv770_dpm_reset_asic(struct radeon_device *rdev)
2083 {
2084         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2085         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
2086
2087         rv770_restrict_performance_levels_before_switch(rdev);
2088         if (pi->dcodt)
2089                 rv770_program_dcodt_before_state_switch(rdev, boot_ps, boot_ps);
2090         rv770_set_boot_state(rdev);
2091         if (pi->dcodt)
2092                 rv770_program_dcodt_after_state_switch(rdev, boot_ps, boot_ps);
2093 }
2094
2095 void rv770_dpm_setup_asic(struct radeon_device *rdev)
2096 {
2097         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2098
2099         r7xx_read_clock_registers(rdev);
2100         rv770_read_voltage_smio_registers(rdev);
2101         rv770_get_memory_type(rdev);
2102         if (pi->dcodt)
2103                 rv770_get_mclk_odt_threshold(rdev);
2104         rv770_get_pcie_gen2_status(rdev);
2105
2106         rv770_enable_acpi_pm(rdev);
2107
2108         if (radeon_aspm != 0) {
2109                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
2110                         rv770_enable_l0s(rdev);
2111                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
2112                         rv770_enable_l1(rdev);
2113                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
2114                         rv770_enable_pll_sleep_in_l1(rdev);
2115         }
2116 }
2117
2118 void rv770_dpm_display_configuration_changed(struct radeon_device *rdev)
2119 {
2120         rv770_program_display_gap(rdev);
2121 }
2122
2123 union power_info {
2124         struct _ATOM_POWERPLAY_INFO info;
2125         struct _ATOM_POWERPLAY_INFO_V2 info_2;
2126         struct _ATOM_POWERPLAY_INFO_V3 info_3;
2127         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
2128         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
2129         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
2130 };
2131
2132 union pplib_clock_info {
2133         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
2134         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
2135         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
2136         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
2137 };
2138
2139 union pplib_power_state {
2140         struct _ATOM_PPLIB_STATE v1;
2141         struct _ATOM_PPLIB_STATE_V2 v2;
2142 };
2143
2144 static void rv7xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
2145                                              struct radeon_ps *rps,
2146                                              struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
2147                                              u8 table_rev)
2148 {
2149         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
2150         rps->class = le16_to_cpu(non_clock_info->usClassification);
2151         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
2152
2153         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
2154                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
2155                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
2156         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
2157                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
2158                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
2159         } else {
2160                 rps->vclk = 0;
2161                 rps->dclk = 0;
2162         }
2163
2164         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
2165                 rdev->pm.dpm.boot_ps = rps;
2166         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
2167                 rdev->pm.dpm.uvd_ps = rps;
2168 }
2169
2170 static void rv7xx_parse_pplib_clock_info(struct radeon_device *rdev,
2171                                          struct radeon_ps *rps, int index,
2172                                          union pplib_clock_info *clock_info)
2173 {
2174         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2175         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2176         struct rv7xx_ps *ps = rv770_get_ps(rps);
2177         u32 sclk, mclk;
2178         u16 vddc;
2179         struct rv7xx_pl *pl;
2180
2181         switch (index) {
2182         case 0:
2183                 pl = &ps->low;
2184                 break;
2185         case 1:
2186                 pl = &ps->medium;
2187                 break;
2188         case 2:
2189         default:
2190                 pl = &ps->high;
2191                 break;
2192         }
2193
2194         if (rdev->family >= CHIP_CEDAR) {
2195                 sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
2196                 sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
2197                 mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
2198                 mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
2199
2200                 pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
2201                 pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
2202                 pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
2203         } else {
2204                 sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
2205                 sclk |= clock_info->r600.ucEngineClockHigh << 16;
2206                 mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
2207                 mclk |= clock_info->r600.ucMemoryClockHigh << 16;
2208
2209                 pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
2210                 pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
2211         }
2212
2213         pl->mclk = mclk;
2214         pl->sclk = sclk;
2215
2216         /* patch up vddc if necessary */
2217         if (pl->vddc == 0xff01) {
2218                 if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
2219                         pl->vddc = vddc;
2220         }
2221
2222         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
2223                 pi->acpi_vddc = pl->vddc;
2224                 if (rdev->family >= CHIP_CEDAR)
2225                         eg_pi->acpi_vddci = pl->vddci;
2226                 if (ps->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
2227                         pi->acpi_pcie_gen2 = true;
2228                 else
2229                         pi->acpi_pcie_gen2 = false;
2230         }
2231
2232         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
2233                 if (rdev->family >= CHIP_BARTS) {
2234                         eg_pi->ulv.supported = true;
2235                         eg_pi->ulv.pl = pl;
2236                 }
2237         }
2238
2239         if (pi->min_vddc_in_table > pl->vddc)
2240                 pi->min_vddc_in_table = pl->vddc;
2241
2242         if (pi->max_vddc_in_table < pl->vddc)
2243                 pi->max_vddc_in_table = pl->vddc;
2244
2245         /* patch up boot state */
2246         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
2247                 u16 vddc, vddci, mvdd;
2248                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
2249                 pl->mclk = rdev->clock.default_mclk;
2250                 pl->sclk = rdev->clock.default_sclk;
2251                 pl->vddc = vddc;
2252                 pl->vddci = vddci;
2253         }
2254
2255         if (rdev->family >= CHIP_BARTS) {
2256                 if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
2257                     ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
2258                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
2259                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
2260                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
2261                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
2262                 }
2263         }
2264 }
2265
2266 int rv7xx_parse_power_table(struct radeon_device *rdev)
2267 {
2268         struct radeon_mode_info *mode_info = &rdev->mode_info;
2269         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
2270         union pplib_power_state *power_state;
2271         int i, j;
2272         union pplib_clock_info *clock_info;
2273         union power_info *power_info;
2274         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
2275         u16 data_offset;
2276         u8 frev, crev;
2277         struct rv7xx_ps *ps;
2278
2279         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
2280                                    &frev, &crev, &data_offset))
2281                 return -EINVAL;
2282         power_info = (union power_info *)((uint8_t*)mode_info->atom_context->bios + data_offset);
2283
2284         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
2285                                   power_info->pplib.ucNumStates, GFP_KERNEL);
2286         if (!rdev->pm.dpm.ps)
2287                 return -ENOMEM;
2288         rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
2289         rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
2290         rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
2291
2292         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
2293                 power_state = (union pplib_power_state *)
2294                         ((uint8_t*)mode_info->atom_context->bios + data_offset +
2295                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
2296                          i * power_info->pplib.ucStateEntrySize);
2297                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
2298                         ((uint8_t*)mode_info->atom_context->bios + data_offset +
2299                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
2300                          (power_state->v1.ucNonClockStateIndex *
2301                           power_info->pplib.ucNonClockSize));
2302                 if (power_info->pplib.ucStateEntrySize - 1) {
2303                         ps = kzalloc(sizeof(struct rv7xx_ps), GFP_KERNEL);
2304                         if (ps == NULL) {
2305                                 kfree(rdev->pm.dpm.ps);
2306                                 return -ENOMEM;
2307                         }
2308                         rdev->pm.dpm.ps[i].ps_priv = ps;
2309                         rv7xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
2310                                                          non_clock_info,
2311                                                          power_info->pplib.ucNonClockSize);
2312                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
2313                                 clock_info = (union pplib_clock_info *)
2314                                         ((uint8_t*)mode_info->atom_context->bios + data_offset +
2315                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
2316                                          (power_state->v1.ucClockStateIndices[j] *
2317                                           power_info->pplib.ucClockInfoSize));
2318                                 rv7xx_parse_pplib_clock_info(rdev,
2319                                                              &rdev->pm.dpm.ps[i], j,
2320                                                              clock_info);
2321                         }
2322                 }
2323         }
2324         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
2325         return 0;
2326 }
2327
2328 void rv770_get_engine_memory_ss(struct radeon_device *rdev)
2329 {
2330         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2331         struct radeon_atom_ss ss;
2332
2333         pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
2334                                                        ASIC_INTERNAL_ENGINE_SS, 0);
2335         pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
2336                                                        ASIC_INTERNAL_MEMORY_SS, 0);
2337
2338         if (pi->sclk_ss || pi->mclk_ss)
2339                 pi->dynamic_ss = true;
2340         else
2341                 pi->dynamic_ss = false;
2342 }
2343
2344 int rv770_dpm_init(struct radeon_device *rdev)
2345 {
2346         struct rv7xx_power_info *pi;
2347         struct atom_clock_dividers dividers;
2348         int ret;
2349
2350         pi = kzalloc(sizeof(struct rv7xx_power_info), GFP_KERNEL);
2351         if (pi == NULL)
2352                 return -ENOMEM;
2353         rdev->pm.dpm.priv = pi;
2354
2355         rv770_get_max_vddc(rdev);
2356
2357         pi->acpi_vddc = 0;
2358         pi->min_vddc_in_table = 0;
2359         pi->max_vddc_in_table = 0;
2360
2361         ret = rv7xx_parse_power_table(rdev);
2362         if (ret)
2363                 return ret;
2364
2365         if (rdev->pm.dpm.voltage_response_time == 0)
2366                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
2367         if (rdev->pm.dpm.backbias_response_time == 0)
2368                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
2369
2370         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2371                                              0, false, &dividers);
2372         if (ret)
2373                 pi->ref_div = dividers.ref_div + 1;
2374         else
2375                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
2376
2377         pi->mclk_strobe_mode_threshold = 30000;
2378         pi->mclk_edc_enable_threshold = 30000;
2379
2380         pi->rlp = RV770_RLP_DFLT;
2381         pi->rmp = RV770_RMP_DFLT;
2382         pi->lhp = RV770_LHP_DFLT;
2383         pi->lmp = RV770_LMP_DFLT;
2384
2385         pi->voltage_control =
2386                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
2387
2388         pi->mvdd_control =
2389                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
2390
2391         rv770_get_engine_memory_ss(rdev);
2392
2393         pi->asi = RV770_ASI_DFLT;
2394         pi->pasi = RV770_HASI_DFLT;
2395         pi->vrc = RV770_VRC_DFLT;
2396
2397         pi->power_gating = false;
2398
2399         pi->gfx_clock_gating = true;
2400
2401         pi->mg_clock_gating = true;
2402         pi->mgcgtssm = true;
2403
2404         pi->dynamic_pcie_gen2 = true;
2405
2406         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
2407                 pi->thermal_protection = true;
2408         else
2409                 pi->thermal_protection = false;
2410
2411         pi->display_gap = true;
2412
2413         if (rdev->flags & RADEON_IS_MOBILITY)
2414                 pi->dcodt = true;
2415         else
2416                 pi->dcodt = false;
2417
2418         pi->ulps = true;
2419
2420         pi->mclk_stutter_mode_threshold = 0;
2421
2422         pi->sram_end = SMC_RAM_END;
2423         pi->state_table_start = RV770_SMC_TABLE_ADDRESS;
2424         pi->soft_regs_start = RV770_SMC_SOFT_REGISTERS_START;
2425
2426         return 0;
2427 }
2428
2429 void rv770_dpm_print_power_state(struct radeon_device *rdev,
2430                                  struct radeon_ps *rps)
2431 {
2432         struct rv7xx_ps *ps = rv770_get_ps(rps);
2433         struct rv7xx_pl *pl;
2434
2435         r600_dpm_print_class_info(rps->class, rps->class2);
2436         r600_dpm_print_cap_info(rps->caps);
2437         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2438         if (rdev->family >= CHIP_CEDAR) {
2439                 pl = &ps->low;
2440                 printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2441                        pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2442                 pl = &ps->medium;
2443                 printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2444                        pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2445                 pl = &ps->high;
2446                 printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2447                        pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2448         } else {
2449                 pl = &ps->low;
2450                 printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
2451                        pl->sclk, pl->mclk, pl->vddc);
2452                 pl = &ps->medium;
2453                 printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
2454                        pl->sclk, pl->mclk, pl->vddc);
2455                 pl = &ps->high;
2456                 printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
2457                        pl->sclk, pl->mclk, pl->vddc);
2458         }
2459         r600_dpm_print_ps_status(rdev, rps);
2460 }
2461
2462 void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
2463                                                        struct seq_file *m)
2464 {
2465         struct radeon_ps *rps = rdev->pm.dpm.current_ps;
2466         struct rv7xx_ps *ps = rv770_get_ps(rps);
2467         struct rv7xx_pl *pl;
2468         u32 current_index =
2469                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
2470                 CURRENT_PROFILE_INDEX_SHIFT;
2471
2472         if (current_index > 2) {
2473                 seq_printf(m, "invalid dpm profile %d\n", current_index);
2474         } else {
2475                 if (current_index == 0)
2476                         pl = &ps->low;
2477                 else if (current_index == 1)
2478                         pl = &ps->medium;
2479                 else /* current_index == 2 */
2480                         pl = &ps->high;
2481                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2482                 if (rdev->family >= CHIP_CEDAR) {
2483                         seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2484                                    current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2485                 } else {
2486                         seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
2487                                    current_index, pl->sclk, pl->mclk, pl->vddc);
2488                 }
2489         }
2490 }
2491
2492 void rv770_dpm_fini(struct radeon_device *rdev)
2493 {
2494         int i;
2495
2496         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2497                 kfree(rdev->pm.dpm.ps[i].ps_priv);
2498         }
2499         kfree(rdev->pm.dpm.ps);
2500         kfree(rdev->pm.dpm.priv);
2501 }
2502
2503 u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low)
2504 {
2505         struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
2506
2507         if (low)
2508                 return requested_state->low.sclk;
2509         else
2510                 return requested_state->high.sclk;
2511 }
2512
2513 u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low)
2514 {
2515         struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
2516
2517         if (low)
2518                 return requested_state->low.mclk;
2519         else
2520                 return requested_state->high.mclk;
2521 }
2522
2523 bool rv770_dpm_vblank_too_short(struct radeon_device *rdev)
2524 {
2525         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
2526
2527         if (vblank_time < 300)
2528                 return true;
2529         else
2530                 return false;
2531
2532 }