sys/dev/drm/radeon/rv6xx_dpm.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <drm/drmP.h>
  26 #include "radeon.h"
  27 #include "radeon_asic.h"
  28 #include "rv6xxd.h"
  29 #include "r600_dpm.h"
  30 #include "rv6xx_dpm.h"
  31 #include "atom.h"
  32 #include <linux/seq_file.h>
  33
  34 static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
  35                                         u32 unscaled_count, u32 unit);
  36
  37 static struct rv6xx_ps *rv6xx_get_ps(struct radeon_ps *rps)
  38 {
  39         struct rv6xx_ps *ps = rps->ps_priv;
  40
  41         return ps;
  42 }
  43
  44 static struct rv6xx_power_info *rv6xx_get_pi(struct radeon_device *rdev)
  45 {
  46         struct rv6xx_power_info *pi = rdev->pm.dpm.priv;
  47
  48         return pi;
  49 }
  50
  51 static void rv6xx_force_pcie_gen1(struct radeon_device *rdev)
  52 {
  53         u32 tmp;
  54         int i;
  55
  56         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
  57         tmp &= LC_GEN2_EN;
  58         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
  59
  60         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
  61         tmp |= LC_INITIATE_LINK_SPEED_CHANGE;
  62         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
  63
  64         for (i = 0; i < rdev->usec_timeout; i++) {
  65                 if (!(RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE))
  66                         break;
  67                 udelay(1);
  68         }
  69
  70         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
  71         tmp &= ~LC_INITIATE_LINK_SPEED_CHANGE;
  72         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
  73 }
  74
  75 static void rv6xx_enable_pcie_gen2_support(struct radeon_device *rdev)
  76 {
  77         u32 tmp;
  78
  79         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
  80
  81         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
  82             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
  83                 tmp |= LC_GEN2_EN;
  84                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
  85         }
  86 }
  87
  88 static void rv6xx_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
  89                                                bool enable)
  90 {
  91         u32 tmp;
  92
  93         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
  94         if (enable)
  95                 tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
  96         else
  97                 tmp |= LC_HW_VOLTAGE_IF_CONTROL(0);
  98         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
  99 }
 100
 101 static void rv6xx_enable_l0s(struct radeon_device *rdev)
 102 {
 103         u32 tmp;
 104
 105         tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
 106         tmp |= LC_L0S_INACTIVITY(3);
 107         WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
 108 }
 109
 110 static void rv6xx_enable_l1(struct radeon_device *rdev)
 111 {
 112         u32 tmp;
 113
 114         tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
 115         tmp &= ~LC_L1_INACTIVITY_MASK;
 116         tmp |= LC_L1_INACTIVITY(4);
 117         tmp &= ~LC_PMI_TO_L1_DIS;
 118         tmp &= ~LC_ASPM_TO_L1_DIS;
 119         WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
 120 }
 121
 122 static void rv6xx_enable_pll_sleep_in_l1(struct radeon_device *rdev)
 123 {
 124         u32 tmp;
 125
 126         tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
 127         tmp |= LC_L1_INACTIVITY(8);
 128         WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
 129
 130         /* NOTE, this is a PCIE indirect reg, not PCIE PORT */
 131         tmp = RREG32_PCIE(PCIE_P_CNTL);
 132         tmp |= P_PLL_PWRDN_IN_L1L23;
 133         tmp &= ~P_PLL_BUF_PDNB;
 134         tmp &= ~P_PLL_PDNB;
 135         tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
 136         WREG32_PCIE(PCIE_P_CNTL, tmp);
 137 }
 138
 139 static int rv6xx_convert_clock_to_stepping(struct radeon_device *rdev,
 140                                            u32 clock, struct rv6xx_sclk_stepping *step)
 141 {
 142         int ret;
 143         struct atom_clock_dividers dividers;
 144
 145         bzero(&dividers, sizeof(dividers));     /* avoid gcc warning */
 146         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 147                                              clock, false, &dividers);
 148         if (ret)
 149                 return ret;
 150
 151         if (dividers.enable_post_div)
 152                 step->post_divider = 2 + (dividers.post_div & 0xF) + (dividers.post_div >> 4);
 153         else
 154                 step->post_divider = 1;
 155
 156         step->vco_frequency = clock * step->post_divider;
 157
 158         return 0;
 159 }
 160
 161 static void rv6xx_output_stepping(struct radeon_device *rdev,
 162                                   u32 step_index, struct rv6xx_sclk_stepping *step)
 163 {
 164         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 165         u32 ref_clk = rdev->clock.spll.reference_freq;
 166         u32 fb_divider;
 167         u32 spll_step_count = rv6xx_scale_count_given_unit(rdev,
 168                                                            R600_SPLLSTEPTIME_DFLT *
 169                                                            pi->spll_ref_div,
 170                                                            R600_SPLLSTEPUNIT_DFLT);
 171
 172         r600_engine_clock_entry_enable(rdev, step_index, true);
 173         r600_engine_clock_entry_enable_pulse_skipping(rdev, step_index, false);
 174
 175         if (step->post_divider == 1)
 176                 r600_engine_clock_entry_enable_post_divider(rdev, step_index, false);
 177         else {
 178                 u32 lo_len = (step->post_divider - 2) / 2;
 179                 u32 hi_len = step->post_divider - 2 - lo_len;
 180
 181                 r600_engine_clock_entry_enable_post_divider(rdev, step_index, true);
 182                 r600_engine_clock_entry_set_post_divider(rdev, step_index, (hi_len << 4) | lo_len);
 183         }
 184
 185         fb_divider = ((step->vco_frequency * pi->spll_ref_div) / ref_clk) >>
 186                 pi->fb_div_scale;
 187
 188         r600_engine_clock_entry_set_reference_divider(rdev, step_index,
 189                                                       pi->spll_ref_div - 1);
 190         r600_engine_clock_entry_set_feedback_divider(rdev, step_index, fb_divider);
 191         r600_engine_clock_entry_set_step_time(rdev, step_index, spll_step_count);
 192
 193 }
 194
 195 static struct rv6xx_sclk_stepping rv6xx_next_vco_step(struct radeon_device *rdev,
 196                                                       struct rv6xx_sclk_stepping *cur,
 197                                                       bool increasing_vco, u32 step_size)
 198 {
 199         struct rv6xx_sclk_stepping next;
 200
 201         next.post_divider = cur->post_divider;
 202
 203         if (increasing_vco)
 204                 next.vco_frequency = (cur->vco_frequency * (100 + step_size)) / 100;
 205         else
 206                 next.vco_frequency = (cur->vco_frequency * 100 + 99 + step_size) / (100 + step_size);
 207
 208         return next;
 209 }
 210
 211 static bool rv6xx_can_step_post_div(struct radeon_device *rdev,
 212                                     struct rv6xx_sclk_stepping *cur,
 213                                     struct rv6xx_sclk_stepping *target)
 214 {
 215         return (cur->post_divider > target->post_divider) &&
 216                 ((cur->vco_frequency * target->post_divider) <=
 217                  (target->vco_frequency * (cur->post_divider - 1)));
 218 }
 219
 220 static struct rv6xx_sclk_stepping rv6xx_next_post_div_step(struct radeon_device *rdev,
 221                                                            struct rv6xx_sclk_stepping *cur,
 222                                                            struct rv6xx_sclk_stepping *target)
 223 {
 224         struct rv6xx_sclk_stepping next = *cur;
 225
 226         while (rv6xx_can_step_post_div(rdev, &next, target))
 227                 next.post_divider--;
 228
 229         return next;
 230 }
 231
 232 static bool rv6xx_reached_stepping_target(struct radeon_device *rdev,
 233                                           struct rv6xx_sclk_stepping *cur,
 234                                           struct rv6xx_sclk_stepping *target,
 235                                           bool increasing_vco)
 236 {
 237         return (increasing_vco && (cur->vco_frequency >= target->vco_frequency)) ||
 238                 (!increasing_vco && (cur->vco_frequency <= target->vco_frequency));
 239 }
 240
 241 static void rv6xx_generate_steps(struct radeon_device *rdev,
 242                                  u32 low, u32 high,
 243                                  u32 start_index, u8 *end_index)
 244 {
 245         struct rv6xx_sclk_stepping cur;
 246         struct rv6xx_sclk_stepping target;
 247         bool increasing_vco;
 248         u32 step_index = start_index;
 249
 250         bzero(&cur, sizeof(cur));       /* silence gcc warnings */
 251         bzero(&target, sizeof(target)); /* silence gcc warnings */
 252         rv6xx_convert_clock_to_stepping(rdev, low, &cur);
 253         rv6xx_convert_clock_to_stepping(rdev, high, &target);
 254
 255         rv6xx_output_stepping(rdev, step_index++, &cur);
 256
 257         increasing_vco = (target.vco_frequency >= cur.vco_frequency);
 258
 259         if (target.post_divider > cur.post_divider)
 260                 cur.post_divider = target.post_divider;
 261
 262         while (1) {
 263                 struct rv6xx_sclk_stepping next;
 264
 265                 if (rv6xx_can_step_post_div(rdev, &cur, &target))
 266                         next = rv6xx_next_post_div_step(rdev, &cur, &target);
 267                 else
 268                         next = rv6xx_next_vco_step(rdev, &cur, increasing_vco, R600_VCOSTEPPCT_DFLT);
 269
 270                 if (rv6xx_reached_stepping_target(rdev, &next, &target, increasing_vco)) {
 271                         struct rv6xx_sclk_stepping tiny =
 272                                 rv6xx_next_vco_step(rdev, &target, !increasing_vco, R600_ENDINGVCOSTEPPCT_DFLT);
 273                         tiny.post_divider = next.post_divider;
 274
 275                         if (!rv6xx_reached_stepping_target(rdev, &tiny, &cur, !increasing_vco))
 276                                 rv6xx_output_stepping(rdev, step_index++, &tiny);
 277
 278                         if ((next.post_divider != target.post_divider) &&
 279                             (next.vco_frequency != target.vco_frequency)) {
 280                                 struct rv6xx_sclk_stepping final_vco;
 281
 282                                 final_vco.vco_frequency = target.vco_frequency;
 283                                 final_vco.post_divider = next.post_divider;
 284
 285                                 rv6xx_output_stepping(rdev, step_index++, &final_vco);
 286                         }
 287
 288                         rv6xx_output_stepping(rdev, step_index++, &target);
 289                         break;
 290                 } else
 291                         rv6xx_output_stepping(rdev, step_index++, &next);
 292
 293                 cur = next;
 294         }
 295
 296         *end_index = (u8)step_index - 1;
 297
 298 }
 299
 300 static void rv6xx_generate_single_step(struct radeon_device *rdev,
 301                                        u32 clock, u32 index)
 302 {
 303         struct rv6xx_sclk_stepping step;
 304
 305         bzero(&step, sizeof(step));     /* silence gcc warnings */
 306         rv6xx_convert_clock_to_stepping(rdev, clock, &step);
 307         rv6xx_output_stepping(rdev, index, &step);
 308 }
 309
 310 static void rv6xx_invalidate_intermediate_steps_range(struct radeon_device *rdev,
 311                                                       u32 start_index, u32 end_index)
 312 {
 313         u32 step_index;
 314
 315         for (step_index = start_index + 1; step_index < end_index; step_index++)
 316                 r600_engine_clock_entry_enable(rdev, step_index, false);
 317 }
 318
 319 static void rv6xx_set_engine_spread_spectrum_clk_s(struct radeon_device *rdev,
 320                                                    u32 index, u32 clk_s)
 321 {
 322         WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
 323                  CLKS(clk_s), ~CLKS_MASK);
 324 }
 325
 326 static void rv6xx_set_engine_spread_spectrum_clk_v(struct radeon_device *rdev,
 327                                                    u32 index, u32 clk_v)
 328 {
 329         WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
 330                  CLKV(clk_v), ~CLKV_MASK);
 331 }
 332
 333 static void rv6xx_enable_engine_spread_spectrum(struct radeon_device *rdev,
 334                                                 u32 index, bool enable)
 335 {
 336         if (enable)
 337                 WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
 338                          SSEN, ~SSEN);
 339         else
 340                 WREG32_P(CG_SPLL_SPREAD_SPECTRUM_LOW + (index * 4),
 341                          0, ~SSEN);
 342 }
 343
 344 static void rv6xx_set_memory_spread_spectrum_clk_s(struct radeon_device *rdev,
 345                                                    u32 clk_s)
 346 {
 347         WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKS(clk_s), ~CLKS_MASK);
 348 }
 349
 350 static void rv6xx_set_memory_spread_spectrum_clk_v(struct radeon_device *rdev,
 351                                                    u32 clk_v)
 352 {
 353         WREG32_P(CG_MPLL_SPREAD_SPECTRUM, CLKV(clk_v), ~CLKV_MASK);
 354 }
 355
 356 static void rv6xx_enable_memory_spread_spectrum(struct radeon_device *rdev,
 357                                                 bool enable)
 358 {
 359         if (enable)
 360                 WREG32_P(CG_MPLL_SPREAD_SPECTRUM, SSEN, ~SSEN);
 361         else
 362                 WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
 363 }
 364
 365 static void rv6xx_enable_dynamic_spread_spectrum(struct radeon_device *rdev,
 366                                                  bool enable)
 367 {
 368         if (enable)
 369                 WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
 370         else
 371                 WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
 372 }
 373
 374 static void rv6xx_memory_clock_entry_enable_post_divider(struct radeon_device *rdev,
 375                                                          u32 index, bool enable)
 376 {
 377         if (enable)
 378                 WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
 379                          LEVEL0_MPLL_DIV_EN, ~LEVEL0_MPLL_DIV_EN);
 380         else
 381                 WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), 0, ~LEVEL0_MPLL_DIV_EN);
 382 }
 383
 384 static void rv6xx_memory_clock_entry_set_post_divider(struct radeon_device *rdev,
 385                                                       u32 index, u32 divider)
 386 {
 387         WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
 388                  LEVEL0_MPLL_POST_DIV(divider), ~LEVEL0_MPLL_POST_DIV_MASK);
 389 }
 390
 391 static void rv6xx_memory_clock_entry_set_feedback_divider(struct radeon_device *rdev,
 392                                                           u32 index, u32 divider)
 393 {
 394         WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4), LEVEL0_MPLL_FB_DIV(divider),
 395                  ~LEVEL0_MPLL_FB_DIV_MASK);
 396 }
 397
 398 static void rv6xx_memory_clock_entry_set_reference_divider(struct radeon_device *rdev,
 399                                                            u32 index, u32 divider)
 400 {
 401         WREG32_P(MPLL_FREQ_LEVEL_0 + (index * 4),
 402                  LEVEL0_MPLL_REF_DIV(divider), ~LEVEL0_MPLL_REF_DIV_MASK);
 403 }
 404
 405 static void rv6xx_vid_response_set_brt(struct radeon_device *rdev, u32 rt)
 406 {
 407         WREG32_P(VID_RT, BRT(rt), ~BRT_MASK);
 408 }
 409
 410 static void rv6xx_enable_engine_feedback_and_reference_sync(struct radeon_device *rdev)
 411 {
 412         WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
 413 }
 414
 415 static u32 rv6xx_clocks_per_unit(u32 unit)
 416 {
 417         u32 tmp = 1 << (2 * unit);
 418
 419         return tmp;
 420 }
 421
 422 static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
 423                                         u32 unscaled_count, u32 unit)
 424 {
 425         u32 count_per_unit = rv6xx_clocks_per_unit(unit);
 426
 427         return (unscaled_count + count_per_unit - 1) / count_per_unit;
 428 }
 429
 430 static u32 rv6xx_compute_count_for_delay(struct radeon_device *rdev,
 431                                          u32 delay_us, u32 unit)
 432 {
 433         u32 ref_clk = rdev->clock.spll.reference_freq;
 434
 435         return rv6xx_scale_count_given_unit(rdev, delay_us * (ref_clk / 100), unit);
 436 }
 437
 438 static void rv6xx_calculate_engine_speed_stepping_parameters(struct radeon_device *rdev,
 439                                                              struct rv6xx_ps *state)
 440 {
 441         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 442
 443         pi->hw.sclks[R600_POWER_LEVEL_LOW] =
 444                 state->low.sclk;
 445         pi->hw.sclks[R600_POWER_LEVEL_MEDIUM] =
 446                 state->medium.sclk;
 447         pi->hw.sclks[R600_POWER_LEVEL_HIGH] =
 448                 state->high.sclk;
 449
 450         pi->hw.low_sclk_index = R600_POWER_LEVEL_LOW;
 451         pi->hw.medium_sclk_index = R600_POWER_LEVEL_MEDIUM;
 452         pi->hw.high_sclk_index = R600_POWER_LEVEL_HIGH;
 453 }
 454
 455 static void rv6xx_calculate_memory_clock_stepping_parameters(struct radeon_device *rdev,
 456                                                              struct rv6xx_ps *state)
 457 {
 458         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 459
 460         pi->hw.mclks[R600_POWER_LEVEL_CTXSW] =
 461                 state->high.mclk;
 462         pi->hw.mclks[R600_POWER_LEVEL_HIGH] =
 463                 state->high.mclk;
 464         pi->hw.mclks[R600_POWER_LEVEL_MEDIUM] =
 465                 state->medium.mclk;
 466         pi->hw.mclks[R600_POWER_LEVEL_LOW] =
 467                 state->low.mclk;
 468
 469         pi->hw.high_mclk_index = R600_POWER_LEVEL_HIGH;
 470
 471         if (state->high.mclk == state->medium.mclk)
 472                 pi->hw.medium_mclk_index =
 473                         pi->hw.high_mclk_index;
 474         else
 475                 pi->hw.medium_mclk_index = R600_POWER_LEVEL_MEDIUM;
 476
 477
 478         if (state->medium.mclk == state->low.mclk)
 479                 pi->hw.low_mclk_index =
 480                         pi->hw.medium_mclk_index;
 481         else
 482                 pi->hw.low_mclk_index = R600_POWER_LEVEL_LOW;
 483 }
 484
 485 static void rv6xx_calculate_voltage_stepping_parameters(struct radeon_device *rdev,
 486                                                         struct rv6xx_ps *state)
 487 {
 488         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 489
 490         pi->hw.vddc[R600_POWER_LEVEL_CTXSW] = state->high.vddc;
 491         pi->hw.vddc[R600_POWER_LEVEL_HIGH] = state->high.vddc;
 492         pi->hw.vddc[R600_POWER_LEVEL_MEDIUM] = state->medium.vddc;
 493         pi->hw.vddc[R600_POWER_LEVEL_LOW] = state->low.vddc;
 494
 495         pi->hw.backbias[R600_POWER_LEVEL_CTXSW] =
 496                 (state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
 497         pi->hw.backbias[R600_POWER_LEVEL_HIGH] =
 498                 (state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
 499         pi->hw.backbias[R600_POWER_LEVEL_MEDIUM] =
 500                 (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
 501         pi->hw.backbias[R600_POWER_LEVEL_LOW] =
 502                 (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? true : false;
 503
 504         pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH] =
 505                 (state->high.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
 506         pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM] =
 507                 (state->medium.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
 508         pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW] =
 509                 (state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? true : false;
 510
 511         pi->hw.high_vddc_index = R600_POWER_LEVEL_HIGH;
 512
 513         if ((state->high.vddc == state->medium.vddc) &&
 514             ((state->high.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ==
 515              (state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)))
 516                 pi->hw.medium_vddc_index =
 517                         pi->hw.high_vddc_index;
 518         else
 519                 pi->hw.medium_vddc_index = R600_POWER_LEVEL_MEDIUM;
 520
 521         if ((state->medium.vddc == state->low.vddc) &&
 522             ((state->medium.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ==
 523              (state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE)))
 524                 pi->hw.low_vddc_index =
 525                         pi->hw.medium_vddc_index;
 526         else
 527                 pi->hw.medium_vddc_index = R600_POWER_LEVEL_LOW;
 528 }
 529
 530 static inline u32 rv6xx_calculate_vco_frequency(u32 ref_clock,
 531                                                 struct atom_clock_dividers *dividers,
 532                                                 u32 fb_divider_scale)
 533 {
 534         return ref_clock * ((dividers->fb_div & ~1) << fb_divider_scale) /
 535                 (dividers->ref_div + 1);
 536 }
 537
 538 static inline u32 rv6xx_calculate_spread_spectrum_clk_v(u32 vco_freq, u32 ref_freq,
 539                                                         u32 ss_rate, u32 ss_percent,
 540                                                         u32 fb_divider_scale)
 541 {
 542         u32 fb_divider = vco_freq / ref_freq;
 543
 544         return (ss_percent * ss_rate * 4 * (fb_divider * fb_divider) /
 545                 (5375 * ((vco_freq * 10) / (4096 >> fb_divider_scale))));
 546 }
 547
 548 static inline u32 rv6xx_calculate_spread_spectrum_clk_s(u32 ss_rate, u32 ref_freq)
 549 {
 550         return (((ref_freq * 10) / (ss_rate * 2)) - 1) / 4;
 551 }
 552
 553 static void rv6xx_program_engine_spread_spectrum(struct radeon_device *rdev,
 554                                                  u32 clock, enum r600_power_level level)
 555 {
 556         u32 ref_clk = rdev->clock.spll.reference_freq;
 557         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 558         struct atom_clock_dividers dividers;
 559         struct radeon_atom_ss ss;
 560         u32 vco_freq, clk_v, clk_s;
 561
 562         rv6xx_enable_engine_spread_spectrum(rdev, level, false);
 563
 564         if (clock && pi->sclk_ss) {
 565                 if (radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, clock, false, &dividers) == 0) {
 566                         vco_freq = rv6xx_calculate_vco_frequency(ref_clk, &dividers,
 567                                                                  pi->fb_div_scale);
 568
 569                         if (radeon_atombios_get_asic_ss_info(rdev, &ss,
 570                                                              ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
 571                                 clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq,
 572                                                                               (ref_clk / (dividers.ref_div + 1)),
 573                                                                               ss.rate,
 574                                                                               ss.percentage,
 575                                                                               pi->fb_div_scale);
 576
 577                                 clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate,
 578                                                                               (ref_clk / (dividers.ref_div + 1)));
 579
 580                                 rv6xx_set_engine_spread_spectrum_clk_v(rdev, level, clk_v);
 581                                 rv6xx_set_engine_spread_spectrum_clk_s(rdev, level, clk_s);
 582                                 rv6xx_enable_engine_spread_spectrum(rdev, level, true);
 583                         }
 584                 }
 585         }
 586 }
 587
 588 static void rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(struct radeon_device *rdev)
 589 {
 590         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 591
 592         rv6xx_program_engine_spread_spectrum(rdev,
 593                                              pi->hw.sclks[R600_POWER_LEVEL_HIGH],
 594                                              R600_POWER_LEVEL_HIGH);
 595
 596         rv6xx_program_engine_spread_spectrum(rdev,
 597                                              pi->hw.sclks[R600_POWER_LEVEL_MEDIUM],
 598                                              R600_POWER_LEVEL_MEDIUM);
 599
 600 }
 601
 602 static int rv6xx_program_mclk_stepping_entry(struct radeon_device *rdev,
 603                                              u32 entry, u32 clock)
 604 {
 605         struct atom_clock_dividers dividers;
 606
 607         if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM, clock, false, &dividers))
 608             return -EINVAL;
 609
 610
 611         rv6xx_memory_clock_entry_set_reference_divider(rdev, entry, dividers.ref_div);
 612         rv6xx_memory_clock_entry_set_feedback_divider(rdev, entry, dividers.fb_div);
 613         rv6xx_memory_clock_entry_set_post_divider(rdev, entry, dividers.post_div);
 614
 615         if (dividers.enable_post_div)
 616                 rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, true);
 617         else
 618                 rv6xx_memory_clock_entry_enable_post_divider(rdev, entry, false);
 619
 620         return 0;
 621 }
 622
 623 static void rv6xx_program_mclk_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
 624 {
 625         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 626         int i;
 627
 628         for (i = 1; i < R600_PM_NUMBER_OF_MCLKS; i++) {
 629                 if (pi->hw.mclks[i])
 630                         rv6xx_program_mclk_stepping_entry(rdev, i,
 631                                                           pi->hw.mclks[i]);
 632         }
 633 }
 634
 635 static void rv6xx_find_memory_clock_with_highest_vco(struct radeon_device *rdev,
 636                                                      u32 requested_memory_clock,
 637                                                      u32 ref_clk,
 638                                                      struct atom_clock_dividers *dividers,
 639                                                      u32 *vco_freq)
 640 {
 641         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 642         struct atom_clock_dividers req_dividers;
 643         u32 vco_freq_temp;
 644
 645         if (radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
 646                                            requested_memory_clock, false, &req_dividers) == 0) {
 647                 vco_freq_temp = rv6xx_calculate_vco_frequency(ref_clk, &req_dividers,
 648                                                               pi->fb_div_scale);
 649
 650                 if (vco_freq_temp > *vco_freq) {
 651                         *dividers = req_dividers;
 652                         *vco_freq = vco_freq_temp;
 653                 }
 654         }
 655 }
 656
 657 static void rv6xx_program_mclk_spread_spectrum_parameters(struct radeon_device *rdev)
 658 {
 659         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 660         u32 ref_clk = rdev->clock.mpll.reference_freq;
 661         struct atom_clock_dividers dividers;
 662         struct radeon_atom_ss ss;
 663         u32 vco_freq = 0, clk_v, clk_s;
 664
 665         bzero(&dividers, sizeof(dividers));     /* avoid gcc warning */
 666         rv6xx_enable_memory_spread_spectrum(rdev, false);
 667
 668         if (pi->mclk_ss) {
 669                 rv6xx_find_memory_clock_with_highest_vco(rdev,
 670                                                          pi->hw.mclks[pi->hw.high_mclk_index],
 671                                                          ref_clk,
 672                                                          &dividers,
 673                                                          &vco_freq);
 674
 675                 rv6xx_find_memory_clock_with_highest_vco(rdev,
 676                                                          pi->hw.mclks[pi->hw.medium_mclk_index],
 677                                                          ref_clk,
 678                                                          &dividers,
 679                                                          &vco_freq);
 680
 681                 rv6xx_find_memory_clock_with_highest_vco(rdev,
 682                                                          pi->hw.mclks[pi->hw.low_mclk_index],
 683                                                          ref_clk,
 684                                                          &dividers,
 685                                                          &vco_freq);
 686
 687                 if (vco_freq) {
 688                         if (radeon_atombios_get_asic_ss_info(rdev, &ss,
 689                                                              ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
 690                                 clk_v = rv6xx_calculate_spread_spectrum_clk_v(vco_freq,
 691                                                                              (ref_clk / (dividers.ref_div + 1)),
 692                                                                              ss.rate,
 693                                                                              ss.percentage,
 694                                                                              pi->fb_div_scale);
 695
 696                                 clk_s = rv6xx_calculate_spread_spectrum_clk_s(ss.rate,
 697                                                                              (ref_clk / (dividers.ref_div + 1)));
 698
 699                                 rv6xx_set_memory_spread_spectrum_clk_v(rdev, clk_v);
 700                                 rv6xx_set_memory_spread_spectrum_clk_s(rdev, clk_s);
 701                                 rv6xx_enable_memory_spread_spectrum(rdev, true);
 702                         }
 703                 }
 704         }
 705 }
 706
 707 static int rv6xx_program_voltage_stepping_entry(struct radeon_device *rdev,
 708                                                 u32 entry, u16 voltage)
 709 {
 710         u32 mask, set_pins;
 711         int ret;
 712
 713         ret = radeon_atom_get_voltage_gpio_settings(rdev, voltage,
 714                                                     SET_VOLTAGE_TYPE_ASIC_VDDC,
 715                                                     &set_pins, &mask);
 716         if (ret)
 717                 return ret;
 718
 719         r600_voltage_control_program_voltages(rdev, entry, set_pins);
 720
 721         return 0;
 722 }
 723
 724 static void rv6xx_program_voltage_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
 725 {
 726         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 727         int i;
 728
 729         for (i = 1; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++)
 730                 rv6xx_program_voltage_stepping_entry(rdev, i,
 731                                                      pi->hw.vddc[i]);
 732
 733 }
 734
 735 static void rv6xx_program_backbias_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
 736 {
 737         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 738
 739         if (pi->hw.backbias[1])
 740                 WREG32_P(VID_UPPER_GPIO_CNTL, MEDIUM_BACKBIAS_VALUE, ~MEDIUM_BACKBIAS_VALUE);
 741         else
 742                 WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~MEDIUM_BACKBIAS_VALUE);
 743
 744         if (pi->hw.backbias[2])
 745                 WREG32_P(VID_UPPER_GPIO_CNTL, HIGH_BACKBIAS_VALUE, ~HIGH_BACKBIAS_VALUE);
 746         else
 747                 WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~HIGH_BACKBIAS_VALUE);
 748 }
 749
 750 static void rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(struct radeon_device *rdev)
 751 {
 752         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 753
 754         rv6xx_program_engine_spread_spectrum(rdev,
 755                                              pi->hw.sclks[R600_POWER_LEVEL_LOW],
 756                                              R600_POWER_LEVEL_LOW);
 757 }
 758
 759 static void rv6xx_program_mclk_stepping_parameters_lowest_entry(struct radeon_device *rdev)
 760 {
 761         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 762
 763         if (pi->hw.mclks[0])
 764                 rv6xx_program_mclk_stepping_entry(rdev, 0,
 765                                                   pi->hw.mclks[0]);
 766 }
 767
 768 static void rv6xx_program_voltage_stepping_parameters_lowest_entry(struct radeon_device *rdev)
 769 {
 770         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 771
 772         rv6xx_program_voltage_stepping_entry(rdev, 0,
 773                                              pi->hw.vddc[0]);
 774
 775 }
 776
 777 static void rv6xx_program_backbias_stepping_parameters_lowest_entry(struct radeon_device *rdev)
 778 {
 779         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 780
 781         if (pi->hw.backbias[0])
 782                 WREG32_P(VID_UPPER_GPIO_CNTL, LOW_BACKBIAS_VALUE, ~LOW_BACKBIAS_VALUE);
 783         else
 784                 WREG32_P(VID_UPPER_GPIO_CNTL, 0, ~LOW_BACKBIAS_VALUE);
 785 }
 786
 787 static u32 calculate_memory_refresh_rate(struct radeon_device *rdev,
 788                                          u32 engine_clock)
 789 {
 790         u32 dram_rows, dram_refresh_rate;
 791         u32 tmp;
 792
 793         tmp = (RREG32(RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
 794         dram_rows = 1 << (tmp + 10);
 795         dram_refresh_rate = 1 << ((RREG32(MC_SEQ_RESERVE_M) & 0x3) + 3);
 796
 797         return ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
 798 }
 799
 800 static void rv6xx_program_memory_timing_parameters(struct radeon_device *rdev)
 801 {
 802         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 803         u32 sqm_ratio;
 804         u32 arb_refresh_rate;
 805         u32 high_clock;
 806
 807         if (pi->hw.sclks[R600_POWER_LEVEL_HIGH] <
 808             (pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40))
 809                 high_clock = pi->hw.sclks[R600_POWER_LEVEL_HIGH];
 810         else
 811                 high_clock =
 812                         pi->hw.sclks[R600_POWER_LEVEL_LOW] * 0xFF / 0x40;
 813
 814         radeon_atom_set_engine_dram_timings(rdev, high_clock, 0);
 815
 816         sqm_ratio = (STATE0(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_LOW]) |
 817                      STATE1(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_MEDIUM]) |
 818                      STATE2(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]) |
 819                      STATE3(64 * high_clock / pi->hw.sclks[R600_POWER_LEVEL_HIGH]));
 820         WREG32(SQM_RATIO, sqm_ratio);
 821
 822         arb_refresh_rate =
 823                 (POWERMODE0(calculate_memory_refresh_rate(rdev,
 824                                                           pi->hw.sclks[R600_POWER_LEVEL_LOW])) |
 825                  POWERMODE1(calculate_memory_refresh_rate(rdev,
 826                                                           pi->hw.sclks[R600_POWER_LEVEL_MEDIUM])) |
 827                  POWERMODE2(calculate_memory_refresh_rate(rdev,
 828                                                           pi->hw.sclks[R600_POWER_LEVEL_HIGH])) |
 829                  POWERMODE3(calculate_memory_refresh_rate(rdev,
 830                                                           pi->hw.sclks[R600_POWER_LEVEL_HIGH])));
 831         WREG32(ARB_RFSH_RATE, arb_refresh_rate);
 832 }
 833
 834 static void rv6xx_program_mpll_timing_parameters(struct radeon_device *rdev)
 835 {
 836         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 837
 838         r600_set_mpll_lock_time(rdev, R600_MPLLLOCKTIME_DFLT *
 839                                 pi->mpll_ref_div);
 840         r600_set_mpll_reset_time(rdev, R600_MPLLRESETTIME_DFLT);
 841 }
 842
 843 static void rv6xx_program_bsp(struct radeon_device *rdev)
 844 {
 845         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 846         u32 ref_clk = rdev->clock.spll.reference_freq;
 847
 848         r600_calculate_u_and_p(R600_ASI_DFLT,
 849                                ref_clk, 16,
 850                                &pi->bsp,
 851                                &pi->bsu);
 852
 853         r600_set_bsp(rdev, pi->bsu, pi->bsp);
 854 }
 855
 856 static void rv6xx_program_at(struct radeon_device *rdev)
 857 {
 858         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 859
 860         r600_set_at(rdev,
 861                     (pi->hw.rp[0] * pi->bsp) / 200,
 862                     (pi->hw.rp[1] * pi->bsp) / 200,
 863                     (pi->hw.lp[2] * pi->bsp) / 200,
 864                     (pi->hw.lp[1] * pi->bsp) / 200);
 865 }
 866
 867 static void rv6xx_program_git(struct radeon_device *rdev)
 868 {
 869         r600_set_git(rdev, R600_GICST_DFLT);
 870 }
 871
 872 static void rv6xx_program_tp(struct radeon_device *rdev)
 873 {
 874         int i;
 875
 876         for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
 877                 r600_set_tc(rdev, i, r600_utc[i], r600_dtc[i]);
 878
 879         r600_select_td(rdev, R600_TD_DFLT);
 880 }
 881
 882 static void rv6xx_program_vc(struct radeon_device *rdev)
 883 {
 884         r600_set_vrc(rdev, R600_VRC_DFLT);
 885 }
 886
 887 static void rv6xx_clear_vc(struct radeon_device *rdev)
 888 {
 889         r600_set_vrc(rdev, 0);
 890 }
 891
 892 static void rv6xx_program_tpp(struct radeon_device *rdev)
 893 {
 894         r600_set_tpu(rdev, R600_TPU_DFLT);
 895         r600_set_tpc(rdev, R600_TPC_DFLT);
 896 }
 897
 898 static void rv6xx_program_sstp(struct radeon_device *rdev)
 899 {
 900         r600_set_sstu(rdev, R600_SSTU_DFLT);
 901         r600_set_sst(rdev, R600_SST_DFLT);
 902 }
 903
 904 static void rv6xx_program_fcp(struct radeon_device *rdev)
 905 {
 906         r600_set_fctu(rdev, R600_FCTU_DFLT);
 907         r600_set_fct(rdev, R600_FCT_DFLT);
 908 }
 909
 910 static void rv6xx_program_vddc3d_parameters(struct radeon_device *rdev)
 911 {
 912         r600_set_vddc3d_oorsu(rdev, R600_VDDC3DOORSU_DFLT);
 913         r600_set_vddc3d_oorphc(rdev, R600_VDDC3DOORPHC_DFLT);
 914         r600_set_vddc3d_oorsdc(rdev, R600_VDDC3DOORSDC_DFLT);
 915         r600_set_ctxcgtt3d_rphc(rdev, R600_CTXCGTT3DRPHC_DFLT);
 916         r600_set_ctxcgtt3d_rsdc(rdev, R600_CTXCGTT3DRSDC_DFLT);
 917 }
 918
 919 static void rv6xx_program_voltage_timing_parameters(struct radeon_device *rdev)
 920 {
 921         u32 rt;
 922
 923         r600_vid_rt_set_vru(rdev, R600_VRU_DFLT);
 924
 925         r600_vid_rt_set_vrt(rdev,
 926                             rv6xx_compute_count_for_delay(rdev,
 927                                                           rdev->pm.dpm.voltage_response_time,
 928                                                           R600_VRU_DFLT));
 929
 930         rt = rv6xx_compute_count_for_delay(rdev,
 931                                            rdev->pm.dpm.backbias_response_time,
 932                                            R600_VRU_DFLT);
 933
 934         rv6xx_vid_response_set_brt(rdev, (rt + 0x1F) >> 5);
 935 }
 936
 937 static void rv6xx_program_engine_speed_parameters(struct radeon_device *rdev)
 938 {
 939         r600_vid_rt_set_ssu(rdev, R600_SPLLSTEPUNIT_DFLT);
 940         rv6xx_enable_engine_feedback_and_reference_sync(rdev);
 941 }
 942
 943 static u64 rv6xx_get_master_voltage_mask(struct radeon_device *rdev)
 944 {
 945         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
 946         u64 master_mask = 0;
 947         int i;
 948
 949         for (i = 0; i < R600_PM_NUMBER_OF_VOLTAGE_LEVELS; i++) {
 950                 u32 tmp_mask, tmp_set_pins;
 951                 int ret;
 952
 953                 ret = radeon_atom_get_voltage_gpio_settings(rdev,
 954                                                             pi->hw.vddc[i],
 955                                                             SET_VOLTAGE_TYPE_ASIC_VDDC,
 956                                                             &tmp_set_pins, &tmp_mask);
 957
 958                 if (ret == 0)
 959                         master_mask |= tmp_mask;
 960         }
 961
 962         return master_mask;
 963 }
 964
 965 static void rv6xx_program_voltage_gpio_pins(struct radeon_device *rdev)
 966 {
 967         r600_voltage_control_enable_pins(rdev,
 968                                          rv6xx_get_master_voltage_mask(rdev));
 969 }
 970
 971 static void rv6xx_enable_static_voltage_control(struct radeon_device *rdev,
 972                                                 struct radeon_ps *new_ps,
 973                                                 bool enable)
 974 {
 975         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
 976
 977         if (enable)
 978                 radeon_atom_set_voltage(rdev,
 979                                         new_state->low.vddc,
 980                                         SET_VOLTAGE_TYPE_ASIC_VDDC);
 981         else
 982                 r600_voltage_control_deactivate_static_control(rdev,
 983                                                                rv6xx_get_master_voltage_mask(rdev));
 984 }
 985
 986 static void rv6xx_enable_display_gap(struct radeon_device *rdev, bool enable)
 987 {
 988         if (enable) {
 989                 u32 tmp = (DISP1_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) |
 990                            DISP2_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM) |
 991                            DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
 992                            DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
 993                            VBI_TIMER_COUNT(0x3FFF) |
 994                            VBI_TIMER_UNIT(7));
 995                 WREG32(CG_DISPLAY_GAP_CNTL, tmp);
 996
 997                 WREG32_P(MCLK_PWRMGT_CNTL, USE_DISPLAY_GAP, ~USE_DISPLAY_GAP);
 998         } else
 999                 WREG32_P(MCLK_PWRMGT_CNTL, 0, ~USE_DISPLAY_GAP);
1000 }
1001
1002 static void rv6xx_program_power_level_enter_state(struct radeon_device *rdev)
1003 {
1004         r600_power_level_set_enter_index(rdev, R600_POWER_LEVEL_MEDIUM);
1005 }
1006
1007 static void rv6xx_calculate_t(u32 l_f, u32 h_f, int h,
1008                               int d_l, int d_r, u8 *l, u8 *r)
1009 {
1010         int a_n, a_d, h_r, l_r;
1011
1012         h_r = d_l;
1013         l_r = 100 - d_r;
1014
1015         a_n = (int)h_f * d_l + (int)l_f * (h - d_r);
1016         a_d = (int)l_f * l_r + (int)h_f * h_r;
1017
1018         if (a_d != 0) {
1019                 *l = d_l - h_r * a_n / a_d;
1020                 *r = d_r + l_r * a_n / a_d;
1021         }
1022 }
1023
1024 static void rv6xx_calculate_ap(struct radeon_device *rdev,
1025                                struct rv6xx_ps *state)
1026 {
1027         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1028
1029         pi->hw.lp[0] = 0;
1030         pi->hw.rp[R600_PM_NUMBER_OF_ACTIVITY_LEVELS - 1]
1031                 = 100;
1032
1033         rv6xx_calculate_t(state->low.sclk,
1034                           state->medium.sclk,
1035                           R600_AH_DFLT,
1036                           R600_LMP_DFLT,
1037                           R600_RLP_DFLT,
1038                           &pi->hw.lp[1],
1039                           &pi->hw.rp[0]);
1040
1041         rv6xx_calculate_t(state->medium.sclk,
1042                           state->high.sclk,
1043                           R600_AH_DFLT,
1044                           R600_LHP_DFLT,
1045                           R600_RMP_DFLT,
1046                           &pi->hw.lp[2],
1047                           &pi->hw.rp[1]);
1048
1049 }
1050
1051 static void rv6xx_calculate_stepping_parameters(struct radeon_device *rdev,
1052                                                 struct radeon_ps *new_ps)
1053 {
1054         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1055
1056         rv6xx_calculate_engine_speed_stepping_parameters(rdev, new_state);
1057         rv6xx_calculate_memory_clock_stepping_parameters(rdev, new_state);
1058         rv6xx_calculate_voltage_stepping_parameters(rdev, new_state);
1059         rv6xx_calculate_ap(rdev, new_state);
1060 }
1061
1062 static void rv6xx_program_stepping_parameters_except_lowest_entry(struct radeon_device *rdev)
1063 {
1064         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1065
1066         rv6xx_program_mclk_stepping_parameters_except_lowest_entry(rdev);
1067         if (pi->voltage_control)
1068                 rv6xx_program_voltage_stepping_parameters_except_lowest_entry(rdev);
1069         rv6xx_program_backbias_stepping_parameters_except_lowest_entry(rdev);
1070         rv6xx_program_sclk_spread_spectrum_parameters_except_lowest_entry(rdev);
1071         rv6xx_program_mclk_spread_spectrum_parameters(rdev);
1072         rv6xx_program_memory_timing_parameters(rdev);
1073 }
1074
1075 static void rv6xx_program_stepping_parameters_lowest_entry(struct radeon_device *rdev)
1076 {
1077         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1078
1079         rv6xx_program_mclk_stepping_parameters_lowest_entry(rdev);
1080         if (pi->voltage_control)
1081                 rv6xx_program_voltage_stepping_parameters_lowest_entry(rdev);
1082         rv6xx_program_backbias_stepping_parameters_lowest_entry(rdev);
1083         rv6xx_program_sclk_spread_spectrum_parameters_lowest_entry(rdev);
1084 }
1085
1086 static void rv6xx_program_power_level_low(struct radeon_device *rdev)
1087 {
1088         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1089
1090         r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW,
1091                                            pi->hw.low_vddc_index);
1092         r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW,
1093                                              pi->hw.low_mclk_index);
1094         r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW,
1095                                              pi->hw.low_sclk_index);
1096         r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,
1097                                           R600_DISPLAY_WATERMARK_LOW);
1098         r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW,
1099                                        pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
1100 }
1101
1102 static void rv6xx_program_power_level_low_to_lowest_state(struct radeon_device *rdev)
1103 {
1104         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1105
1106         r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_LOW, 0);
1107         r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_LOW, 0);
1108         r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_LOW, 0);
1109
1110         r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_LOW,
1111                                           R600_DISPLAY_WATERMARK_LOW);
1112
1113         r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_LOW,
1114                                        pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
1115
1116 }
1117
1118 static void rv6xx_program_power_level_medium(struct radeon_device *rdev)
1119 {
1120         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1121
1122         r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM,
1123                                           pi->hw.medium_vddc_index);
1124         r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
1125                                             pi->hw.medium_mclk_index);
1126         r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
1127                                             pi->hw.medium_sclk_index);
1128         r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM,
1129                                          R600_DISPLAY_WATERMARK_LOW);
1130         r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM,
1131                                       pi->hw.pcie_gen2[R600_POWER_LEVEL_MEDIUM]);
1132 }
1133
1134 static void rv6xx_program_power_level_medium_for_transition(struct radeon_device *rdev)
1135 {
1136         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1137
1138         rv6xx_program_mclk_stepping_entry(rdev,
1139                                           R600_POWER_LEVEL_CTXSW,
1140                                           pi->hw.mclks[pi->hw.low_mclk_index]);
1141
1142         r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_MEDIUM, 1);
1143
1144         r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
1145                                              R600_POWER_LEVEL_CTXSW);
1146         r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_MEDIUM,
1147                                              pi->hw.medium_sclk_index);
1148
1149         r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_MEDIUM,
1150                                           R600_DISPLAY_WATERMARK_LOW);
1151
1152         rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false);
1153
1154         r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_MEDIUM,
1155                                        pi->hw.pcie_gen2[R600_POWER_LEVEL_LOW]);
1156 }
1157
1158 static void rv6xx_program_power_level_high(struct radeon_device *rdev)
1159 {
1160         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1161
1162         r600_power_level_set_voltage_index(rdev, R600_POWER_LEVEL_HIGH,
1163                                            pi->hw.high_vddc_index);
1164         r600_power_level_set_mem_clock_index(rdev, R600_POWER_LEVEL_HIGH,
1165                                              pi->hw.high_mclk_index);
1166         r600_power_level_set_eng_clock_index(rdev, R600_POWER_LEVEL_HIGH,
1167                                              pi->hw.high_sclk_index);
1168
1169         r600_power_level_set_watermark_id(rdev, R600_POWER_LEVEL_HIGH,
1170                                           R600_DISPLAY_WATERMARK_HIGH);
1171
1172         r600_power_level_set_pcie_gen2(rdev, R600_POWER_LEVEL_HIGH,
1173                                        pi->hw.pcie_gen2[R600_POWER_LEVEL_HIGH]);
1174 }
1175
1176 static void rv6xx_enable_backbias(struct radeon_device *rdev, bool enable)
1177 {
1178         if (enable)
1179                 WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL,
1180                          ~(BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL));
1181         else
1182                 WREG32_P(GENERAL_PWRMGT, 0,
1183                          ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN | BACKBIAS_DPM_CNTL));
1184 }
1185
1186 static void rv6xx_program_display_gap(struct radeon_device *rdev)
1187 {
1188         u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
1189
1190         tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
1191         if (rdev->pm.dpm.new_active_crtcs & 1) {
1192                 tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1193                 tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1194         } else if (rdev->pm.dpm.new_active_crtcs & 2) {
1195                 tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1196                 tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1197         } else {
1198                 tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1199                 tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1200         }
1201         WREG32(CG_DISPLAY_GAP_CNTL, tmp);
1202 }
1203
1204 static void rv6xx_set_sw_voltage_to_safe(struct radeon_device *rdev,
1205                                          struct radeon_ps *new_ps,
1206                                          struct radeon_ps *old_ps)
1207 {
1208         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1209         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1210         u16 safe_voltage;
1211
1212         safe_voltage = (new_state->low.vddc >= old_state->low.vddc) ?
1213                 new_state->low.vddc : old_state->low.vddc;
1214
1215         rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
1216                                              safe_voltage);
1217
1218         WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW),
1219                  ~SW_GPIO_INDEX_MASK);
1220 }
1221
1222 static void rv6xx_set_sw_voltage_to_low(struct radeon_device *rdev,
1223                                         struct radeon_ps *old_ps)
1224 {
1225         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1226
1227         rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
1228                                              old_state->low.vddc);
1229
1230         WREG32_P(GENERAL_PWRMGT, SW_GPIO_INDEX(R600_POWER_LEVEL_CTXSW),
1231                 ~SW_GPIO_INDEX_MASK);
1232 }
1233
1234 static void rv6xx_set_safe_backbias(struct radeon_device *rdev,
1235                                     struct radeon_ps *new_ps,
1236                                     struct radeon_ps *old_ps)
1237 {
1238         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1239         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1240
1241         if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) &&
1242             (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE))
1243                 WREG32_P(GENERAL_PWRMGT, BACKBIAS_VALUE, ~BACKBIAS_VALUE);
1244         else
1245                 WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_VALUE);
1246 }
1247
1248 static void rv6xx_set_safe_pcie_gen2(struct radeon_device *rdev,
1249                                      struct radeon_ps *new_ps,
1250                                      struct radeon_ps *old_ps)
1251 {
1252         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1253         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1254
1255         if ((new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) !=
1256             (old_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2))
1257                 rv6xx_force_pcie_gen1(rdev);
1258 }
1259
1260 static void rv6xx_enable_dynamic_voltage_control(struct radeon_device *rdev,
1261                                                  bool enable)
1262 {
1263         if (enable)
1264                 WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
1265         else
1266                 WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
1267 }
1268
1269 static void rv6xx_enable_dynamic_backbias_control(struct radeon_device *rdev,
1270                                                   bool enable)
1271 {
1272         if (enable)
1273                 WREG32_P(GENERAL_PWRMGT, BACKBIAS_DPM_CNTL, ~BACKBIAS_DPM_CNTL);
1274         else
1275                 WREG32_P(GENERAL_PWRMGT, 0, ~BACKBIAS_DPM_CNTL);
1276 }
1277
1278 static int rv6xx_step_sw_voltage(struct radeon_device *rdev,
1279                                  u16 initial_voltage,
1280                                  u16 target_voltage)
1281 {
1282         u16 current_voltage;
1283         u16 true_target_voltage;
1284         u16 voltage_step;
1285         int signed_voltage_step;
1286
1287         if ((radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
1288                                           &voltage_step)) ||
1289             (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
1290                                                initial_voltage, &current_voltage)) ||
1291             (radeon_atom_round_to_true_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC,
1292                                                target_voltage, &true_target_voltage)))
1293                 return -EINVAL;
1294
1295         if (true_target_voltage < current_voltage)
1296                 signed_voltage_step = -(int)voltage_step;
1297         else
1298                 signed_voltage_step = voltage_step;
1299
1300         while (current_voltage != true_target_voltage) {
1301                 current_voltage += signed_voltage_step;
1302                 rv6xx_program_voltage_stepping_entry(rdev, R600_POWER_LEVEL_CTXSW,
1303                                                      current_voltage);
1304                 msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000);
1305         }
1306
1307         return 0;
1308 }
1309
1310 static int rv6xx_step_voltage_if_increasing(struct radeon_device *rdev,
1311                                             struct radeon_ps *new_ps,
1312                                             struct radeon_ps *old_ps)
1313 {
1314         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1315         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1316
1317         if (new_state->low.vddc > old_state->low.vddc)
1318                 return rv6xx_step_sw_voltage(rdev,
1319                                              old_state->low.vddc,
1320                                              new_state->low.vddc);
1321
1322         return 0;
1323 }
1324
1325 static int rv6xx_step_voltage_if_decreasing(struct radeon_device *rdev,
1326                                             struct radeon_ps *new_ps,
1327                                             struct radeon_ps *old_ps)
1328 {
1329         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1330         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1331
1332         if (new_state->low.vddc < old_state->low.vddc)
1333                 return rv6xx_step_sw_voltage(rdev,
1334                                              old_state->low.vddc,
1335                                              new_state->low.vddc);
1336         else
1337                 return 0;
1338 }
1339
1340 static void rv6xx_enable_high(struct radeon_device *rdev)
1341 {
1342         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1343
1344         if ((pi->restricted_levels < 1) ||
1345             (pi->restricted_levels == 3))
1346                 r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true);
1347 }
1348
1349 static void rv6xx_enable_medium(struct radeon_device *rdev)
1350 {
1351         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1352
1353         if (pi->restricted_levels < 2)
1354                 r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
1355 }
1356
1357 static void rv6xx_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
1358 {
1359         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1360         bool want_thermal_protection;
1361         enum radeon_dpm_event_src dpm_event_src;
1362
1363         switch (sources) {
1364         case 0:
1365         default:
1366                 want_thermal_protection = false;
1367                 break;
1368         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
1369                 want_thermal_protection = true;
1370                 dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
1371                 break;
1372
1373         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1374                 want_thermal_protection = true;
1375                 dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
1376                 break;
1377
1378         case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1379               (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1380                 want_thermal_protection = true;
1381                 dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
1382                 break;
1383         }
1384
1385         if (want_thermal_protection) {
1386                 WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
1387                 if (pi->thermal_protection)
1388                         WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
1389         } else {
1390                 WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
1391         }
1392 }
1393
1394 static void rv6xx_enable_auto_throttle_source(struct radeon_device *rdev,
1395                                               enum radeon_dpm_auto_throttle_src source,
1396                                               bool enable)
1397 {
1398         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1399
1400         if (enable) {
1401                 if (!(pi->active_auto_throttle_sources & (1 << source))) {
1402                         pi->active_auto_throttle_sources |= 1 << source;
1403                         rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1404                 }
1405         } else {
1406                 if (pi->active_auto_throttle_sources & (1 << source)) {
1407                         pi->active_auto_throttle_sources &= ~(1 << source);
1408                         rv6xx_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1409                 }
1410         }
1411 }
1412
1413
1414 static void rv6xx_enable_thermal_protection(struct radeon_device *rdev,
1415                                             bool enable)
1416 {
1417         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1418
1419         if (pi->active_auto_throttle_sources)
1420                 r600_enable_thermal_protection(rdev, enable);
1421 }
1422
1423 static void rv6xx_generate_transition_stepping(struct radeon_device *rdev,
1424                                                struct radeon_ps *new_ps,
1425                                                struct radeon_ps *old_ps)
1426 {
1427         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1428         struct rv6xx_ps *old_state = rv6xx_get_ps(old_ps);
1429         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1430
1431         rv6xx_generate_steps(rdev,
1432                              old_state->low.sclk,
1433                              new_state->low.sclk,
1434                              0, &pi->hw.medium_sclk_index);
1435 }
1436
1437 static void rv6xx_generate_low_step(struct radeon_device *rdev,
1438                                     struct radeon_ps *new_ps)
1439 {
1440         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1441         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1442
1443         pi->hw.low_sclk_index = 0;
1444         rv6xx_generate_single_step(rdev,
1445                                    new_state->low.sclk,
1446                                    0);
1447 }
1448
1449 static void rv6xx_invalidate_intermediate_steps(struct radeon_device *rdev)
1450 {
1451         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1452
1453         rv6xx_invalidate_intermediate_steps_range(rdev, 0,
1454                                                   pi->hw.medium_sclk_index);
1455 }
1456
1457 static void rv6xx_generate_stepping_table(struct radeon_device *rdev,
1458                                           struct radeon_ps *new_ps)
1459 {
1460         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1461         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1462
1463         pi->hw.low_sclk_index = 0;
1464
1465         rv6xx_generate_steps(rdev,
1466                              new_state->low.sclk,
1467                              new_state->medium.sclk,
1468                              0,
1469                              &pi->hw.medium_sclk_index);
1470         rv6xx_generate_steps(rdev,
1471                              new_state->medium.sclk,
1472                              new_state->high.sclk,
1473                              pi->hw.medium_sclk_index,
1474                              &pi->hw.high_sclk_index);
1475 }
1476
1477 static void rv6xx_enable_spread_spectrum(struct radeon_device *rdev,
1478                                          bool enable)
1479 {
1480         if (enable)
1481                 rv6xx_enable_dynamic_spread_spectrum(rdev, true);
1482         else {
1483                 rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_LOW, false);
1484                 rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_MEDIUM, false);
1485                 rv6xx_enable_engine_spread_spectrum(rdev, R600_POWER_LEVEL_HIGH, false);
1486                 rv6xx_enable_dynamic_spread_spectrum(rdev, false);
1487                 rv6xx_enable_memory_spread_spectrum(rdev, false);
1488         }
1489 }
1490
1491 static void rv6xx_reset_lvtm_data_sync(struct radeon_device *rdev)
1492 {
1493         if (ASIC_IS_DCE3(rdev))
1494                 WREG32_P(DCE3_LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG);
1495         else
1496                 WREG32_P(LVTMA_DATA_SYNCHRONIZATION, LVTMA_PFREQCHG, ~LVTMA_PFREQCHG);
1497 }
1498
1499 static void rv6xx_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
1500                                            struct radeon_ps *new_ps,
1501                                            bool enable)
1502 {
1503         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1504
1505         if (enable) {
1506                 rv6xx_enable_bif_dynamic_pcie_gen2(rdev, true);
1507                 rv6xx_enable_pcie_gen2_support(rdev);
1508                 r600_enable_dynamic_pcie_gen2(rdev, true);
1509         } else {
1510                 if (!(new_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2))
1511                         rv6xx_force_pcie_gen1(rdev);
1512                 rv6xx_enable_bif_dynamic_pcie_gen2(rdev, false);
1513                 r600_enable_dynamic_pcie_gen2(rdev, false);
1514         }
1515 }
1516
1517 static void rv6xx_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
1518                                                      struct radeon_ps *new_ps,
1519                                                      struct radeon_ps *old_ps)
1520 {
1521         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1522         struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps);
1523
1524         if ((new_ps->vclk == old_ps->vclk) &&
1525             (new_ps->dclk == old_ps->dclk))
1526                 return;
1527
1528         if (new_state->high.sclk >= current_state->high.sclk)
1529                 return;
1530
1531         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1532 }
1533
1534 static void rv6xx_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
1535                                                     struct radeon_ps *new_ps,
1536                                                     struct radeon_ps *old_ps)
1537 {
1538         struct rv6xx_ps *new_state = rv6xx_get_ps(new_ps);
1539         struct rv6xx_ps *current_state = rv6xx_get_ps(old_ps);
1540
1541         if ((new_ps->vclk == old_ps->vclk) &&
1542             (new_ps->dclk == old_ps->dclk))
1543                 return;
1544
1545         if (new_state->high.sclk < current_state->high.sclk)
1546                 return;
1547
1548         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1549 }
1550
1551 int rv6xx_dpm_enable(struct radeon_device *rdev)
1552 {
1553         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1554         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
1555
1556         if (r600_dynamicpm_enabled(rdev))
1557                 return -EINVAL;
1558
1559         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1560                 rv6xx_enable_backbias(rdev, true);
1561
1562         if (pi->dynamic_ss)
1563                 rv6xx_enable_spread_spectrum(rdev, true);
1564
1565         rv6xx_program_mpll_timing_parameters(rdev);
1566         rv6xx_program_bsp(rdev);
1567         rv6xx_program_git(rdev);
1568         rv6xx_program_tp(rdev);
1569         rv6xx_program_tpp(rdev);
1570         rv6xx_program_sstp(rdev);
1571         rv6xx_program_fcp(rdev);
1572         rv6xx_program_vddc3d_parameters(rdev);
1573         rv6xx_program_voltage_timing_parameters(rdev);
1574         rv6xx_program_engine_speed_parameters(rdev);
1575
1576         rv6xx_enable_display_gap(rdev, true);
1577         if (pi->display_gap == false)
1578                 rv6xx_enable_display_gap(rdev, false);
1579
1580         rv6xx_program_power_level_enter_state(rdev);
1581
1582         rv6xx_calculate_stepping_parameters(rdev, boot_ps);
1583
1584         if (pi->voltage_control)
1585                 rv6xx_program_voltage_gpio_pins(rdev);
1586
1587         rv6xx_generate_stepping_table(rdev, boot_ps);
1588
1589         rv6xx_program_stepping_parameters_except_lowest_entry(rdev);
1590         rv6xx_program_stepping_parameters_lowest_entry(rdev);
1591
1592         rv6xx_program_power_level_low(rdev);
1593         rv6xx_program_power_level_medium(rdev);
1594         rv6xx_program_power_level_high(rdev);
1595         rv6xx_program_vc(rdev);
1596         rv6xx_program_at(rdev);
1597
1598         r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
1599         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
1600         r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, true);
1601
1602         rv6xx_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
1603
1604         r600_start_dpm(rdev);
1605
1606         if (pi->voltage_control)
1607                 rv6xx_enable_static_voltage_control(rdev, boot_ps, false);
1608
1609         if (pi->dynamic_pcie_gen2)
1610                 rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, true);
1611
1612         if (pi->gfx_clock_gating)
1613                 r600_gfx_clockgating_enable(rdev, true);
1614
1615         return 0;
1616 }
1617
1618 void rv6xx_dpm_disable(struct radeon_device *rdev)
1619 {
1620         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1621         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
1622
1623         if (!r600_dynamicpm_enabled(rdev))
1624                 return;
1625
1626         r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
1627         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
1628         rv6xx_enable_display_gap(rdev, false);
1629         rv6xx_clear_vc(rdev);
1630         r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
1631
1632         if (pi->thermal_protection)
1633                 r600_enable_thermal_protection(rdev, false);
1634
1635         r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
1636         r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
1637         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
1638
1639         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1640                 rv6xx_enable_backbias(rdev, false);
1641
1642         rv6xx_enable_spread_spectrum(rdev, false);
1643
1644         if (pi->voltage_control)
1645                 rv6xx_enable_static_voltage_control(rdev, boot_ps, true);
1646
1647         if (pi->dynamic_pcie_gen2)
1648                 rv6xx_enable_dynamic_pcie_gen2(rdev, boot_ps, false);
1649
1650         if (rdev->irq.installed &&
1651             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1652                 rdev->irq.dpm_thermal = false;
1653                 radeon_irq_set(rdev);
1654         }
1655
1656         if (pi->gfx_clock_gating)
1657                 r600_gfx_clockgating_enable(rdev, false);
1658
1659         r600_stop_dpm(rdev);
1660 }
1661
1662 int rv6xx_dpm_set_power_state(struct radeon_device *rdev)
1663 {
1664         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
1665         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
1666         struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
1667         int ret;
1668
1669         pi->restricted_levels = 0;
1670
1671         rv6xx_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1672
1673         rv6xx_clear_vc(rdev);
1674         r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
1675         r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
1676
1677         if (pi->thermal_protection)
1678                 r600_enable_thermal_protection(rdev, false);
1679
1680         r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
1681         r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
1682         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
1683
1684         rv6xx_generate_transition_stepping(rdev, new_ps, old_ps);
1685         rv6xx_program_power_level_medium_for_transition(rdev);
1686
1687         if (pi->voltage_control) {
1688                 rv6xx_set_sw_voltage_to_safe(rdev, new_ps, old_ps);
1689                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1690                         rv6xx_set_sw_voltage_to_low(rdev, old_ps);
1691         }
1692
1693         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1694                 rv6xx_set_safe_backbias(rdev, new_ps, old_ps);
1695
1696         if (pi->dynamic_pcie_gen2)
1697                 rv6xx_set_safe_pcie_gen2(rdev, new_ps, old_ps);
1698
1699         if (pi->voltage_control)
1700                 rv6xx_enable_dynamic_voltage_control(rdev, false);
1701
1702         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1703                 rv6xx_enable_dynamic_backbias_control(rdev, false);
1704
1705         if (pi->voltage_control) {
1706                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1707                         rv6xx_step_voltage_if_increasing(rdev, new_ps, old_ps);
1708                 msleep((rdev->pm.dpm.voltage_response_time + 999) / 1000);
1709         }
1710
1711         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, true);
1712         r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false);
1713         r600_wait_for_power_level_unequal(rdev, R600_POWER_LEVEL_LOW);
1714
1715         rv6xx_generate_low_step(rdev, new_ps);
1716         rv6xx_invalidate_intermediate_steps(rdev);
1717         rv6xx_calculate_stepping_parameters(rdev, new_ps);
1718         rv6xx_program_stepping_parameters_lowest_entry(rdev);
1719         rv6xx_program_power_level_low_to_lowest_state(rdev);
1720
1721         r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
1722         r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
1723         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
1724
1725         if (pi->voltage_control) {
1726                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC) {
1727                         ret = rv6xx_step_voltage_if_decreasing(rdev, new_ps, old_ps);
1728                         if (ret)
1729                                 return ret;
1730                 }
1731                 rv6xx_enable_dynamic_voltage_control(rdev, true);
1732         }
1733
1734         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1735                 rv6xx_enable_dynamic_backbias_control(rdev, true);
1736
1737         if (pi->dynamic_pcie_gen2)
1738                 rv6xx_enable_dynamic_pcie_gen2(rdev, new_ps, true);
1739
1740         rv6xx_reset_lvtm_data_sync(rdev);
1741
1742         rv6xx_generate_stepping_table(rdev, new_ps);
1743         rv6xx_program_stepping_parameters_except_lowest_entry(rdev);
1744         rv6xx_program_power_level_low(rdev);
1745         rv6xx_program_power_level_medium(rdev);
1746         rv6xx_program_power_level_high(rdev);
1747         rv6xx_enable_medium(rdev);
1748         rv6xx_enable_high(rdev);
1749
1750         if (pi->thermal_protection)
1751                 rv6xx_enable_thermal_protection(rdev, true);
1752         rv6xx_program_vc(rdev);
1753         rv6xx_program_at(rdev);
1754
1755         rv6xx_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1756
1757         return 0;
1758 }
1759
1760 void rv6xx_setup_asic(struct radeon_device *rdev)
1761 {
1762         r600_enable_acpi_pm(rdev);
1763
1764         if (radeon_aspm != 0) {
1765                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
1766                         rv6xx_enable_l0s(rdev);
1767                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
1768                         rv6xx_enable_l1(rdev);
1769                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
1770                         rv6xx_enable_pll_sleep_in_l1(rdev);
1771         }
1772 }
1773
1774 void rv6xx_dpm_display_configuration_changed(struct radeon_device *rdev)
1775 {
1776         rv6xx_program_display_gap(rdev);
1777 }
1778
1779 union power_info {
1780         struct _ATOM_POWERPLAY_INFO info;
1781         struct _ATOM_POWERPLAY_INFO_V2 info_2;
1782         struct _ATOM_POWERPLAY_INFO_V3 info_3;
1783         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1784         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1785         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1786 };
1787
1788 union pplib_clock_info {
1789         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1790         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1791         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1792         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1793 };
1794
1795 union pplib_power_state {
1796         struct _ATOM_PPLIB_STATE v1;
1797         struct _ATOM_PPLIB_STATE_V2 v2;
1798 };
1799
1800 static void rv6xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
1801                                              struct radeon_ps *rps,
1802                                              struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info)
1803 {
1804         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1805         rps->class = le16_to_cpu(non_clock_info->usClassification);
1806         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1807
1808         if (r600_is_uvd_state(rps->class, rps->class2)) {
1809                 rps->vclk = RV6XX_DEFAULT_VCLK_FREQ;
1810                 rps->dclk = RV6XX_DEFAULT_DCLK_FREQ;
1811         } else {
1812                 rps->vclk = 0;
1813                 rps->dclk = 0;
1814         }
1815
1816         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
1817                 rdev->pm.dpm.boot_ps = rps;
1818         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1819                 rdev->pm.dpm.uvd_ps = rps;
1820 }
1821
1822 static void rv6xx_parse_pplib_clock_info(struct radeon_device *rdev,
1823                                          struct radeon_ps *rps, int index,
1824                                          union pplib_clock_info *clock_info)
1825 {
1826         struct rv6xx_ps *ps = rv6xx_get_ps(rps);
1827         u32 sclk, mclk;
1828         u16 vddc;
1829         struct rv6xx_pl *pl;
1830
1831         switch (index) {
1832         case 0:
1833                 pl = &ps->low;
1834                 break;
1835         case 1:
1836                 pl = &ps->medium;
1837                 break;
1838         case 2:
1839         default:
1840                 pl = &ps->high;
1841                 break;
1842         }
1843
1844         sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
1845         sclk |= clock_info->r600.ucEngineClockHigh << 16;
1846         mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
1847         mclk |= clock_info->r600.ucMemoryClockHigh << 16;
1848
1849         pl->mclk = mclk;
1850         pl->sclk = sclk;
1851         pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
1852         pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
1853
1854         /* patch up vddc if necessary */
1855         if (pl->vddc == 0xff01) {
1856                 if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc) == 0)
1857                         pl->vddc = vddc;
1858         }
1859
1860         /* fix up pcie gen2 */
1861         if (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) {
1862                 if ((rdev->family == CHIP_RV610) || (rdev->family == CHIP_RV630)) {
1863                         if (pl->vddc < 1100)
1864                                 pl->flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
1865                 }
1866         }
1867
1868         /* patch up boot state */
1869         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1870                 u16 vddc, vddci, mvdd;
1871                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
1872                 pl->mclk = rdev->clock.default_mclk;
1873                 pl->sclk = rdev->clock.default_sclk;
1874                 pl->vddc = vddc;
1875         }
1876 }
1877
1878 static int rv6xx_parse_power_table(struct radeon_device *rdev)
1879 {
1880         struct radeon_mode_info *mode_info = &rdev->mode_info;
1881         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1882         union pplib_power_state *power_state;
1883         int i, j;
1884         union pplib_clock_info *clock_info;
1885         union power_info *power_info;
1886         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1887         u16 data_offset;
1888         u8 frev, crev;
1889         struct rv6xx_ps *ps;
1890
1891         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1892                                    &frev, &crev, &data_offset))
1893                 return -EINVAL;
1894         power_info = (union power_info *)((uint8_t*)mode_info->atom_context->bios + data_offset);
1895
1896         rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
1897                                   power_info->pplib.ucNumStates, GFP_KERNEL);
1898         if (!rdev->pm.dpm.ps)
1899                 return -ENOMEM;
1900
1901         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
1902                 power_state = (union pplib_power_state *)
1903                         ((uint8_t*)mode_info->atom_context->bios + data_offset +
1904                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
1905                          i * power_info->pplib.ucStateEntrySize);
1906                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1907                         ((uint8_t*)mode_info->atom_context->bios + data_offset +
1908                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
1909                          (power_state->v1.ucNonClockStateIndex *
1910                           power_info->pplib.ucNonClockSize));
1911                 if (power_info->pplib.ucStateEntrySize - 1) {
1912                         u8 *idx;
1913                         ps = kzalloc(sizeof(struct rv6xx_ps), GFP_KERNEL);
1914                         if (ps == NULL) {
1915                                 kfree(rdev->pm.dpm.ps);
1916                                 return -ENOMEM;
1917                         }
1918                         rdev->pm.dpm.ps[i].ps_priv = ps;
1919                         rv6xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1920                                                          non_clock_info);
1921                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
1922                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
1923                                 clock_info = (union pplib_clock_info *)
1924                                         ((uint8_t*)mode_info->atom_context->bios + data_offset +
1925                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
1926                                          (idx[j] * power_info->pplib.ucClockInfoSize));
1927                                 rv6xx_parse_pplib_clock_info(rdev,
1928                                                              &rdev->pm.dpm.ps[i], j,
1929                                                              clock_info);
1930                         }
1931                 }
1932         }
1933         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
1934         return 0;
1935 }
1936
1937 int rv6xx_dpm_init(struct radeon_device *rdev)
1938 {
1939         struct radeon_atom_ss ss;
1940         struct atom_clock_dividers dividers;
1941         struct rv6xx_power_info *pi;
1942         int ret;
1943
1944         bzero(&dividers, sizeof(dividers));     /* avoid gcc warning */
1945         pi = kzalloc(sizeof(struct rv6xx_power_info), GFP_KERNEL);
1946         if (pi == NULL)
1947                 return -ENOMEM;
1948         rdev->pm.dpm.priv = pi;
1949
1950         ret = r600_get_platform_caps(rdev);
1951         if (ret)
1952                 return ret;
1953
1954         ret = rv6xx_parse_power_table(rdev);
1955         if (ret)
1956                 return ret;
1957
1958         if (rdev->pm.dpm.voltage_response_time == 0)
1959                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
1960         if (rdev->pm.dpm.backbias_response_time == 0)
1961                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
1962
1963         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1964                                              0, false, &dividers);
1965         if (ret)
1966                 pi->spll_ref_div = dividers.ref_div + 1;
1967         else
1968                 pi->spll_ref_div = R600_REFERENCEDIVIDER_DFLT;
1969
1970         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
1971                                              0, false, &dividers);
1972         if (ret)
1973                 pi->mpll_ref_div = dividers.ref_div + 1;
1974         else
1975                 pi->mpll_ref_div = R600_REFERENCEDIVIDER_DFLT;
1976
1977         if (rdev->family >= CHIP_RV670)
1978                 pi->fb_div_scale = 1;
1979         else
1980                 pi->fb_div_scale = 0;
1981
1982         pi->voltage_control =
1983                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
1984
1985         pi->gfx_clock_gating = true;
1986
1987         pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
1988                                                        ASIC_INTERNAL_ENGINE_SS, 0);
1989         pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
1990                                                        ASIC_INTERNAL_MEMORY_SS, 0);
1991
1992         /* Disable sclk ss, causes hangs on a lot of systems */
1993         pi->sclk_ss = false;
1994
1995         if (pi->sclk_ss || pi->mclk_ss)
1996                 pi->dynamic_ss = true;
1997         else
1998                 pi->dynamic_ss = false;
1999
2000         pi->dynamic_pcie_gen2 = true;
2001
2002         if (pi->gfx_clock_gating &&
2003             (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE))
2004                 pi->thermal_protection = true;
2005         else
2006                 pi->thermal_protection = false;
2007
2008         pi->display_gap = true;
2009
2010         return 0;
2011 }
2012
2013 void rv6xx_dpm_print_power_state(struct radeon_device *rdev,
2014                                  struct radeon_ps *rps)
2015 {
2016         struct rv6xx_ps *ps = rv6xx_get_ps(rps);
2017         struct rv6xx_pl *pl;
2018
2019         r600_dpm_print_class_info(rps->class, rps->class2);
2020         r600_dpm_print_cap_info(rps->caps);
2021         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2022         pl = &ps->low;
2023         printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
2024                pl->sclk, pl->mclk, pl->vddc);
2025         pl = &ps->medium;
2026         printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
2027                pl->sclk, pl->mclk, pl->vddc);
2028         pl = &ps->high;
2029         printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
2030                pl->sclk, pl->mclk, pl->vddc);
2031         r600_dpm_print_ps_status(rdev, rps);
2032 }
2033
2034 void rv6xx_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
2035                                                        struct seq_file *m)
2036 {
2037         struct radeon_ps *rps = rdev->pm.dpm.current_ps;
2038         struct rv6xx_ps *ps = rv6xx_get_ps(rps);
2039         struct rv6xx_pl *pl;
2040         u32 current_index =
2041                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
2042                 CURRENT_PROFILE_INDEX_SHIFT;
2043
2044         if (current_index > 2) {
2045                 seq_printf(m, "invalid dpm profile %d\n", current_index);
2046         } else {
2047                 if (current_index == 0)
2048                         pl = &ps->low;
2049                 else if (current_index == 1)
2050                         pl = &ps->medium;
2051                 else /* current_index == 2 */
2052                         pl = &ps->high;
2053                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2054                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
2055                            current_index, pl->sclk, pl->mclk, pl->vddc);
2056         }
2057 }
2058
2059 void rv6xx_dpm_fini(struct radeon_device *rdev)
2060 {
2061         int i;
2062
2063         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2064                 kfree(rdev->pm.dpm.ps[i].ps_priv);
2065         }
2066         kfree(rdev->pm.dpm.ps);
2067         kfree(rdev->pm.dpm.priv);
2068 }
2069
2070 u32 rv6xx_dpm_get_sclk(struct radeon_device *rdev, bool low)
2071 {
2072         struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps);
2073
2074         if (low)
2075                 return requested_state->low.sclk;
2076         else
2077                 return requested_state->high.sclk;
2078 }
2079
2080 u32 rv6xx_dpm_get_mclk(struct radeon_device *rdev, bool low)
2081 {
2082         struct rv6xx_ps *requested_state = rv6xx_get_ps(rdev->pm.dpm.requested_ps);
2083
2084         if (low)
2085                 return requested_state->low.mclk;
2086         else
2087                 return requested_state->high.mclk;
2088 }
2089
2090 int rv6xx_dpm_force_performance_level(struct radeon_device *rdev,
2091                                       enum radeon_dpm_forced_level level)
2092 {
2093         struct rv6xx_power_info *pi = rv6xx_get_pi(rdev);
2094
2095         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
2096                 pi->restricted_levels = 3;
2097         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
2098                 pi->restricted_levels = 2;
2099         } else {
2100                 pi->restricted_levels = 0;
2101         }
2102
2103         rv6xx_clear_vc(rdev);
2104         r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, true);
2105         r600_set_at(rdev, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF);
2106         r600_wait_for_power_level(rdev, R600_POWER_LEVEL_LOW);
2107         r600_power_level_enable(rdev, R600_POWER_LEVEL_HIGH, false);
2108         r600_power_level_enable(rdev, R600_POWER_LEVEL_MEDIUM, false);
2109         rv6xx_enable_medium(rdev);
2110         rv6xx_enable_high(rdev);
2111         if (pi->restricted_levels == 3)
2112                 r600_power_level_enable(rdev, R600_POWER_LEVEL_LOW, false);
2113         rv6xx_program_vc(rdev);
2114         rv6xx_program_at(rdev);
2115
2116         rdev->pm.dpm.forced_level = level;
2117
2118         return 0;
2119 }