sys/dev/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include "i915_drv.h"
  30 #include "intel_drv.h"
  31
  32 /**
  33  * DOC: runtime pm
  34  *
  35  * The i915 driver supports dynamic enabling and disabling of entire hardware
  36  * blocks at runtime. This is especially important on the display side where
  37  * software is supposed to control many power gates manually on recent hardware,
  38  * since on the GT side a lot of the power management is done by the hardware.
  39  * But even there some manual control at the device level is required.
  40  *
  41  * Since i915 supports a diverse set of platforms with a unified codebase and
  42  * hardware engineers just love to shuffle functionality around between power
  43  * domains there's a sizeable amount of indirection required. This file provides
  44  * generic functions to the driver for grabbing and releasing references for
  45  * abstract power domains. It then maps those to the actual power wells
  46  * present for a given platform.
  47  */
  48
  49 #define GEN9_ENABLE_DC5(dev) 0
  50 #define SKL_ENABLE_DC6(dev) IS_SKYLAKE(dev)
  51
  52 #define for_each_power_well(i, power_well, domain_mask, power_domains)  \
  53         for (i = 0;                                                     \
  54              i < (power_domains)->power_well_count &&                   \
  55                  ((power_well) = &(power_domains)->power_wells[i]);     \
  56              i++)                                                       \
  57                 if ((power_well)->domains & (domain_mask))
  58
  59 #define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
  60         for (i = (power_domains)->power_well_count - 1;                  \
  61              i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
  62              i--)                                                        \
  63                 if ((power_well)->domains & (domain_mask))
  64
  65 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  66                                     int power_well_id);
  67
  68 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
  69                                     struct i915_power_well *power_well)
  70 {
  71         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
  72         power_well->ops->enable(dev_priv, power_well);
  73         power_well->hw_enabled = true;
  74 }
  75
  76 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
  77                                      struct i915_power_well *power_well)
  78 {
  79         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
  80         power_well->hw_enabled = false;
  81         power_well->ops->disable(dev_priv, power_well);
  82 }
  83
  84 /*
  85  * We should only use the power well if we explicitly asked the hardware to
  86  * enable it, so check if it's enabled and also check if we've requested it to
  87  * be enabled.
  88  */
  89 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
  90                                    struct i915_power_well *power_well)
  91 {
  92         return I915_READ(HSW_PWR_WELL_DRIVER) ==
  93                      (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
  94 }
  95
  96 /**
  97  * __intel_display_power_is_enabled - unlocked check for a power domain
  98  * @dev_priv: i915 device instance
  99  * @domain: power domain to check
 100  *
 101  * This is the unlocked version of intel_display_power_is_enabled() and should
 102  * only be used from error capture and recovery code where deadlocks are
 103  * possible.
 104  *
 105  * Returns:
 106  * True when the power domain is enabled, false otherwise.
 107  */
 108 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 109                                       enum intel_display_power_domain domain)
 110 {
 111         struct i915_power_domains *power_domains;
 112         struct i915_power_well *power_well;
 113         bool is_enabled;
 114         int i;
 115
 116         if (dev_priv->pm.suspended)
 117                 return false;
 118
 119         power_domains = &dev_priv->power_domains;
 120
 121         is_enabled = true;
 122
 123         for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
 124                 if (power_well->always_on)
 125                         continue;
 126
 127                 if (!power_well->hw_enabled) {
 128                         is_enabled = false;
 129                         break;
 130                 }
 131         }
 132
 133         return is_enabled;
 134 }
 135
 136 /**
 137  * intel_display_power_is_enabled - check for a power domain
 138  * @dev_priv: i915 device instance
 139  * @domain: power domain to check
 140  *
 141  * This function can be used to check the hw power domain state. It is mostly
 142  * used in hardware state readout functions. Everywhere else code should rely
 143  * upon explicit power domain reference counting to ensure that the hardware
 144  * block is powered up before accessing it.
 145  *
 146  * Callers must hold the relevant modesetting locks to ensure that concurrent
 147  * threads can't disable the power well while the caller tries to read a few
 148  * registers.
 149  *
 150  * Returns:
 151  * True when the power domain is enabled, false otherwise.
 152  */
 153 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 154                                     enum intel_display_power_domain domain)
 155 {
 156         struct i915_power_domains *power_domains;
 157         bool ret;
 158
 159         power_domains = &dev_priv->power_domains;
 160
 161         mutex_lock(&power_domains->lock);
 162         ret = __intel_display_power_is_enabled(dev_priv, domain);
 163         mutex_unlock(&power_domains->lock);
 164
 165         return ret;
 166 }
 167
 168 /**
 169  * intel_display_set_init_power - set the initial power domain state
 170  * @dev_priv: i915 device instance
 171  * @enable: whether to enable or disable the initial power domain state
 172  *
 173  * For simplicity our driver load/unload and system suspend/resume code assumes
 174  * that all power domains are always enabled. This functions controls the state
 175  * of this little hack. While the initial power domain state is enabled runtime
 176  * pm is effectively disabled.
 177  */
 178 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 179                                   bool enable)
 180 {
 181         if (dev_priv->power_domains.init_power_on == enable)
 182                 return;
 183
 184         if (enable)
 185                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 186         else
 187                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 188
 189         dev_priv->power_domains.init_power_on = enable;
 190 }
 191
 192 /*
 193  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 194  * when not needed anymore. We have 4 registers that can request the power well
 195  * to be enabled, and it will only be disabled if none of the registers is
 196  * requesting it to be enabled.
 197  */
 198 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 199 {
 200         struct drm_device *dev = dev_priv->dev;
 201
 202         /*
 203          * After we re-enable the power well, if we touch VGA register 0x3d5
 204          * we'll get unclaimed register interrupts. This stops after we write
 205          * anything to the VGA MSR register. The vgacon module uses this
 206          * register all the time, so if we unbind our driver and, as a
 207          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 208          * console_unlock(). So make here we touch the VGA MSR register, making
 209          * sure vgacon can keep working normally without triggering interrupts
 210          * and error messages.
 211          */
 212 #if 0
 213         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
 214         outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 215         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
 216 #endif
 217
 218         if (IS_BROADWELL(dev))
 219                 gen8_irq_power_well_post_enable(dev_priv,
 220                                                 1 << PIPE_C | 1 << PIPE_B);
 221 }
 222
 223 static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 224                                        struct i915_power_well *power_well)
 225 {
 226         struct drm_device *dev = dev_priv->dev;
 227
 228         /*
 229          * After we re-enable the power well, if we touch VGA register 0x3d5
 230          * we'll get unclaimed register interrupts. This stops after we write
 231          * anything to the VGA MSR register. The vgacon module uses this
 232          * register all the time, so if we unbind our driver and, as a
 233          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 234          * console_unlock(). So make here we touch the VGA MSR register, making
 235          * sure vgacon can keep working normally without triggering interrupts
 236          * and error messages.
 237          */
 238         if (power_well->data == SKL_DISP_PW_2) {
 239 #if 0
 240                 vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
 241                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 242                 vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
 243 #endif
 244
 245                 gen8_irq_power_well_post_enable(dev_priv,
 246                                                 1 << PIPE_C | 1 << PIPE_B);
 247         }
 248
 249         if (power_well->data == SKL_DISP_PW_1) {
 250                 if (!dev_priv->power_domains.initializing)
 251                         intel_prepare_ddi(dev);
 252                 gen8_irq_power_well_post_enable(dev_priv, 1 << PIPE_A);
 253         }
 254 }
 255
 256 static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 257                                struct i915_power_well *power_well, bool enable)
 258 {
 259         bool is_enabled, enable_requested;
 260         uint32_t tmp;
 261
 262         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
 263         is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
 264         enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
 265
 266         if (enable) {
 267                 if (!enable_requested)
 268                         I915_WRITE(HSW_PWR_WELL_DRIVER,
 269                                    HSW_PWR_WELL_ENABLE_REQUEST);
 270
 271                 if (!is_enabled) {
 272                         DRM_DEBUG_KMS("Enabling power well\n");
 273                         if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
 274                                       HSW_PWR_WELL_STATE_ENABLED), 20))
 275                                 DRM_ERROR("Timeout enabling power well\n");
 276                         hsw_power_well_post_enable(dev_priv);
 277                 }
 278
 279         } else {
 280                 if (enable_requested) {
 281                         I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
 282                         POSTING_READ(HSW_PWR_WELL_DRIVER);
 283                         DRM_DEBUG_KMS("Requesting to disable the power well\n");
 284                 }
 285         }
 286 }
 287
 288 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
 289         BIT(POWER_DOMAIN_TRANSCODER_A) |                \
 290         BIT(POWER_DOMAIN_PIPE_B) |                      \
 291         BIT(POWER_DOMAIN_TRANSCODER_B) |                \
 292         BIT(POWER_DOMAIN_PIPE_C) |                      \
 293         BIT(POWER_DOMAIN_TRANSCODER_C) |                \
 294         BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |         \
 295         BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |         \
 296         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
 297         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
 298         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
 299         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
 300         BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
 301         BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
 302         BIT(POWER_DOMAIN_PORT_DDI_E_2_LANES) |          \
 303         BIT(POWER_DOMAIN_AUX_B) |                       \
 304         BIT(POWER_DOMAIN_AUX_C) |                       \
 305         BIT(POWER_DOMAIN_AUX_D) |                       \
 306         BIT(POWER_DOMAIN_AUDIO) |                       \
 307         BIT(POWER_DOMAIN_VGA) |                         \
 308         BIT(POWER_DOMAIN_INIT))
 309 #define SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS (         \
 310         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
 311         BIT(POWER_DOMAIN_PLLS) |                        \
 312         BIT(POWER_DOMAIN_PIPE_A) |                      \
 313         BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
 314         BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |         \
 315         BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
 316         BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
 317         BIT(POWER_DOMAIN_AUX_A) |                       \
 318         BIT(POWER_DOMAIN_INIT))
 319 #define SKL_DISPLAY_DDI_A_E_POWER_DOMAINS (             \
 320         BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
 321         BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
 322         BIT(POWER_DOMAIN_PORT_DDI_E_2_LANES) |          \
 323         BIT(POWER_DOMAIN_INIT))
 324 #define SKL_DISPLAY_DDI_B_POWER_DOMAINS (               \
 325         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
 326         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
 327         BIT(POWER_DOMAIN_INIT))
 328 #define SKL_DISPLAY_DDI_C_POWER_DOMAINS (               \
 329         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
 330         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
 331         BIT(POWER_DOMAIN_INIT))
 332 #define SKL_DISPLAY_DDI_D_POWER_DOMAINS (               \
 333         BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
 334         BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
 335         BIT(POWER_DOMAIN_INIT))
 336 #define SKL_DISPLAY_MISC_IO_POWER_DOMAINS (             \
 337         SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS |         \
 338         BIT(POWER_DOMAIN_PLLS) |                        \
 339         BIT(POWER_DOMAIN_INIT))
 340 #define SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS (           \
 341         (POWER_DOMAIN_MASK & ~(SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS |  \
 342         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
 343         SKL_DISPLAY_DDI_A_E_POWER_DOMAINS |             \
 344         SKL_DISPLAY_DDI_B_POWER_DOMAINS |               \
 345         SKL_DISPLAY_DDI_C_POWER_DOMAINS |               \
 346         SKL_DISPLAY_DDI_D_POWER_DOMAINS |               \
 347         SKL_DISPLAY_MISC_IO_POWER_DOMAINS)) |           \
 348         BIT(POWER_DOMAIN_INIT))
 349
 350 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
 351         BIT(POWER_DOMAIN_TRANSCODER_A) |                \
 352         BIT(POWER_DOMAIN_PIPE_B) |                      \
 353         BIT(POWER_DOMAIN_TRANSCODER_B) |                \
 354         BIT(POWER_DOMAIN_PIPE_C) |                      \
 355         BIT(POWER_DOMAIN_TRANSCODER_C) |                \
 356         BIT(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |         \
 357         BIT(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |         \
 358         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
 359         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
 360         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
 361         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
 362         BIT(POWER_DOMAIN_AUX_B) |                       \
 363         BIT(POWER_DOMAIN_AUX_C) |                       \
 364         BIT(POWER_DOMAIN_AUDIO) |                       \
 365         BIT(POWER_DOMAIN_VGA) |                         \
 366         BIT(POWER_DOMAIN_INIT))
 367 #define BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS (         \
 368         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
 369         BIT(POWER_DOMAIN_PIPE_A) |                      \
 370         BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
 371         BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |         \
 372         BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
 373         BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
 374         BIT(POWER_DOMAIN_AUX_A) |                       \
 375         BIT(POWER_DOMAIN_PLLS) |                        \
 376         BIT(POWER_DOMAIN_INIT))
 377 #define BXT_DISPLAY_ALWAYS_ON_POWER_DOMAINS (           \
 378         (POWER_DOMAIN_MASK & ~(BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS |  \
 379         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS)) |       \
 380         BIT(POWER_DOMAIN_INIT))
 381
 382 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 383 {
 384         struct drm_device *dev = dev_priv->dev;
 385
 386         WARN(!IS_BROXTON(dev), "Platform doesn't support DC9.\n");
 387         WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 388                 "DC9 already programmed to be enabled.\n");
 389         WARN(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 390                 "DC5 still not disabled to enable DC9.\n");
 391         WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on.\n");
 392         WARN(intel_irqs_enabled(dev_priv), "Interrupts not disabled yet.\n");
 393
 394          /*
 395           * TODO: check for the following to verify the conditions to enter DC9
 396           * state are satisfied:
 397           * 1] Check relevant display engine registers to verify if mode set
 398           * disable sequence was followed.
 399           * 2] Check if display uninitialize sequence is initialized.
 400           */
 401 }
 402
 403 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 404 {
 405         WARN(intel_irqs_enabled(dev_priv), "Interrupts not disabled yet.\n");
 406         WARN(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 407                 "DC9 already programmed to be disabled.\n");
 408         WARN(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 409                 "DC5 still not disabled.\n");
 410
 411          /*
 412           * TODO: check for the following to verify DC9 state was indeed
 413           * entered before programming to disable it:
 414           * 1] Check relevant display engine registers to verify if mode
 415           *  set disable sequence was followed.
 416           * 2] Check if display uninitialize sequence is initialized.
 417           */
 418 }
 419
 420 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 421 {
 422         uint32_t val;
 423
 424         assert_can_enable_dc9(dev_priv);
 425
 426         DRM_DEBUG_KMS("Enabling DC9\n");
 427
 428         val = I915_READ(DC_STATE_EN);
 429         val |= DC_STATE_EN_DC9;
 430         I915_WRITE(DC_STATE_EN, val);
 431         POSTING_READ(DC_STATE_EN);
 432 }
 433
 434 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 435 {
 436         uint32_t val;
 437
 438         assert_can_disable_dc9(dev_priv);
 439
 440         DRM_DEBUG_KMS("Disabling DC9\n");
 441
 442         val = I915_READ(DC_STATE_EN);
 443         val &= ~DC_STATE_EN_DC9;
 444         I915_WRITE(DC_STATE_EN, val);
 445         POSTING_READ(DC_STATE_EN);
 446 }
 447
 448 static void gen9_set_dc_state_debugmask_memory_up(
 449                         struct drm_i915_private *dev_priv)
 450 {
 451         uint32_t val;
 452
 453         /* The below bit doesn't need to be cleared ever afterwards */
 454         val = I915_READ(DC_STATE_DEBUG);
 455         if (!(val & DC_STATE_DEBUG_MASK_MEMORY_UP)) {
 456                 val |= DC_STATE_DEBUG_MASK_MEMORY_UP;
 457                 I915_WRITE(DC_STATE_DEBUG, val);
 458                 POSTING_READ(DC_STATE_DEBUG);
 459         }
 460 }
 461
 462 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 463 {
 464         struct drm_device *dev = dev_priv->dev;
 465         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 466                                         SKL_DISP_PW_2);
 467
 468         WARN(!IS_SKYLAKE(dev), "Platform doesn't support DC5.\n");
 469         WARN(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n");
 470         WARN(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 471
 472         WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 473                                 "DC5 already programmed to be enabled.\n");
 474         WARN(dev_priv->pm.suspended,
 475                 "DC5 cannot be enabled, if platform is runtime-suspended.\n");
 476
 477         assert_csr_loaded(dev_priv);
 478 }
 479
 480 static void assert_can_disable_dc5(struct drm_i915_private *dev_priv)
 481 {
 482         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 483                                         SKL_DISP_PW_2);
 484         /*
 485          * During initialization, the firmware may not be loaded yet.
 486          * We still want to make sure that the DC enabling flag is cleared.
 487          */
 488         if (dev_priv->power_domains.initializing)
 489                 return;
 490
 491         WARN(!pg2_enabled, "PG2 not enabled to disable DC5.\n");
 492         WARN(dev_priv->pm.suspended,
 493                 "Disabling of DC5 while platform is runtime-suspended should never happen.\n");
 494 }
 495
 496 static void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 497 {
 498         uint32_t val;
 499
 500         assert_can_enable_dc5(dev_priv);
 501
 502         DRM_DEBUG_KMS("Enabling DC5\n");
 503
 504         gen9_set_dc_state_debugmask_memory_up(dev_priv);
 505
 506         val = I915_READ(DC_STATE_EN);
 507         val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK;
 508         val |= DC_STATE_EN_UPTO_DC5;
 509         I915_WRITE(DC_STATE_EN, val);
 510         POSTING_READ(DC_STATE_EN);
 511 }
 512
 513 static void gen9_disable_dc5(struct drm_i915_private *dev_priv)
 514 {
 515         uint32_t val;
 516
 517         assert_can_disable_dc5(dev_priv);
 518
 519         DRM_DEBUG_KMS("Disabling DC5\n");
 520
 521         val = I915_READ(DC_STATE_EN);
 522         val &= ~DC_STATE_EN_UPTO_DC5;
 523         I915_WRITE(DC_STATE_EN, val);
 524         POSTING_READ(DC_STATE_EN);
 525 }
 526
 527 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 528 {
 529         struct drm_device *dev = dev_priv->dev;
 530
 531         WARN(!IS_SKYLAKE(dev), "Platform doesn't support DC6.\n");
 532         WARN(!HAS_RUNTIME_PM(dev), "Runtime PM not enabled.\n");
 533         WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 534                 "Backlight is not disabled.\n");
 535         WARN((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 536                 "DC6 already programmed to be enabled.\n");
 537
 538         assert_csr_loaded(dev_priv);
 539 }
 540
 541 static void assert_can_disable_dc6(struct drm_i915_private *dev_priv)
 542 {
 543         /*
 544          * During initialization, the firmware may not be loaded yet.
 545          * We still want to make sure that the DC enabling flag is cleared.
 546          */
 547         if (dev_priv->power_domains.initializing)
 548                 return;
 549
 550         assert_csr_loaded(dev_priv);
 551         WARN(!(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 552                 "DC6 already programmed to be disabled.\n");
 553 }
 554
 555 static void skl_enable_dc6(struct drm_i915_private *dev_priv)
 556 {
 557         uint32_t val;
 558
 559         assert_can_enable_dc6(dev_priv);
 560
 561         DRM_DEBUG_KMS("Enabling DC6\n");
 562
 563         gen9_set_dc_state_debugmask_memory_up(dev_priv);
 564
 565         val = I915_READ(DC_STATE_EN);
 566         val &= ~DC_STATE_EN_UPTO_DC5_DC6_MASK;
 567         val |= DC_STATE_EN_UPTO_DC6;
 568         I915_WRITE(DC_STATE_EN, val);
 569         POSTING_READ(DC_STATE_EN);
 570 }
 571
 572 static void skl_disable_dc6(struct drm_i915_private *dev_priv)
 573 {
 574         uint32_t val;
 575
 576         assert_can_disable_dc6(dev_priv);
 577
 578         DRM_DEBUG_KMS("Disabling DC6\n");
 579
 580         val = I915_READ(DC_STATE_EN);
 581         val &= ~DC_STATE_EN_UPTO_DC6;
 582         I915_WRITE(DC_STATE_EN, val);
 583         POSTING_READ(DC_STATE_EN);
 584 }
 585
 586 static void skl_set_power_well(struct drm_i915_private *dev_priv,
 587                         struct i915_power_well *power_well, bool enable)
 588 {
 589         struct drm_device *dev = dev_priv->dev;
 590         uint32_t tmp, fuse_status;
 591         uint32_t req_mask, state_mask;
 592         bool is_enabled, enable_requested, check_fuse_status = false;
 593
 594         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
 595         fuse_status = I915_READ(SKL_FUSE_STATUS);
 596
 597         switch (power_well->data) {
 598         case SKL_DISP_PW_1:
 599                 if (wait_for((I915_READ(SKL_FUSE_STATUS) &
 600                         SKL_FUSE_PG0_DIST_STATUS), 1)) {
 601                         DRM_ERROR("PG0 not enabled\n");
 602                         return;
 603                 }
 604                 break;
 605         case SKL_DISP_PW_2:
 606                 if (!(fuse_status & SKL_FUSE_PG1_DIST_STATUS)) {
 607                         DRM_ERROR("PG1 in disabled state\n");
 608                         return;
 609                 }
 610                 break;
 611         case SKL_DISP_PW_DDI_A_E:
 612         case SKL_DISP_PW_DDI_B:
 613         case SKL_DISP_PW_DDI_C:
 614         case SKL_DISP_PW_DDI_D:
 615         case SKL_DISP_PW_MISC_IO:
 616                 break;
 617         default:
 618                 WARN(1, "Unknown power well %lu\n", power_well->data);
 619                 return;
 620         }
 621
 622         req_mask = SKL_POWER_WELL_REQ(power_well->data);
 623         enable_requested = tmp & req_mask;
 624         state_mask = SKL_POWER_WELL_STATE(power_well->data);
 625         is_enabled = tmp & state_mask;
 626
 627         if (enable) {
 628                 if (!enable_requested) {
 629                         WARN((tmp & state_mask) &&
 630                                 !I915_READ(HSW_PWR_WELL_BIOS),
 631                                 "Invalid for power well status to be enabled, unless done by the BIOS, \
 632                                 when request is to disable!\n");
 633                         if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
 634                                 power_well->data == SKL_DISP_PW_2) {
 635                                 if (SKL_ENABLE_DC6(dev)) {
 636                                         skl_disable_dc6(dev_priv);
 637                                         /*
 638                                          * DDI buffer programming unnecessary during driver-load/resume
 639                                          * as it's already done during modeset initialization then.
 640                                          * It's also invalid here as encoder list is still uninitialized.
 641                                          */
 642                                         if (!dev_priv->power_domains.initializing)
 643                                                 intel_prepare_ddi(dev);
 644                                 } else {
 645                                         gen9_disable_dc5(dev_priv);
 646                                 }
 647                         }
 648                         I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
 649                 }
 650
 651                 if (!is_enabled) {
 652                         DRM_DEBUG_KMS("Enabling %s\n", power_well->name);
 653                         if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
 654                                 state_mask), 1))
 655                                 DRM_ERROR("%s enable timeout\n",
 656                                         power_well->name);
 657                         check_fuse_status = true;
 658                 }
 659         } else {
 660                 if (enable_requested) {
 661                         I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
 662                         POSTING_READ(HSW_PWR_WELL_DRIVER);
 663                         DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
 664
 665                         if ((GEN9_ENABLE_DC5(dev) || SKL_ENABLE_DC6(dev)) &&
 666                                 power_well->data == SKL_DISP_PW_2) {
 667                                 enum csr_state state;
 668                                 /* TODO: wait for a completion event or
 669                                  * similar here instead of busy
 670                                  * waiting using wait_for function.
 671                                  */
 672                                 wait_for((state = intel_csr_load_status_get(dev_priv)) !=
 673                                                 FW_UNINITIALIZED, 1000);
 674                                 if (state != FW_LOADED)
 675                                         DRM_ERROR("CSR firmware not ready (%d)\n",
 676                                                         state);
 677                                 else
 678                                         if (SKL_ENABLE_DC6(dev))
 679                                                 skl_enable_dc6(dev_priv);
 680                                         else
 681                                                 gen9_enable_dc5(dev_priv);
 682                         }
 683                 }
 684         }
 685
 686         if (check_fuse_status) {
 687                 if (power_well->data == SKL_DISP_PW_1) {
 688                         if (wait_for((I915_READ(SKL_FUSE_STATUS) &
 689                                 SKL_FUSE_PG1_DIST_STATUS), 1))
 690                                 DRM_ERROR("PG1 distributing status timeout\n");
 691                 } else if (power_well->data == SKL_DISP_PW_2) {
 692                         if (wait_for((I915_READ(SKL_FUSE_STATUS) &
 693                                 SKL_FUSE_PG2_DIST_STATUS), 1))
 694                                 DRM_ERROR("PG2 distributing status timeout\n");
 695                 }
 696         }
 697
 698         if (enable && !is_enabled)
 699                 skl_power_well_post_enable(dev_priv, power_well);
 700 }
 701
 702 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 703                                    struct i915_power_well *power_well)
 704 {
 705         hsw_set_power_well(dev_priv, power_well, power_well->count > 0);
 706
 707         /*
 708          * We're taking over the BIOS, so clear any requests made by it since
 709          * the driver is in charge now.
 710          */
 711         if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
 712                 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
 713 }
 714
 715 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 716                                   struct i915_power_well *power_well)
 717 {
 718         hsw_set_power_well(dev_priv, power_well, true);
 719 }
 720
 721 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 722                                    struct i915_power_well *power_well)
 723 {
 724         hsw_set_power_well(dev_priv, power_well, false);
 725 }
 726
 727 static bool skl_power_well_enabled(struct drm_i915_private *dev_priv,
 728                                         struct i915_power_well *power_well)
 729 {
 730         uint32_t mask = SKL_POWER_WELL_REQ(power_well->data) |
 731                 SKL_POWER_WELL_STATE(power_well->data);
 732
 733         return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask;
 734 }
 735
 736 static void skl_power_well_sync_hw(struct drm_i915_private *dev_priv,
 737                                 struct i915_power_well *power_well)
 738 {
 739         skl_set_power_well(dev_priv, power_well, power_well->count > 0);
 740
 741         /* Clear any request made by BIOS as driver is taking over */
 742         I915_WRITE(HSW_PWR_WELL_BIOS, 0);
 743 }
 744
 745 static void skl_power_well_enable(struct drm_i915_private *dev_priv,
 746                                 struct i915_power_well *power_well)
 747 {
 748         skl_set_power_well(dev_priv, power_well, true);
 749 }
 750
 751 static void skl_power_well_disable(struct drm_i915_private *dev_priv,
 752                                 struct i915_power_well *power_well)
 753 {
 754         skl_set_power_well(dev_priv, power_well, false);
 755 }
 756
 757 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 758                                            struct i915_power_well *power_well)
 759 {
 760 }
 761
 762 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 763                                              struct i915_power_well *power_well)
 764 {
 765         return true;
 766 }
 767
 768 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 769                                struct i915_power_well *power_well, bool enable)
 770 {
 771         enum punit_power_well power_well_id = power_well->data;
 772         u32 mask;
 773         u32 state;
 774         u32 ctrl;
 775
 776         mask = PUNIT_PWRGT_MASK(power_well_id);
 777         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 778                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 779
 780         mutex_lock(&dev_priv->rps.hw_lock);
 781
 782 #define COND \
 783         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 784
 785         if (COND)
 786                 goto out;
 787
 788         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 789         ctrl &= ~mask;
 790         ctrl |= state;
 791         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 792
 793         if (wait_for(COND, 100))
 794                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 795                           state,
 796                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 797
 798 #undef COND
 799
 800 out:
 801         mutex_unlock(&dev_priv->rps.hw_lock);
 802 }
 803
 804 static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
 805                                    struct i915_power_well *power_well)
 806 {
 807         vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
 808 }
 809
 810 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 811                                   struct i915_power_well *power_well)
 812 {
 813         vlv_set_power_well(dev_priv, power_well, true);
 814 }
 815
 816 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 817                                    struct i915_power_well *power_well)
 818 {
 819         vlv_set_power_well(dev_priv, power_well, false);
 820 }
 821
 822 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 823                                    struct i915_power_well *power_well)
 824 {
 825         int power_well_id = power_well->data;
 826         bool enabled = false;
 827         u32 mask;
 828         u32 state;
 829         u32 ctrl;
 830
 831         mask = PUNIT_PWRGT_MASK(power_well_id);
 832         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 833
 834         mutex_lock(&dev_priv->rps.hw_lock);
 835
 836         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 837         /*
 838          * We only ever set the power-on and power-gate states, anything
 839          * else is unexpected.
 840          */
 841         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 842                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 843         if (state == ctrl)
 844                 enabled = true;
 845
 846         /*
 847          * A transient state at this point would mean some unexpected party
 848          * is poking at the power controls too.
 849          */
 850         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 851         WARN_ON(ctrl != state);
 852
 853         mutex_unlock(&dev_priv->rps.hw_lock);
 854
 855         return enabled;
 856 }
 857
 858 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 859 {
 860
 861         lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE);
 862         valleyview_enable_display_irqs(dev_priv);
 863         lockmgr(&dev_priv->irq_lock, LK_RELEASE);
 864
 865         /*
 866          * During driver initialization/resume we can avoid restoring the
 867          * part of the HW/SW state that will be inited anyway explicitly.
 868          */
 869         if (dev_priv->power_domains.initializing)
 870                 return;
 871
 872         intel_hpd_init(dev_priv);
 873
 874         i915_redisable_vga_power_on(dev_priv->dev);
 875 }
 876
 877 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 878 {
 879         lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE);
 880         valleyview_disable_display_irqs(dev_priv);
 881         lockmgr(&dev_priv->irq_lock, LK_RELEASE);
 882
 883         vlv_power_sequencer_reset(dev_priv);
 884 }
 885
 886 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 887                                           struct i915_power_well *power_well)
 888 {
 889         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
 890
 891         vlv_set_power_well(dev_priv, power_well, true);
 892
 893         vlv_display_power_well_init(dev_priv);
 894 }
 895
 896 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 897                                            struct i915_power_well *power_well)
 898 {
 899         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
 900
 901         vlv_display_power_well_deinit(dev_priv);
 902
 903         vlv_set_power_well(dev_priv, power_well, false);
 904 }
 905
 906 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 907                                            struct i915_power_well *power_well)
 908 {
 909         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
 910
 911         /*
 912          * Enable the CRI clock source so we can get at the
 913          * display and the reference clock for VGA
 914          * hotplug / manual detection.
 915          */
 916         I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | DPLL_VGA_MODE_DIS |
 917                    DPLL_REF_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
 918         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 919
 920         vlv_set_power_well(dev_priv, power_well, true);
 921
 922         /*
 923          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 924          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 925          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 926          *   b. The other bits such as sfr settings / modesel may all
 927          *      be set to 0.
 928          *
 929          * This should only be done on init and resume from S3 with
 930          * both PLLs disabled, or we risk losing DPIO and PLL
 931          * synchronization.
 932          */
 933         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
 934 }
 935
 936 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 937                                             struct i915_power_well *power_well)
 938 {
 939         enum i915_pipe pipe;
 940
 941         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
 942
 943         for_each_pipe(dev_priv, pipe)
 944                 assert_pll_disabled(dev_priv, pipe);
 945
 946         /* Assert common reset */
 947         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
 948
 949         vlv_set_power_well(dev_priv, power_well, false);
 950 }
 951
 952 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 953                                            struct i915_power_well *power_well)
 954 {
 955         enum dpio_phy phy;
 956
 957         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
 958                      power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
 959
 960         /*
 961          * Enable the CRI clock source so we can get at the
 962          * display and the reference clock for VGA
 963          * hotplug / manual detection.
 964          */
 965         if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 966                 phy = DPIO_PHY0;
 967                 I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | DPLL_VGA_MODE_DIS |
 968                            DPLL_REF_CLK_ENABLE_VLV);
 969                 I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) | DPLL_VGA_MODE_DIS |
 970                            DPLL_REF_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
 971         } else {
 972                 phy = DPIO_PHY1;
 973                 I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) | DPLL_VGA_MODE_DIS |
 974                            DPLL_REF_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
 975         }
 976         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 977         vlv_set_power_well(dev_priv, power_well, true);
 978
 979         /* Poll for phypwrgood signal */
 980         if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1))
 981                 DRM_ERROR("Display PHY %d is not power up\n", phy);
 982
 983         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
 984         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
 985 }
 986
 987 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 988                                             struct i915_power_well *power_well)
 989 {
 990         enum dpio_phy phy;
 991
 992         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
 993                      power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
 994
 995         if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 996                 phy = DPIO_PHY0;
 997                 assert_pll_disabled(dev_priv, PIPE_A);
 998                 assert_pll_disabled(dev_priv, PIPE_B);
 999         } else {
1000                 phy = DPIO_PHY1;
1001                 assert_pll_disabled(dev_priv, PIPE_C);
1002         }
1003
1004         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1005         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1006
1007         vlv_set_power_well(dev_priv, power_well, false);
1008 }
1009
1010 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1011                                         struct i915_power_well *power_well)
1012 {
1013         enum i915_pipe pipe = power_well->data;
1014         bool enabled;
1015         u32 state, ctrl;
1016
1017         mutex_lock(&dev_priv->rps.hw_lock);
1018
1019         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1020         /*
1021          * We only ever set the power-on and power-gate states, anything
1022          * else is unexpected.
1023          */
1024         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1025         enabled = state == DP_SSS_PWR_ON(pipe);
1026
1027         /*
1028          * A transient state at this point would mean some unexpected party
1029          * is poking at the power controls too.
1030          */
1031         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1032         WARN_ON(ctrl << 16 != state);
1033
1034         mutex_unlock(&dev_priv->rps.hw_lock);
1035
1036         return enabled;
1037 }
1038
1039 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1040                                     struct i915_power_well *power_well,
1041                                     bool enable)
1042 {
1043         enum i915_pipe pipe = power_well->data;
1044         u32 state;
1045         u32 ctrl;
1046
1047         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1048
1049         mutex_lock(&dev_priv->rps.hw_lock);
1050
1051 #define COND \
1052         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1053
1054         if (COND)
1055                 goto out;
1056
1057         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1058         ctrl &= ~DP_SSC_MASK(pipe);
1059         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1060         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1061
1062         if (wait_for(COND, 100))
1063                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1064                           state,
1065                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1066
1067 #undef COND
1068
1069 out:
1070         mutex_unlock(&dev_priv->rps.hw_lock);
1071 }
1072
1073 static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
1074                                         struct i915_power_well *power_well)
1075 {
1076         WARN_ON_ONCE(power_well->data != PIPE_A);
1077
1078         chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0);
1079 }
1080
1081 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1082                                        struct i915_power_well *power_well)
1083 {
1084         WARN_ON_ONCE(power_well->data != PIPE_A);
1085
1086         chv_set_pipe_power_well(dev_priv, power_well, true);
1087
1088         vlv_display_power_well_init(dev_priv);
1089 }
1090
1091 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1092                                         struct i915_power_well *power_well)
1093 {
1094         WARN_ON_ONCE(power_well->data != PIPE_A);
1095
1096         vlv_display_power_well_deinit(dev_priv);
1097
1098         chv_set_pipe_power_well(dev_priv, power_well, false);
1099 }
1100
1101 /**
1102  * intel_display_power_get - grab a power domain reference
1103  * @dev_priv: i915 device instance
1104  * @domain: power domain to reference
1105  *
1106  * This function grabs a power domain reference for @domain and ensures that the
1107  * power domain and all its parents are powered up. Therefore users should only
1108  * grab a reference to the innermost power domain they need.
1109  *
1110  * Any power domain reference obtained by this function must have a symmetric
1111  * call to intel_display_power_put() to release the reference again.
1112  */
1113 void intel_display_power_get(struct drm_i915_private *dev_priv,
1114                              enum intel_display_power_domain domain)
1115 {
1116         struct i915_power_domains *power_domains;
1117         struct i915_power_well *power_well;
1118         int i;
1119
1120         intel_runtime_pm_get(dev_priv);
1121
1122         power_domains = &dev_priv->power_domains;
1123
1124         mutex_lock(&power_domains->lock);
1125
1126         for_each_power_well(i, power_well, BIT(domain), power_domains) {
1127                 if (!power_well->count++)
1128                         intel_power_well_enable(dev_priv, power_well);
1129         }
1130
1131         power_domains->domain_use_count[domain]++;
1132
1133         mutex_unlock(&power_domains->lock);
1134 }
1135
1136 /**
1137  * intel_display_power_put - release a power domain reference
1138  * @dev_priv: i915 device instance
1139  * @domain: power domain to reference
1140  *
1141  * This function drops the power domain reference obtained by
1142  * intel_display_power_get() and might power down the corresponding hardware
1143  * block right away if this is the last reference.
1144  */
1145 void intel_display_power_put(struct drm_i915_private *dev_priv,
1146                              enum intel_display_power_domain domain)
1147 {
1148         struct i915_power_domains *power_domains;
1149         struct i915_power_well *power_well;
1150         int i;
1151
1152         power_domains = &dev_priv->power_domains;
1153
1154         mutex_lock(&power_domains->lock);
1155
1156         WARN_ON(!power_domains->domain_use_count[domain]);
1157         power_domains->domain_use_count[domain]--;
1158
1159         for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
1160                 WARN_ON(!power_well->count);
1161
1162                 if (!--power_well->count && i915.disable_power_well)
1163                         intel_power_well_disable(dev_priv, power_well);
1164         }
1165
1166         mutex_unlock(&power_domains->lock);
1167
1168         intel_runtime_pm_put(dev_priv);
1169 }
1170
1171 #define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
1172
1173 #define HSW_ALWAYS_ON_POWER_DOMAINS (                   \
1174         BIT(POWER_DOMAIN_PIPE_A) |                      \
1175         BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
1176         BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
1177         BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
1178         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
1179         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
1180         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
1181         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
1182         BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
1183         BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
1184         BIT(POWER_DOMAIN_PORT_CRT) |                    \
1185         BIT(POWER_DOMAIN_PLLS) |                        \
1186         BIT(POWER_DOMAIN_AUX_A) |                       \
1187         BIT(POWER_DOMAIN_AUX_B) |                       \
1188         BIT(POWER_DOMAIN_AUX_C) |                       \
1189         BIT(POWER_DOMAIN_AUX_D) |                       \
1190         BIT(POWER_DOMAIN_INIT))
1191 #define HSW_DISPLAY_POWER_DOMAINS (                             \
1192         (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) |    \
1193         BIT(POWER_DOMAIN_INIT))
1194
1195 #define BDW_ALWAYS_ON_POWER_DOMAINS (                   \
1196         HSW_ALWAYS_ON_POWER_DOMAINS |                   \
1197         BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
1198 #define BDW_DISPLAY_POWER_DOMAINS (                             \
1199         (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) |    \
1200         BIT(POWER_DOMAIN_INIT))
1201
1202 #define VLV_ALWAYS_ON_POWER_DOMAINS     BIT(POWER_DOMAIN_INIT)
1203 #define VLV_DISPLAY_POWER_DOMAINS       POWER_DOMAIN_MASK
1204
1205 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1206         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
1207         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
1208         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
1209         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
1210         BIT(POWER_DOMAIN_PORT_CRT) |            \
1211         BIT(POWER_DOMAIN_AUX_B) |               \
1212         BIT(POWER_DOMAIN_AUX_C) |               \
1213         BIT(POWER_DOMAIN_INIT))
1214
1215 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1216         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
1217         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
1218         BIT(POWER_DOMAIN_AUX_B) |               \
1219         BIT(POWER_DOMAIN_INIT))
1220
1221 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1222         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
1223         BIT(POWER_DOMAIN_AUX_B) |               \
1224         BIT(POWER_DOMAIN_INIT))
1225
1226 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1227         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
1228         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
1229         BIT(POWER_DOMAIN_AUX_C) |               \
1230         BIT(POWER_DOMAIN_INIT))
1231
1232 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1233         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
1234         BIT(POWER_DOMAIN_AUX_C) |               \
1235         BIT(POWER_DOMAIN_INIT))
1236
1237 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1238         BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
1239         BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
1240         BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
1241         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
1242         BIT(POWER_DOMAIN_AUX_B) |               \
1243         BIT(POWER_DOMAIN_AUX_C) |               \
1244         BIT(POWER_DOMAIN_INIT))
1245
1246 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1247         BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |  \
1248         BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
1249         BIT(POWER_DOMAIN_AUX_D) |               \
1250         BIT(POWER_DOMAIN_INIT))
1251
1252 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1253         .sync_hw = i9xx_always_on_power_well_noop,
1254         .enable = i9xx_always_on_power_well_noop,
1255         .disable = i9xx_always_on_power_well_noop,
1256         .is_enabled = i9xx_always_on_power_well_enabled,
1257 };
1258
1259 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1260         .sync_hw = chv_pipe_power_well_sync_hw,
1261         .enable = chv_pipe_power_well_enable,
1262         .disable = chv_pipe_power_well_disable,
1263         .is_enabled = chv_pipe_power_well_enabled,
1264 };
1265
1266 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1267         .sync_hw = vlv_power_well_sync_hw,
1268         .enable = chv_dpio_cmn_power_well_enable,
1269         .disable = chv_dpio_cmn_power_well_disable,
1270         .is_enabled = vlv_power_well_enabled,
1271 };
1272
1273 static struct i915_power_well i9xx_always_on_power_well[] = {
1274         {
1275                 .name = "always-on",
1276                 .always_on = 1,
1277                 .domains = POWER_DOMAIN_MASK,
1278                 .ops = &i9xx_always_on_power_well_ops,
1279         },
1280 };
1281
1282 static const struct i915_power_well_ops hsw_power_well_ops = {
1283         .sync_hw = hsw_power_well_sync_hw,
1284         .enable = hsw_power_well_enable,
1285         .disable = hsw_power_well_disable,
1286         .is_enabled = hsw_power_well_enabled,
1287 };
1288
1289 static const struct i915_power_well_ops skl_power_well_ops = {
1290         .sync_hw = skl_power_well_sync_hw,
1291         .enable = skl_power_well_enable,
1292         .disable = skl_power_well_disable,
1293         .is_enabled = skl_power_well_enabled,
1294 };
1295
1296 static struct i915_power_well hsw_power_wells[] = {
1297         {
1298                 .name = "always-on",
1299                 .always_on = 1,
1300                 .domains = HSW_ALWAYS_ON_POWER_DOMAINS,
1301                 .ops = &i9xx_always_on_power_well_ops,
1302         },
1303         {
1304                 .name = "display",
1305                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1306                 .ops = &hsw_power_well_ops,
1307         },
1308 };
1309
1310 static struct i915_power_well bdw_power_wells[] = {
1311         {
1312                 .name = "always-on",
1313                 .always_on = 1,
1314                 .domains = BDW_ALWAYS_ON_POWER_DOMAINS,
1315                 .ops = &i9xx_always_on_power_well_ops,
1316         },
1317         {
1318                 .name = "display",
1319                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1320                 .ops = &hsw_power_well_ops,
1321         },
1322 };
1323
1324 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1325         .sync_hw = vlv_power_well_sync_hw,
1326         .enable = vlv_display_power_well_enable,
1327         .disable = vlv_display_power_well_disable,
1328         .is_enabled = vlv_power_well_enabled,
1329 };
1330
1331 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1332         .sync_hw = vlv_power_well_sync_hw,
1333         .enable = vlv_dpio_cmn_power_well_enable,
1334         .disable = vlv_dpio_cmn_power_well_disable,
1335         .is_enabled = vlv_power_well_enabled,
1336 };
1337
1338 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1339         .sync_hw = vlv_power_well_sync_hw,
1340         .enable = vlv_power_well_enable,
1341         .disable = vlv_power_well_disable,
1342         .is_enabled = vlv_power_well_enabled,
1343 };
1344
1345 static struct i915_power_well vlv_power_wells[] = {
1346         {
1347                 .name = "always-on",
1348                 .always_on = 1,
1349                 .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
1350                 .ops = &i9xx_always_on_power_well_ops,
1351         },
1352         {
1353                 .name = "display",
1354                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1355                 .data = PUNIT_POWER_WELL_DISP2D,
1356                 .ops = &vlv_display_power_well_ops,
1357         },
1358         {
1359                 .name = "dpio-tx-b-01",
1360                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1361                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1362                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1363                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1364                 .ops = &vlv_dpio_power_well_ops,
1365                 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
1366         },
1367         {
1368                 .name = "dpio-tx-b-23",
1369                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1370                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1371                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1372                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1373                 .ops = &vlv_dpio_power_well_ops,
1374                 .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
1375         },
1376         {
1377                 .name = "dpio-tx-c-01",
1378                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1379                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1380                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1381                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1382                 .ops = &vlv_dpio_power_well_ops,
1383                 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
1384         },
1385         {
1386                 .name = "dpio-tx-c-23",
1387                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1388                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1389                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1390                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1391                 .ops = &vlv_dpio_power_well_ops,
1392                 .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
1393         },
1394         {
1395                 .name = "dpio-common",
1396                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
1397                 .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
1398                 .ops = &vlv_dpio_cmn_power_well_ops,
1399         },
1400 };
1401
1402 static struct i915_power_well chv_power_wells[] = {
1403         {
1404                 .name = "always-on",
1405                 .always_on = 1,
1406                 .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
1407                 .ops = &i9xx_always_on_power_well_ops,
1408         },
1409         {
1410                 .name = "display",
1411                 /*
1412                  * Pipe A power well is the new disp2d well. Pipe B and C
1413                  * power wells don't actually exist. Pipe A power well is
1414                  * required for any pipe to work.
1415                  */
1416                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1417                 .data = PIPE_A,
1418                 .ops = &chv_pipe_power_well_ops,
1419         },
1420         {
1421                 .name = "dpio-common-bc",
1422                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
1423                 .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
1424                 .ops = &chv_dpio_cmn_power_well_ops,
1425         },
1426         {
1427                 .name = "dpio-common-d",
1428                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
1429                 .data = PUNIT_POWER_WELL_DPIO_CMN_D,
1430                 .ops = &chv_dpio_cmn_power_well_ops,
1431         },
1432 };
1433
1434 static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
1435                                                  int power_well_id)
1436 {
1437         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1438         struct i915_power_well *power_well;
1439         int i;
1440
1441         for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
1442                 if (power_well->data == power_well_id)
1443                         return power_well;
1444         }
1445
1446         return NULL;
1447 }
1448
1449 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
1450                                     int power_well_id)
1451 {
1452         struct i915_power_well *power_well;
1453         bool ret;
1454
1455         power_well = lookup_power_well(dev_priv, power_well_id);
1456         ret = power_well->ops->is_enabled(dev_priv, power_well);
1457
1458         return ret;
1459 }
1460
1461 static struct i915_power_well skl_power_wells[] = {
1462         {
1463                 .name = "always-on",
1464                 .always_on = 1,
1465                 .domains = SKL_DISPLAY_ALWAYS_ON_POWER_DOMAINS,
1466                 .ops = &i9xx_always_on_power_well_ops,
1467         },
1468         {
1469                 .name = "power well 1",
1470                 .domains = SKL_DISPLAY_POWERWELL_1_POWER_DOMAINS,
1471                 .ops = &skl_power_well_ops,
1472                 .data = SKL_DISP_PW_1,
1473         },
1474         {
1475                 .name = "MISC IO power well",
1476                 .domains = SKL_DISPLAY_MISC_IO_POWER_DOMAINS,
1477                 .ops = &skl_power_well_ops,
1478                 .data = SKL_DISP_PW_MISC_IO,
1479         },
1480         {
1481                 .name = "power well 2",
1482                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
1483                 .ops = &skl_power_well_ops,
1484                 .data = SKL_DISP_PW_2,
1485         },
1486         {
1487                 .name = "DDI A/E power well",
1488                 .domains = SKL_DISPLAY_DDI_A_E_POWER_DOMAINS,
1489                 .ops = &skl_power_well_ops,
1490                 .data = SKL_DISP_PW_DDI_A_E,
1491         },
1492         {
1493                 .name = "DDI B power well",
1494                 .domains = SKL_DISPLAY_DDI_B_POWER_DOMAINS,
1495                 .ops = &skl_power_well_ops,
1496                 .data = SKL_DISP_PW_DDI_B,
1497         },
1498         {
1499                 .name = "DDI C power well",
1500                 .domains = SKL_DISPLAY_DDI_C_POWER_DOMAINS,
1501                 .ops = &skl_power_well_ops,
1502                 .data = SKL_DISP_PW_DDI_C,
1503         },
1504         {
1505                 .name = "DDI D power well",
1506                 .domains = SKL_DISPLAY_DDI_D_POWER_DOMAINS,
1507                 .ops = &skl_power_well_ops,
1508                 .data = SKL_DISP_PW_DDI_D,
1509         },
1510 };
1511
1512 static struct i915_power_well bxt_power_wells[] = {
1513         {
1514                 .name = "always-on",
1515                 .always_on = 1,
1516                 .domains = BXT_DISPLAY_ALWAYS_ON_POWER_DOMAINS,
1517                 .ops = &i9xx_always_on_power_well_ops,
1518         },
1519         {
1520                 .name = "power well 1",
1521                 .domains = BXT_DISPLAY_POWERWELL_1_POWER_DOMAINS,
1522                 .ops = &skl_power_well_ops,
1523                 .data = SKL_DISP_PW_1,
1524         },
1525         {
1526                 .name = "power well 2",
1527                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
1528                 .ops = &skl_power_well_ops,
1529                 .data = SKL_DISP_PW_2,
1530         }
1531 };
1532
1533 #define set_power_wells(power_domains, __power_wells) ({                \
1534         (power_domains)->power_wells = (__power_wells);                 \
1535         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
1536 })
1537
1538 /**
1539  * intel_power_domains_init - initializes the power domain structures
1540  * @dev_priv: i915 device instance
1541  *
1542  * Initializes the power domain structures for @dev_priv depending upon the
1543  * supported platform.
1544  */
1545 int intel_power_domains_init(struct drm_i915_private *dev_priv)
1546 {
1547         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1548
1549         lockinit(&power_domains->lock, "i915pl", 0, LK_CANRECURSE);
1550
1551         /*
1552          * The enabling order will be from lower to higher indexed wells,
1553          * the disabling order is reversed.
1554          */
1555         if (IS_HASWELL(dev_priv->dev)) {
1556                 set_power_wells(power_domains, hsw_power_wells);
1557         } else if (IS_BROADWELL(dev_priv->dev)) {
1558                 set_power_wells(power_domains, bdw_power_wells);
1559         } else if (IS_SKYLAKE(dev_priv->dev)) {
1560                 set_power_wells(power_domains, skl_power_wells);
1561         } else if (IS_BROXTON(dev_priv->dev)) {
1562                 set_power_wells(power_domains, bxt_power_wells);
1563         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1564                 set_power_wells(power_domains, chv_power_wells);
1565         } else if (IS_VALLEYVIEW(dev_priv->dev)) {
1566                 set_power_wells(power_domains, vlv_power_wells);
1567         } else {
1568                 set_power_wells(power_domains, i9xx_always_on_power_well);
1569         }
1570
1571         return 0;
1572 }
1573
1574 static void intel_runtime_pm_disable(struct drm_i915_private *dev_priv)
1575 {
1576 #if 0
1577         struct drm_device *dev = dev_priv->dev;
1578         struct device *device = &dev->pdev->dev;
1579
1580         if (!HAS_RUNTIME_PM(dev))
1581                 return;
1582
1583         if (!intel_enable_rc6(dev))
1584                 return;
1585
1586         /* Make sure we're not suspended first. */
1587         pm_runtime_get_sync(device);
1588         pm_runtime_disable(device);
1589 #endif
1590 }
1591
1592 /**
1593  * intel_power_domains_fini - finalizes the power domain structures
1594  * @dev_priv: i915 device instance
1595  *
1596  * Finalizes the power domain structures for @dev_priv depending upon the
1597  * supported platform. This function also disables runtime pm and ensures that
1598  * the device stays powered up so that the driver can be reloaded.
1599  */
1600 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
1601 {
1602         intel_runtime_pm_disable(dev_priv);
1603
1604         /* The i915.ko module is still not prepared to be loaded when
1605          * the power well is not enabled, so just enable it in case
1606          * we're going to unload/reload. */
1607         intel_display_set_init_power(dev_priv, true);
1608 }
1609
1610 static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
1611 {
1612         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1613         struct i915_power_well *power_well;
1614         int i;
1615
1616         mutex_lock(&power_domains->lock);
1617         for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
1618                 power_well->ops->sync_hw(dev_priv, power_well);
1619                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
1620                                                                      power_well);
1621         }
1622         mutex_unlock(&power_domains->lock);
1623 }
1624
1625 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
1626 {
1627         struct i915_power_well *cmn_bc =
1628                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1629         struct i915_power_well *cmn_d =
1630                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1631
1632         /*
1633          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
1634          * workaround never ever read DISPLAY_PHY_CONTROL, and
1635          * instead maintain a shadow copy ourselves. Use the actual
1636          * power well state to reconstruct the expected initial
1637          * value.
1638          */
1639         dev_priv->chv_phy_control =
1640                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1641                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1642                 PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY0, DPIO_CH0) |
1643                 PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY0, DPIO_CH1) |
1644                 PHY_CH_POWER_MODE(PHY_CH_SU_PSR, DPIO_PHY1, DPIO_CH0);
1645         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc))
1646                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1647         if (cmn_d->ops->is_enabled(dev_priv, cmn_d))
1648                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1649 }
1650
1651 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
1652 {
1653         struct i915_power_well *cmn =
1654                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1655         struct i915_power_well *disp2d =
1656                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
1657
1658         /* If the display might be already active skip this */
1659         if (cmn->ops->is_enabled(dev_priv, cmn) &&
1660             disp2d->ops->is_enabled(dev_priv, disp2d) &&
1661             I915_READ(DPIO_CTL) & DPIO_CMNRST)
1662                 return;
1663
1664         DRM_DEBUG_KMS("toggling display PHY side reset\n");
1665
1666         /* cmnlane needs DPLL registers */
1667         disp2d->ops->enable(dev_priv, disp2d);
1668
1669         /*
1670          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1671          * Need to assert and de-assert PHY SB reset by gating the
1672          * common lane power, then un-gating it.
1673          * Simply ungating isn't enough to reset the PHY enough to get
1674          * ports and lanes running.
1675          */
1676         cmn->ops->disable(dev_priv, cmn);
1677 }
1678
1679 /**
1680  * intel_power_domains_init_hw - initialize hardware power domain state
1681  * @dev_priv: i915 device instance
1682  *
1683  * This function initializes the hardware power domain state and enables all
1684  * power domains using intel_display_set_init_power().
1685  */
1686 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
1687 {
1688         struct drm_device *dev = dev_priv->dev;
1689         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1690
1691         power_domains->initializing = true;
1692
1693         if (IS_CHERRYVIEW(dev)) {
1694                 chv_phy_control_init(dev_priv);
1695         } else if (IS_VALLEYVIEW(dev)) {
1696                 mutex_lock(&power_domains->lock);
1697                 vlv_cmnlane_wa(dev_priv);
1698                 mutex_unlock(&power_domains->lock);
1699         }
1700
1701         /* For now, we need the power well to be always enabled. */
1702         intel_display_set_init_power(dev_priv, true);
1703         intel_power_domains_resume(dev_priv);
1704         power_domains->initializing = false;
1705 }
1706
1707 /**
1708  * intel_aux_display_runtime_get - grab an auxiliary power domain reference
1709  * @dev_priv: i915 device instance
1710  *
1711  * This function grabs a power domain reference for the auxiliary power domain
1712  * (for access to the GMBUS and DP AUX blocks) and ensures that it and all its
1713  * parents are powered up. Therefore users should only grab a reference to the
1714  * innermost power domain they need.
1715  *
1716  * Any power domain reference obtained by this function must have a symmetric
1717  * call to intel_aux_display_runtime_put() to release the reference again.
1718  */
1719 void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
1720 {
1721         intel_runtime_pm_get(dev_priv);
1722 }
1723
1724 /**
1725  * intel_aux_display_runtime_put - release an auxiliary power domain reference
1726  * @dev_priv: i915 device instance
1727  *
1728  * This function drops the auxiliary power domain reference obtained by
1729  * intel_aux_display_runtime_get() and might power down the corresponding
1730  * hardware block right away if this is the last reference.
1731  */
1732 void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
1733 {
1734         intel_runtime_pm_put(dev_priv);
1735 }
1736
1737 /**
1738  * intel_runtime_pm_get - grab a runtime pm reference
1739  * @dev_priv: i915 device instance
1740  *
1741  * This function grabs a device-level runtime pm reference (mostly used for GEM
1742  * code to ensure the GTT or GT is on) and ensures that it is powered up.
1743  *
1744  * Any runtime pm reference obtained by this function must have a symmetric
1745  * call to intel_runtime_pm_put() to release the reference again.
1746  */
1747 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
1748 {
1749         struct drm_device *dev = dev_priv->dev;
1750 #if 0
1751         struct device *device = &dev->pdev->dev;
1752 #endif
1753
1754         if (!HAS_RUNTIME_PM(dev))
1755                 return;
1756
1757 #if 0
1758         pm_runtime_get_sync(device);
1759 #endif
1760         WARN(dev_priv->pm.suspended, "Device still suspended.\n");
1761 }
1762
1763 /**
1764  * intel_runtime_pm_get_noresume - grab a runtime pm reference
1765  * @dev_priv: i915 device instance
1766  *
1767  * This function grabs a device-level runtime pm reference (mostly used for GEM
1768  * code to ensure the GTT or GT is on).
1769  *
1770  * It will _not_ power up the device but instead only check that it's powered
1771  * on.  Therefore it is only valid to call this functions from contexts where
1772  * the device is known to be powered up and where trying to power it up would
1773  * result in hilarity and deadlocks. That pretty much means only the system
1774  * suspend/resume code where this is used to grab runtime pm references for
1775  * delayed setup down in work items.
1776  *
1777  * Any runtime pm reference obtained by this function must have a symmetric
1778  * call to intel_runtime_pm_put() to release the reference again.
1779  */
1780 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
1781 {
1782         struct drm_device *dev = dev_priv->dev;
1783 #if 0
1784         struct device *device = &dev->pdev->dev;
1785 #endif
1786
1787         if (!HAS_RUNTIME_PM(dev))
1788                 return;
1789
1790         WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n");
1791 #if 0
1792         pm_runtime_get_noresume(device);
1793 #endif
1794 }
1795
1796 /**
1797  * intel_runtime_pm_put - release a runtime pm reference
1798  * @dev_priv: i915 device instance
1799  *
1800  * This function drops the device-level runtime pm reference obtained by
1801  * intel_runtime_pm_get() and might power down the corresponding
1802  * hardware block right away if this is the last reference.
1803  */
1804 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
1805 {
1806 #if 0
1807         struct drm_device *dev = dev_priv->dev;
1808         struct device *device = &dev->pdev->dev;
1809
1810         if (!HAS_RUNTIME_PM(dev))
1811                 return;
1812
1813         pm_runtime_mark_last_busy(device);
1814         pm_runtime_put_autosuspend(device);
1815 #endif
1816 }
1817
1818 /**
1819  * intel_runtime_pm_enable - enable runtime pm
1820  * @dev_priv: i915 device instance
1821  *
1822  * This function enables runtime pm at the end of the driver load sequence.
1823  *
1824  * Note that this function does currently not enable runtime pm for the
1825  * subordinate display power domains. That is only done on the first modeset
1826  * using intel_display_set_init_power().
1827  */
1828 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
1829 {
1830         struct drm_device *dev = dev_priv->dev;
1831 #if 0
1832         struct device *device = &dev->pdev->dev;
1833 #endif
1834
1835         if (!HAS_RUNTIME_PM(dev))
1836                 return;
1837
1838 #if 0
1839         pm_runtime_set_active(device);
1840 #endif
1841
1842         /*
1843          * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
1844          * requirement.
1845          */
1846         if (!intel_enable_rc6(dev)) {
1847                 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
1848                 return;
1849         }
1850
1851 #if 0
1852         pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
1853         pm_runtime_mark_last_busy(device);
1854         pm_runtime_use_autosuspend(device);
1855
1856         pm_runtime_put_autosuspend(device);
1857 #endif
1858 }
1859