sys/dev/drm/i915/i915_drv.c

   1 /* i915_drv.c -- i830,i845,i855,i865,i915 driver -*- linux-c -*-
   2  */
   3 /*
   4  *
   5  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the
  10  * "Software"), to deal in the Software without restriction, including
  11  * without limitation the rights to use, copy, modify, merge, publish,
  12  * distribute, sub license, and/or sell copies of the Software, and to
  13  * permit persons to whom the Software is furnished to do so, subject to
  14  * the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the
  17  * next paragraph) shall be included in all copies or substantial portions
  18  * of the Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  23  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  24  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  25  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  26  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  27  *
  28  */
  29
  30 #include <drm/drmP.h>
  31 #include <drm/i915_drm.h>
  32 #include "i915_drv.h"
  33 #include <drm/drm_pciids.h>
  34 #include "intel_drv.h"
  35
  36 #include <linux/module.h>
  37 #include <drm/drm_crtc_helper.h>
  38
  39 static struct drm_driver driver;
  40
  41 #define GEN_DEFAULT_PIPEOFFSETS \
  42         .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
  43                           PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \
  44         .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
  45                            TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \
  46         .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET }
  47
  48 #define GEN_CHV_PIPEOFFSETS \
  49         .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
  50                           CHV_PIPE_C_OFFSET }, \
  51         .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
  52                            CHV_TRANSCODER_C_OFFSET, }, \
  53         .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \
  54                              CHV_PALETTE_C_OFFSET }
  55
  56 #define CURSOR_OFFSETS \
  57         .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET }
  58
  59 #define IVB_CURSOR_OFFSETS \
  60         .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET }
  61
  62 static const struct intel_device_info intel_i830_info = {
  63         .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2,
  64         .has_overlay = 1, .overlay_needs_physical = 1,
  65         .ring_mask = RENDER_RING,
  66         GEN_DEFAULT_PIPEOFFSETS,
  67         CURSOR_OFFSETS,
  68 };
  69
  70 static const struct intel_device_info intel_845g_info = {
  71         .gen = 2, .num_pipes = 1,
  72         .has_overlay = 1, .overlay_needs_physical = 1,
  73         .ring_mask = RENDER_RING,
  74         GEN_DEFAULT_PIPEOFFSETS,
  75         CURSOR_OFFSETS,
  76 };
  77
  78 static const struct intel_device_info intel_i85x_info = {
  79         .gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2,
  80         .cursor_needs_physical = 1,
  81         .has_overlay = 1, .overlay_needs_physical = 1,
  82         .has_fbc = 1,
  83         .ring_mask = RENDER_RING,
  84         GEN_DEFAULT_PIPEOFFSETS,
  85         CURSOR_OFFSETS,
  86 };
  87
  88 static const struct intel_device_info intel_i865g_info = {
  89         .gen = 2, .num_pipes = 1,
  90         .has_overlay = 1, .overlay_needs_physical = 1,
  91         .ring_mask = RENDER_RING,
  92         GEN_DEFAULT_PIPEOFFSETS,
  93         CURSOR_OFFSETS,
  94 };
  95
  96 static const struct intel_device_info intel_i915g_info = {
  97         .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2,
  98         .has_overlay = 1, .overlay_needs_physical = 1,
  99         .ring_mask = RENDER_RING,
 100         GEN_DEFAULT_PIPEOFFSETS,
 101         CURSOR_OFFSETS,
 102 };
 103 static const struct intel_device_info intel_i915gm_info = {
 104         .gen = 3, .is_mobile = 1, .num_pipes = 2,
 105         .cursor_needs_physical = 1,
 106         .has_overlay = 1, .overlay_needs_physical = 1,
 107         .supports_tv = 1,
 108         .has_fbc = 1,
 109         .ring_mask = RENDER_RING,
 110         GEN_DEFAULT_PIPEOFFSETS,
 111         CURSOR_OFFSETS,
 112 };
 113 static const struct intel_device_info intel_i945g_info = {
 114         .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2,
 115         .has_overlay = 1, .overlay_needs_physical = 1,
 116         .ring_mask = RENDER_RING,
 117         GEN_DEFAULT_PIPEOFFSETS,
 118         CURSOR_OFFSETS,
 119 };
 120 static const struct intel_device_info intel_i945gm_info = {
 121         .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2,
 122         .has_hotplug = 1, .cursor_needs_physical = 1,
 123         .has_overlay = 1, .overlay_needs_physical = 1,
 124         .supports_tv = 1,
 125         .has_fbc = 1,
 126         .ring_mask = RENDER_RING,
 127         GEN_DEFAULT_PIPEOFFSETS,
 128         CURSOR_OFFSETS,
 129 };
 130
 131 static const struct intel_device_info intel_i965g_info = {
 132         .gen = 4, .is_broadwater = 1, .num_pipes = 2,
 133         .has_hotplug = 1,
 134         .has_overlay = 1,
 135         .ring_mask = RENDER_RING,
 136         GEN_DEFAULT_PIPEOFFSETS,
 137         CURSOR_OFFSETS,
 138 };
 139
 140 static const struct intel_device_info intel_i965gm_info = {
 141         .gen = 4, .is_crestline = 1, .num_pipes = 2,
 142         .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
 143         .has_overlay = 1,
 144         .supports_tv = 1,
 145         .ring_mask = RENDER_RING,
 146         GEN_DEFAULT_PIPEOFFSETS,
 147         CURSOR_OFFSETS,
 148 };
 149
 150 static const struct intel_device_info intel_g33_info = {
 151         .gen = 3, .is_g33 = 1, .num_pipes = 2,
 152         .need_gfx_hws = 1, .has_hotplug = 1,
 153         .has_overlay = 1,
 154         .ring_mask = RENDER_RING,
 155         GEN_DEFAULT_PIPEOFFSETS,
 156         CURSOR_OFFSETS,
 157 };
 158
 159 static const struct intel_device_info intel_g45_info = {
 160         .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2,
 161         .has_pipe_cxsr = 1, .has_hotplug = 1,
 162         .ring_mask = RENDER_RING | BSD_RING,
 163         GEN_DEFAULT_PIPEOFFSETS,
 164         CURSOR_OFFSETS,
 165 };
 166
 167 static const struct intel_device_info intel_gm45_info = {
 168         .gen = 4, .is_g4x = 1, .num_pipes = 2,
 169         .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
 170         .has_pipe_cxsr = 1, .has_hotplug = 1,
 171         .supports_tv = 1,
 172         .ring_mask = RENDER_RING | BSD_RING,
 173         GEN_DEFAULT_PIPEOFFSETS,
 174         CURSOR_OFFSETS,
 175 };
 176
 177 static const struct intel_device_info intel_pineview_info = {
 178         .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2,
 179         .need_gfx_hws = 1, .has_hotplug = 1,
 180         .has_overlay = 1,
 181         GEN_DEFAULT_PIPEOFFSETS,
 182         CURSOR_OFFSETS,
 183 };
 184
 185 static const struct intel_device_info intel_ironlake_d_info = {
 186         .gen = 5, .num_pipes = 2,
 187         .need_gfx_hws = 1, .has_hotplug = 1,
 188         .ring_mask = RENDER_RING | BSD_RING,
 189         GEN_DEFAULT_PIPEOFFSETS,
 190         CURSOR_OFFSETS,
 191 };
 192
 193 static const struct intel_device_info intel_ironlake_m_info = {
 194         .gen = 5, .is_mobile = 1, .num_pipes = 2,
 195         .need_gfx_hws = 1, .has_hotplug = 1,
 196         .has_fbc = 1,
 197         .ring_mask = RENDER_RING | BSD_RING,
 198         GEN_DEFAULT_PIPEOFFSETS,
 199         CURSOR_OFFSETS,
 200 };
 201
 202 static const struct intel_device_info intel_sandybridge_d_info = {
 203         .gen = 6, .num_pipes = 2,
 204         .need_gfx_hws = 1, .has_hotplug = 1,
 205         .has_fbc = 1,
 206         .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
 207         .has_llc = 1,
 208         GEN_DEFAULT_PIPEOFFSETS,
 209         CURSOR_OFFSETS,
 210 };
 211
 212 static const struct intel_device_info intel_sandybridge_m_info = {
 213         .gen = 6, .is_mobile = 1, .num_pipes = 2,
 214         .need_gfx_hws = 1, .has_hotplug = 1,
 215         .has_fbc = 1,
 216         .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
 217         .has_llc = 1,
 218         GEN_DEFAULT_PIPEOFFSETS,
 219         CURSOR_OFFSETS,
 220 };
 221
 222 #define GEN7_FEATURES  \
 223         .gen = 7, .num_pipes = 3, \
 224         .need_gfx_hws = 1, .has_hotplug = 1, \
 225         .has_fbc = 1, \
 226         .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
 227         .has_llc = 1
 228
 229 static const struct intel_device_info intel_ivybridge_d_info = {
 230         GEN7_FEATURES,
 231         .is_ivybridge = 1,
 232         GEN_DEFAULT_PIPEOFFSETS,
 233         IVB_CURSOR_OFFSETS,
 234 };
 235
 236 static const struct intel_device_info intel_ivybridge_m_info = {
 237         GEN7_FEATURES,
 238         .is_ivybridge = 1,
 239         .is_mobile = 1,
 240         GEN_DEFAULT_PIPEOFFSETS,
 241         IVB_CURSOR_OFFSETS,
 242 };
 243
 244 static const struct intel_device_info intel_ivybridge_q_info = {
 245         GEN7_FEATURES,
 246         .is_ivybridge = 1,
 247         .num_pipes = 0, /* legal, last one wins */
 248         GEN_DEFAULT_PIPEOFFSETS,
 249         IVB_CURSOR_OFFSETS,
 250 };
 251
 252 static const struct intel_device_info intel_valleyview_m_info = {
 253         GEN7_FEATURES,
 254         .is_mobile = 1,
 255         .num_pipes = 2,
 256         .is_valleyview = 1,
 257         .display_mmio_offset = VLV_DISPLAY_BASE,
 258         .has_fbc = 0, /* legal, last one wins */
 259         .has_llc = 0, /* legal, last one wins */
 260         GEN_DEFAULT_PIPEOFFSETS,
 261         CURSOR_OFFSETS,
 262 };
 263
 264 static const struct intel_device_info intel_valleyview_d_info = {
 265         GEN7_FEATURES,
 266         .num_pipes = 2,
 267         .is_valleyview = 1,
 268         .display_mmio_offset = VLV_DISPLAY_BASE,
 269         .has_fbc = 0, /* legal, last one wins */
 270         .has_llc = 0, /* legal, last one wins */
 271         GEN_DEFAULT_PIPEOFFSETS,
 272         CURSOR_OFFSETS,
 273 };
 274
 275 static const struct intel_device_info intel_haswell_d_info = {
 276         GEN7_FEATURES,
 277         .is_haswell = 1,
 278         .has_ddi = 1,
 279         .has_fpga_dbg = 1,
 280         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 281         GEN_DEFAULT_PIPEOFFSETS,
 282         IVB_CURSOR_OFFSETS,
 283 };
 284
 285 static const struct intel_device_info intel_haswell_m_info = {
 286         GEN7_FEATURES,
 287         .is_haswell = 1,
 288         .is_mobile = 1,
 289         .has_ddi = 1,
 290         .has_fpga_dbg = 1,
 291         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 292         GEN_DEFAULT_PIPEOFFSETS,
 293         IVB_CURSOR_OFFSETS,
 294 };
 295
 296 static const struct intel_device_info intel_broadwell_d_info = {
 297         .gen = 8, .num_pipes = 3,
 298         .need_gfx_hws = 1, .has_hotplug = 1,
 299         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 300         .has_llc = 1,
 301         .has_ddi = 1,
 302         .has_fpga_dbg = 1,
 303         .has_fbc = 1,
 304         GEN_DEFAULT_PIPEOFFSETS,
 305         IVB_CURSOR_OFFSETS,
 306 };
 307
 308 static const struct intel_device_info intel_broadwell_m_info = {
 309         .gen = 8, .is_mobile = 1, .num_pipes = 3,
 310         .need_gfx_hws = 1, .has_hotplug = 1,
 311         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 312         .has_llc = 1,
 313         .has_ddi = 1,
 314         .has_fpga_dbg = 1,
 315         .has_fbc = 1,
 316         GEN_DEFAULT_PIPEOFFSETS,
 317         IVB_CURSOR_OFFSETS,
 318 };
 319
 320 static const struct intel_device_info intel_broadwell_gt3d_info = {
 321         .gen = 8, .num_pipes = 3,
 322         .need_gfx_hws = 1, .has_hotplug = 1,
 323         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 324         .has_llc = 1,
 325         .has_ddi = 1,
 326         .has_fpga_dbg = 1,
 327         .has_fbc = 1,
 328         GEN_DEFAULT_PIPEOFFSETS,
 329         IVB_CURSOR_OFFSETS,
 330 };
 331
 332 static const struct intel_device_info intel_broadwell_gt3m_info = {
 333         .gen = 8, .is_mobile = 1, .num_pipes = 3,
 334         .need_gfx_hws = 1, .has_hotplug = 1,
 335         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 336         .has_llc = 1,
 337         .has_ddi = 1,
 338         .has_fpga_dbg = 1,
 339         .has_fbc = 1,
 340         GEN_DEFAULT_PIPEOFFSETS,
 341         IVB_CURSOR_OFFSETS,
 342 };
 343
 344 static const struct intel_device_info intel_cherryview_info = {
 345         .gen = 8, .num_pipes = 3,
 346         .need_gfx_hws = 1, .has_hotplug = 1,
 347         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 348         .is_valleyview = 1,
 349         .display_mmio_offset = VLV_DISPLAY_BASE,
 350         GEN_CHV_PIPEOFFSETS,
 351         CURSOR_OFFSETS,
 352 };
 353
 354 static const struct intel_device_info intel_skylake_info = {
 355         .is_skylake = 1,
 356         .gen = 9, .num_pipes = 3,
 357         .need_gfx_hws = 1, .has_hotplug = 1,
 358         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 359         .has_llc = 1,
 360         .has_ddi = 1,
 361         .has_fpga_dbg = 1,
 362         .has_fbc = 1,
 363         GEN_DEFAULT_PIPEOFFSETS,
 364         IVB_CURSOR_OFFSETS,
 365 };
 366
 367 static const struct intel_device_info intel_skylake_gt3_info = {
 368         .is_skylake = 1,
 369         .gen = 9, .num_pipes = 3,
 370         .need_gfx_hws = 1, .has_hotplug = 1,
 371         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 372         .has_llc = 1,
 373         .has_ddi = 1,
 374         .has_fpga_dbg = 1,
 375         .has_fbc = 1,
 376         GEN_DEFAULT_PIPEOFFSETS,
 377         IVB_CURSOR_OFFSETS,
 378 };
 379
 380 static const struct intel_device_info intel_broxton_info = {
 381         .is_preliminary = 1,
 382         .gen = 9,
 383         .need_gfx_hws = 1, .has_hotplug = 1,
 384         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 385         .num_pipes = 3,
 386         .has_ddi = 1,
 387         .has_fpga_dbg = 1,
 388         .has_fbc = 1,
 389         GEN_DEFAULT_PIPEOFFSETS,
 390         IVB_CURSOR_OFFSETS,
 391 };
 392
 393 /*
 394  * Make sure any device matches here are from most specific to most
 395  * general.  For example, since the Quanta match is based on the subsystem
 396  * and subvendor IDs, we need it to come before the more general IVB
 397  * PCI ID matches, otherwise we'll use the wrong info struct above.
 398  */
 399 #define INTEL_PCI_IDS \
 400         INTEL_I830_IDS(&intel_i830_info),       \
 401         INTEL_I845G_IDS(&intel_845g_info),      \
 402         INTEL_I85X_IDS(&intel_i85x_info),       \
 403         INTEL_I865G_IDS(&intel_i865g_info),     \
 404         INTEL_I915G_IDS(&intel_i915g_info),     \
 405         INTEL_I915GM_IDS(&intel_i915gm_info),   \
 406         INTEL_I945G_IDS(&intel_i945g_info),     \
 407         INTEL_I945GM_IDS(&intel_i945gm_info),   \
 408         INTEL_I965G_IDS(&intel_i965g_info),     \
 409         INTEL_G33_IDS(&intel_g33_info),         \
 410         INTEL_I965GM_IDS(&intel_i965gm_info),   \
 411         INTEL_GM45_IDS(&intel_gm45_info),       \
 412         INTEL_G45_IDS(&intel_g45_info),         \
 413         INTEL_PINEVIEW_IDS(&intel_pineview_info),       \
 414         INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),   \
 415         INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),   \
 416         INTEL_SNB_D_IDS(&intel_sandybridge_d_info),     \
 417         INTEL_SNB_M_IDS(&intel_sandybridge_m_info),     \
 418         INTEL_IVB_M_IDS(&intel_ivybridge_m_info),       \
 419         INTEL_IVB_D_IDS(&intel_ivybridge_d_info),       \
 420         INTEL_HSW_D_IDS(&intel_haswell_d_info), \
 421         INTEL_HSW_M_IDS(&intel_haswell_m_info), \
 422         INTEL_VLV_M_IDS(&intel_valleyview_m_info),      \
 423         INTEL_VLV_D_IDS(&intel_valleyview_d_info),      \
 424         INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),   \
 425         INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),   \
 426         INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), \
 427         INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), \
 428         INTEL_CHV_IDS(&intel_cherryview_info),  \
 429         INTEL_SKL_GT1_IDS(&intel_skylake_info), \
 430         INTEL_SKL_GT2_IDS(&intel_skylake_info), \
 431         INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info),     \
 432         INTEL_BXT_IDS(&intel_broxton_info)
 433
 434 static const struct pci_device_id pciidlist[] = {               /* aka */
 435         INTEL_PCI_IDS,
 436         {0, 0}
 437 };
 438
 439 #define PCI_VENDOR_INTEL        0x8086
 440
 441 static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
 442 {
 443         enum intel_pch ret = PCH_NOP;
 444
 445         /*
 446          * In a virtualized passthrough environment we can be in a
 447          * setup where the ISA bridge is not able to be passed through.
 448          * In this case, a south bridge can be emulated and we have to
 449          * make an educated guess as to which PCH is really there.
 450          */
 451
 452         if (IS_GEN5(dev)) {
 453                 ret = PCH_IBX;
 454                 DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
 455         } else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
 456                 ret = PCH_CPT;
 457                 DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
 458         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
 459                 ret = PCH_LPT;
 460                 DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
 461         } else if (IS_SKYLAKE(dev)) {
 462                 ret = PCH_SPT;
 463                 DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
 464         }
 465
 466         return ret;
 467 }
 468
 469 void intel_detect_pch(struct drm_device *dev)
 470 {
 471         struct drm_i915_private *dev_priv = dev->dev_private;
 472         struct device *pch = NULL;
 473         struct pci_devinfo *di;
 474
 475         /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
 476          * (which really amounts to a PCH but no South Display).
 477          */
 478         if (INTEL_INFO(dev)->num_pipes == 0) {
 479                 dev_priv->pch_type = PCH_NOP;
 480                 return;
 481         }
 482
 483         /* XXX The ISA bridge probe causes some old Core2 machines to hang */
 484         if (INTEL_INFO(dev)->gen < 5)
 485                 return;
 486
 487         /*
 488          * The reason to probe ISA bridge instead of Dev31:Fun0 is to
 489          * make graphics device passthrough work easy for VMM, that only
 490          * need to expose ISA bridge to let driver know the real hardware
 491          * underneath. This is a requirement from virtualization team.
 492          *
 493          * In some virtualized environments (e.g. XEN), there is irrelevant
 494          * ISA bridge in the system. To work reliably, we should scan trhough
 495          * all the ISA bridge devices and check for the first match, instead
 496          * of only checking the first one.
 497          */
 498         di = NULL;
 499
 500         while ((pch = pci_iterate_class(&di, PCIC_BRIDGE, PCIS_BRIDGE_ISA))) {
 501                 if (pci_get_vendor(pch) == PCI_VENDOR_INTEL) {
 502                         unsigned short id = pci_get_device(pch) & INTEL_PCH_DEVICE_ID_MASK;
 503                         dev_priv->pch_id = id;
 504
 505                         if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
 506                                 dev_priv->pch_type = PCH_IBX;
 507                                 DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
 508                                 WARN_ON(!IS_GEN5(dev));
 509                         } else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
 510                                 dev_priv->pch_type = PCH_CPT;
 511                                 DRM_DEBUG_KMS("Found CougarPoint PCH\n");
 512                                 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
 513                         } else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
 514                                 /* PantherPoint is CPT compatible */
 515                                 dev_priv->pch_type = PCH_CPT;
 516                                 DRM_DEBUG_KMS("Found PantherPoint PCH\n");
 517                                 WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
 518                         } else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
 519                                 dev_priv->pch_type = PCH_LPT;
 520                                 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
 521                                 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
 522                                 WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
 523                         } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
 524                                 dev_priv->pch_type = PCH_LPT;
 525                                 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
 526                                 WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
 527                                 WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
 528                         } else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
 529                                 dev_priv->pch_type = PCH_SPT;
 530                                 DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
 531                                 WARN_ON(!IS_SKYLAKE(dev));
 532                         } else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
 533                                 dev_priv->pch_type = PCH_SPT;
 534                                 DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
 535                                 WARN_ON(!IS_SKYLAKE(dev));
 536                         } else if (id == INTEL_PCH_P2X_DEVICE_ID_TYPE) {
 537                                 dev_priv->pch_type = intel_virt_detect_pch(dev);
 538                         } else
 539                                 continue;
 540
 541                         break;
 542                 }
 543         }
 544         if (!pch)
 545                 DRM_DEBUG_KMS("No PCH found.\n");
 546
 547 #if 0
 548         pci_dev_put(pch);
 549 #endif
 550 }
 551
 552 bool i915_semaphore_is_enabled(struct drm_device *dev)
 553 {
 554         if (INTEL_INFO(dev)->gen < 6)
 555                 return false;
 556
 557         if (i915.semaphores >= 0)
 558                 return i915.semaphores;
 559
 560         /* TODO: make semaphores and Execlists play nicely together */
 561         if (i915.enable_execlists)
 562                 return false;
 563
 564         /* Until we get further testing... */
 565         if (IS_GEN8(dev))
 566                 return false;
 567
 568 #ifdef CONFIG_INTEL_IOMMU
 569         /* Enable semaphores on SNB when IO remapping is off */
 570         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
 571                 return false;
 572 #endif
 573
 574         return true;
 575 }
 576
 577 #ifdef __DragonFly__
 578 #define IS_BUILTIN(blah)        0
 579 #endif
 580
 581 void i915_firmware_load_error_print(const char *fw_path, int err)
 582 {
 583         DRM_ERROR("failed to load firmware %s (%d)\n", fw_path, err);
 584
 585         /*
 586          * If the reason is not known assume -ENOENT since that's the most
 587          * usual failure mode.
 588          */
 589         if (!err)
 590                 err = -ENOENT;
 591
 592         if (!(IS_BUILTIN(CONFIG_DRM_I915) && err == -ENOENT))
 593                 return;
 594
 595         DRM_ERROR(
 596           "The driver is built-in, so to load the firmware you need to\n"
 597           "include it either in the kernel (see CONFIG_EXTRA_FIRMWARE) or\n"
 598           "in your initrd/initramfs image.\n");
 599 }
 600
 601 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
 602 {
 603         struct drm_device *dev = dev_priv->dev;
 604         struct drm_encoder *encoder;
 605
 606         drm_modeset_lock_all(dev);
 607         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 608                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
 609
 610                 if (intel_encoder->suspend)
 611                         intel_encoder->suspend(intel_encoder);
 612         }
 613         drm_modeset_unlock_all(dev);
 614 }
 615
 616 static int intel_suspend_complete(struct drm_i915_private *dev_priv);
 617 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
 618                               bool rpm_resume);
 619 static int skl_resume_prepare(struct drm_i915_private *dev_priv);
 620 static int bxt_resume_prepare(struct drm_i915_private *dev_priv);
 621
 622
 623 static int i915_drm_suspend(struct drm_device *dev)
 624 {
 625         struct drm_i915_private *dev_priv = dev->dev_private;
 626         pci_power_t opregion_target_state;
 627         int error;
 628
 629         /* ignore lid events during suspend */
 630         mutex_lock(&dev_priv->modeset_restore_lock);
 631         dev_priv->modeset_restore = MODESET_SUSPENDED;
 632         mutex_unlock(&dev_priv->modeset_restore_lock);
 633
 634         /* We do a lot of poking in a lot of registers, make sure they work
 635          * properly. */
 636         intel_display_set_init_power(dev_priv, true);
 637
 638         drm_kms_helper_poll_disable(dev);
 639
 640 #if 0
 641         pci_save_state(dev->pdev);
 642 #endif
 643
 644         error = i915_gem_suspend(dev);
 645         if (error) {
 646                 dev_err(dev->pdev->dev,
 647                         "GEM idle failed, resume might fail\n");
 648                 return error;
 649         }
 650
 651         intel_guc_suspend(dev);
 652
 653         intel_suspend_gt_powersave(dev);
 654
 655         /*
 656          * Disable CRTCs directly since we want to preserve sw state
 657          * for _thaw. Also, power gate the CRTC power wells.
 658          */
 659         drm_modeset_lock_all(dev);
 660         intel_display_suspend(dev);
 661         drm_modeset_unlock_all(dev);
 662
 663 #if 0
 664         intel_dp_mst_suspend(dev);
 665 #endif
 666
 667         intel_runtime_pm_disable_interrupts(dev_priv);
 668         intel_hpd_cancel_work(dev_priv);
 669
 670         intel_suspend_encoders(dev_priv);
 671
 672         intel_suspend_hw(dev);
 673
 674         i915_gem_suspend_gtt_mappings(dev);
 675
 676         i915_save_state(dev);
 677
 678         opregion_target_state = PCI_D3cold;
 679 #if IS_ENABLED(CONFIG_ACPI_SLEEP)
 680         if (acpi_target_system_state() < ACPI_STATE_S3)
 681                 opregion_target_state = PCI_D1;
 682 #endif
 683         intel_opregion_notify_adapter(dev, opregion_target_state);
 684
 685         intel_uncore_forcewake_reset(dev, false);
 686         intel_opregion_fini(dev);
 687
 688 #if 0
 689         intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
 690 #endif
 691
 692         dev_priv->suspend_count++;
 693
 694         intel_display_set_init_power(dev_priv, false);
 695
 696         return 0;
 697 }
 698
 699 static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
 700 {
 701         struct drm_i915_private *dev_priv = drm_dev->dev_private;
 702         int ret;
 703
 704         ret = intel_suspend_complete(dev_priv);
 705
 706         if (ret) {
 707                 DRM_ERROR("Suspend complete failed: %d\n", ret);
 708
 709                 return ret;
 710         }
 711
 712 #if 0
 713         pci_disable_device(drm_dev->pdev);
 714         /*
 715          * During hibernation on some platforms the BIOS may try to access
 716          * the device even though it's already in D3 and hang the machine. So
 717          * leave the device in D0 on those platforms and hope the BIOS will
 718          * power down the device properly. The issue was seen on multiple old
 719          * GENs with different BIOS vendors, so having an explicit blacklist
 720          * is inpractical; apply the workaround on everything pre GEN6. The
 721          * platforms where the issue was seen:
 722          * Lenovo Thinkpad X301, X61s, X60, T60, X41
 723          * Fujitsu FSC S7110
 724          * Acer Aspire 1830T
 725          */
 726         if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
 727                 pci_set_power_state(drm_dev->pdev, PCI_D3hot);
 728 #endif
 729
 730         return 0;
 731 }
 732
 733 int i915_suspend_switcheroo(device_t kdev)
 734 {
 735         struct drm_device *dev = device_get_softc(kdev);
 736         int error;
 737
 738         if (!dev || !dev->dev_private) {
 739                 DRM_ERROR("dev: %p\n", dev);
 740                 DRM_ERROR("DRM not initialized, aborting suspend.\n");
 741                 return -ENODEV;
 742         }
 743
 744 #if 0
 745         if (WARN_ON_ONCE(state.event != PM_EVENT_SUSPEND &&
 746                          state.event != PM_EVENT_FREEZE))
 747                 return -EINVAL;
 748 #endif
 749
 750         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 751                 return 0;
 752
 753         error = i915_drm_suspend(dev);
 754         if (error)
 755                 return error;
 756
 757         return i915_drm_suspend_late(dev, false);
 758 }
 759
 760 static int i915_drm_resume(struct drm_device *dev)
 761 {
 762         struct drm_i915_private *dev_priv = dev->dev_private;
 763
 764         mutex_lock(&dev->struct_mutex);
 765         i915_gem_restore_gtt_mappings(dev);
 766         mutex_unlock(&dev->struct_mutex);
 767
 768         i915_restore_state(dev);
 769         intel_opregion_setup(dev);
 770
 771         intel_init_pch_refclk(dev);
 772         drm_mode_config_reset(dev);
 773
 774         /*
 775          * Interrupts have to be enabled before any batches are run. If not the
 776          * GPU will hang. i915_gem_init_hw() will initiate batches to
 777          * update/restore the context.
 778          *
 779          * Modeset enabling in intel_modeset_init_hw() also needs working
 780          * interrupts.
 781          */
 782         intel_runtime_pm_enable_interrupts(dev_priv);
 783
 784         mutex_lock(&dev->struct_mutex);
 785         if (i915_gem_init_hw(dev)) {
 786                 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
 787                         atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
 788         }
 789         mutex_unlock(&dev->struct_mutex);
 790
 791         intel_guc_resume(dev);
 792
 793         intel_modeset_init_hw(dev);
 794
 795         spin_lock_irq(&dev_priv->irq_lock);
 796         if (dev_priv->display.hpd_irq_setup)
 797                 dev_priv->display.hpd_irq_setup(dev);
 798         spin_unlock_irq(&dev_priv->irq_lock);
 799
 800         drm_modeset_lock_all(dev);
 801         intel_display_resume(dev);
 802         drm_modeset_unlock_all(dev);
 803
 804         intel_dp_mst_resume(dev);
 805
 806         /*
 807          * ... but also need to make sure that hotplug processing
 808          * doesn't cause havoc. Like in the driver load code we don't
 809          * bother with the tiny race here where we might loose hotplug
 810          * notifications.
 811          * */
 812         intel_hpd_init(dev_priv);
 813         /* Config may have changed between suspend and resume */
 814         drm_helper_hpd_irq_event(dev);
 815
 816         intel_opregion_init(dev);
 817
 818         intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
 819
 820         mutex_lock(&dev_priv->modeset_restore_lock);
 821         dev_priv->modeset_restore = MODESET_DONE;
 822         mutex_unlock(&dev_priv->modeset_restore_lock);
 823
 824 #if 0
 825         intel_opregion_notify_adapter(dev, PCI_D0);
 826 #endif
 827
 828         drm_kms_helper_poll_enable(dev);
 829
 830         return 0;
 831 }
 832
 833 static int i915_drm_resume_early(struct drm_device *dev)
 834 {
 835         struct drm_i915_private *dev_priv = dev->dev_private;
 836         int ret = 0;
 837
 838         /*
 839          * We have a resume ordering issue with the snd-hda driver also
 840          * requiring our device to be power up. Due to the lack of a
 841          * parent/child relationship we currently solve this with an early
 842          * resume hook.
 843          *
 844          * FIXME: This should be solved with a special hdmi sink device or
 845          * similar so that power domains can be employed.
 846          */
 847 #if 0
 848         if (pci_enable_device(dev->pdev))
 849                 return -EIO;
 850
 851         pci_set_master(dev->pdev);
 852 #endif
 853
 854         if (IS_VALLEYVIEW(dev_priv))
 855                 ret = vlv_resume_prepare(dev_priv, false);
 856         if (ret)
 857                 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
 858                           ret);
 859
 860         intel_uncore_early_sanitize(dev, true);
 861
 862         if (IS_BROXTON(dev))
 863                 ret = bxt_resume_prepare(dev_priv);
 864         else if (IS_SKYLAKE(dev_priv))
 865                 ret = skl_resume_prepare(dev_priv);
 866         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 867                 hsw_disable_pc8(dev_priv);
 868
 869         intel_uncore_sanitize(dev);
 870         intel_power_domains_init_hw(dev_priv);
 871
 872         return ret;
 873 }
 874
 875 int i915_resume_switcheroo(struct drm_device *dev)
 876 {
 877         int ret;
 878
 879         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 880                 return 0;
 881
 882         ret = i915_drm_resume_early(dev);
 883         if (ret)
 884                 return ret;
 885
 886         return i915_drm_resume(dev);
 887 }
 888
 889 /* XXX Hack for the old *BSD drm code base
 890  * The device id field is set at probe time */
 891 static drm_pci_id_list_t i915_attach_list[] = {
 892         {0x8086, 0, 0, "Intel i915 GPU"},
 893         {0, 0, 0, NULL}
 894 };
 895
 896 struct intel_device_info *
 897 i915_get_device_id(int device)
 898 {
 899         const struct pci_device_id *did;
 900
 901         for (did = &pciidlist[0]; did->device != 0; did++) {
 902                 if (did->device != device)
 903                         continue;
 904                 return (struct intel_device_info *)did->driver_data;
 905         }
 906         return (NULL);
 907 }
 908
 909 extern devclass_t drm_devclass;
 910
 911 /**
 912  * i915_reset - reset chip after a hang
 913  * @dev: drm device to reset
 914  *
 915  * Reset the chip.  Useful if a hang is detected. Returns zero on successful
 916  * reset or otherwise an error code.
 917  *
 918  * Procedure is fairly simple:
 919  *   - reset the chip using the reset reg
 920  *   - re-init context state
 921  *   - re-init hardware status page
 922  *   - re-init ring buffer
 923  *   - re-init interrupt state
 924  *   - re-init display
 925  */
 926 int i915_reset(struct drm_device *dev)
 927 {
 928         struct drm_i915_private *dev_priv = dev->dev_private;
 929         bool simulated;
 930         int ret;
 931
 932         intel_reset_gt_powersave(dev);
 933
 934         mutex_lock(&dev->struct_mutex);
 935
 936         i915_gem_reset(dev);
 937
 938         simulated = dev_priv->gpu_error.stop_rings != 0;
 939
 940         ret = intel_gpu_reset(dev);
 941
 942         /* Also reset the gpu hangman. */
 943         if (simulated) {
 944                 DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
 945                 dev_priv->gpu_error.stop_rings = 0;
 946                 if (ret == -ENODEV) {
 947                         DRM_INFO("Reset not implemented, but ignoring "
 948                                  "error for simulated gpu hangs\n");
 949                         ret = 0;
 950                 }
 951         }
 952
 953         if (i915_stop_ring_allow_warn(dev_priv))
 954                 pr_notice("drm/i915: Resetting chip after gpu hang\n");
 955
 956         if (ret) {
 957                 DRM_ERROR("Failed to reset chip: %i\n", ret);
 958                 mutex_unlock(&dev->struct_mutex);
 959                 return ret;
 960         }
 961
 962         intel_overlay_reset(dev_priv);
 963
 964         /* Ok, now get things going again... */
 965
 966         /*
 967          * Everything depends on having the GTT running, so we need to start
 968          * there.  Fortunately we don't need to do this unless we reset the
 969          * chip at a PCI level.
 970          *
 971          * Next we need to restore the context, but we don't use those
 972          * yet either...
 973          *
 974          * Ring buffer needs to be re-initialized in the KMS case, or if X
 975          * was running at the time of the reset (i.e. we weren't VT
 976          * switched away).
 977          */
 978
 979         /* Used to prevent gem_check_wedged returning -EAGAIN during gpu reset */
 980         dev_priv->gpu_error.reload_in_reset = true;
 981
 982         ret = i915_gem_init_hw(dev);
 983
 984         dev_priv->gpu_error.reload_in_reset = false;
 985
 986         mutex_unlock(&dev->struct_mutex);
 987         if (ret) {
 988                 DRM_ERROR("Failed hw init on reset %d\n", ret);
 989                 return ret;
 990         }
 991
 992         /*
 993          * rps/rc6 re-init is necessary to restore state lost after the
 994          * reset and the re-install of gt irqs. Skip for ironlake per
 995          * previous concerns that it doesn't respond well to some forms
 996          * of re-init after reset.
 997          */
 998         if (INTEL_INFO(dev)->gen > 5)
 999                 intel_enable_gt_powersave(dev);
1000
1001         return 0;
1002 }
1003
1004 static int i915_pci_probe(device_t kdev)
1005 {
1006         int device, i = 0;
1007
1008         if (pci_get_class(kdev) != PCIC_DISPLAY)
1009                 return ENXIO;
1010
1011         if (pci_get_vendor(kdev) != PCI_VENDOR_INTEL)
1012                 return ENXIO;
1013
1014         device = pci_get_device(kdev);
1015
1016         for (i = 0; pciidlist[i].device != 0; i++) {
1017                 if (pciidlist[i].device == device) {
1018                         i915_attach_list[0].device = device;
1019                         return 0;
1020                 }
1021         }
1022
1023         return ENXIO;
1024 }
1025
1026 #if 0
1027 static void
1028 i915_pci_remove(struct pci_dev *pdev)
1029 {
1030         struct drm_device *dev = pci_get_drvdata(pdev);
1031
1032         drm_put_dev(dev);
1033 }
1034
1035 static int i915_pm_suspend(struct device *dev)
1036 {
1037         struct pci_dev *pdev = to_pci_dev(dev);
1038         struct drm_device *drm_dev = pci_get_drvdata(pdev);
1039
1040         if (!drm_dev || !drm_dev->dev_private) {
1041                 dev_err(dev, "DRM not initialized, aborting suspend.\n");
1042                 return -ENODEV;
1043         }
1044
1045         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1046                 return 0;
1047
1048         return i915_drm_suspend(drm_dev);
1049 }
1050
1051 static int i915_pm_suspend_late(struct device *dev)
1052 {
1053         struct drm_device *drm_dev = dev_to_i915(dev)->dev;
1054
1055         /*
1056          * We have a suspend ordering issue with the snd-hda driver also
1057          * requiring our device to be power up. Due to the lack of a
1058          * parent/child relationship we currently solve this with an late
1059          * suspend hook.
1060          *
1061          * FIXME: This should be solved with a special hdmi sink device or
1062          * similar so that power domains can be employed.
1063          */
1064         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1065                 return 0;
1066
1067         return i915_drm_suspend_late(drm_dev, false);
1068 }
1069
1070 static int i915_pm_poweroff_late(struct device *dev)
1071 {
1072         struct drm_device *drm_dev = dev_to_i915(dev)->dev;
1073
1074         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1075                 return 0;
1076
1077         return i915_drm_suspend_late(drm_dev, true);
1078 }
1079
1080 static int i915_pm_resume_early(struct device *dev)
1081 {
1082         struct drm_device *drm_dev = dev_to_i915(dev)->dev;
1083
1084         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1085                 return 0;
1086
1087         return i915_drm_resume_early(drm_dev);
1088 }
1089
1090 static int i915_pm_resume(struct device *dev)
1091 {
1092         struct drm_device *drm_dev = dev_to_i915(dev)->dev;
1093
1094         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1095                 return 0;
1096
1097         return i915_drm_resume(drm_dev);
1098 }
1099 #endif
1100
1101 static int skl_suspend_complete(struct drm_i915_private *dev_priv)
1102 {
1103         /* Enabling DC6 is not a hard requirement to enter runtime D3 */
1104
1105         skl_uninit_cdclk(dev_priv);
1106
1107         return 0;
1108 }
1109
1110 static int hsw_suspend_complete(struct drm_i915_private *dev_priv)
1111 {
1112         hsw_enable_pc8(dev_priv);
1113
1114         return 0;
1115 }
1116
1117 static int bxt_suspend_complete(struct drm_i915_private *dev_priv)
1118 {
1119         struct drm_device *dev = dev_priv->dev;
1120
1121         /* TODO: when DC5 support is added disable DC5 here. */
1122
1123         broxton_ddi_phy_uninit(dev);
1124         broxton_uninit_cdclk(dev);
1125         bxt_enable_dc9(dev_priv);
1126
1127         return 0;
1128 }
1129
1130 static int bxt_resume_prepare(struct drm_i915_private *dev_priv)
1131 {
1132         struct drm_device *dev = dev_priv->dev;
1133
1134         /* TODO: when CSR FW support is added make sure the FW is loaded */
1135
1136         bxt_disable_dc9(dev_priv);
1137
1138         /*
1139          * TODO: when DC5 support is added enable DC5 here if the CSR FW
1140          * is available.
1141          */
1142         broxton_init_cdclk(dev);
1143         broxton_ddi_phy_init(dev);
1144         intel_prepare_ddi(dev);
1145
1146         return 0;
1147 }
1148
1149 static int skl_resume_prepare(struct drm_i915_private *dev_priv)
1150 {
1151         struct drm_device *dev = dev_priv->dev;
1152
1153         skl_init_cdclk(dev_priv);
1154         intel_csr_load_program(dev);
1155
1156         return 0;
1157 }
1158
1159 /*
1160  * Save all Gunit registers that may be lost after a D3 and a subsequent
1161  * S0i[R123] transition. The list of registers needing a save/restore is
1162  * defined in the VLV2_S0IXRegs document. This documents marks all Gunit
1163  * registers in the following way:
1164  * - Driver: saved/restored by the driver
1165  * - Punit : saved/restored by the Punit firmware
1166  * - No, w/o marking: no need to save/restore, since the register is R/O or
1167  *                    used internally by the HW in a way that doesn't depend
1168  *                    keeping the content across a suspend/resume.
1169  * - Debug : used for debugging
1170  *
1171  * We save/restore all registers marked with 'Driver', with the following
1172  * exceptions:
1173  * - Registers out of use, including also registers marked with 'Debug'.
1174  *   These have no effect on the driver's operation, so we don't save/restore
1175  *   them to reduce the overhead.
1176  * - Registers that are fully setup by an initialization function called from
1177  *   the resume path. For example many clock gating and RPS/RC6 registers.
1178  * - Registers that provide the right functionality with their reset defaults.
1179  *
1180  * TODO: Except for registers that based on the above 3 criteria can be safely
1181  * ignored, we save/restore all others, practically treating the HW context as
1182  * a black-box for the driver. Further investigation is needed to reduce the
1183  * saved/restored registers even further, by following the same 3 criteria.
1184  */
1185 static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv)
1186 {
1187         struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
1188         int i;
1189
1190         /* GAM 0x4000-0x4770 */
1191         s->wr_watermark         = I915_READ(GEN7_WR_WATERMARK);
1192         s->gfx_prio_ctrl        = I915_READ(GEN7_GFX_PRIO_CTRL);
1193         s->arb_mode             = I915_READ(ARB_MODE);
1194         s->gfx_pend_tlb0        = I915_READ(GEN7_GFX_PEND_TLB0);
1195         s->gfx_pend_tlb1        = I915_READ(GEN7_GFX_PEND_TLB1);
1196
1197         for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
1198                 s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i));
1199
1200         s->media_max_req_count  = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT);
1201         s->gfx_max_req_count    = I915_READ(GEN7_GFX_MAX_REQ_COUNT);
1202
1203         s->render_hwsp          = I915_READ(RENDER_HWS_PGA_GEN7);
1204         s->ecochk               = I915_READ(GAM_ECOCHK);
1205         s->bsd_hwsp             = I915_READ(BSD_HWS_PGA_GEN7);
1206         s->blt_hwsp             = I915_READ(BLT_HWS_PGA_GEN7);
1207
1208         s->tlb_rd_addr          = I915_READ(GEN7_TLB_RD_ADDR);
1209
1210         /* MBC 0x9024-0x91D0, 0x8500 */
1211         s->g3dctl               = I915_READ(VLV_G3DCTL);
1212         s->gsckgctl             = I915_READ(VLV_GSCKGCTL);
1213         s->mbctl                = I915_READ(GEN6_MBCTL);
1214
1215         /* GCP 0x9400-0x9424, 0x8100-0x810C */
1216         s->ucgctl1              = I915_READ(GEN6_UCGCTL1);
1217         s->ucgctl3              = I915_READ(GEN6_UCGCTL3);
1218         s->rcgctl1              = I915_READ(GEN6_RCGCTL1);
1219         s->rcgctl2              = I915_READ(GEN6_RCGCTL2);
1220         s->rstctl               = I915_READ(GEN6_RSTCTL);
1221         s->misccpctl            = I915_READ(GEN7_MISCCPCTL);
1222
1223         /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
1224         s->gfxpause             = I915_READ(GEN6_GFXPAUSE);
1225         s->rpdeuhwtc            = I915_READ(GEN6_RPDEUHWTC);
1226         s->rpdeuc               = I915_READ(GEN6_RPDEUC);
1227         s->ecobus               = I915_READ(ECOBUS);
1228         s->pwrdwnupctl          = I915_READ(VLV_PWRDWNUPCTL);
1229         s->rp_down_timeout      = I915_READ(GEN6_RP_DOWN_TIMEOUT);
1230         s->rp_deucsw            = I915_READ(GEN6_RPDEUCSW);
1231         s->rcubmabdtmr          = I915_READ(GEN6_RCUBMABDTMR);
1232         s->rcedata              = I915_READ(VLV_RCEDATA);
1233         s->spare2gh             = I915_READ(VLV_SPAREG2H);
1234
1235         /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
1236         s->gt_imr               = I915_READ(GTIMR);
1237         s->gt_ier               = I915_READ(GTIER);
1238         s->pm_imr               = I915_READ(GEN6_PMIMR);
1239         s->pm_ier               = I915_READ(GEN6_PMIER);
1240
1241         for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
1242                 s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i));
1243
1244         /* GT SA CZ domain, 0x100000-0x138124 */
1245         s->tilectl              = I915_READ(TILECTL);
1246         s->gt_fifoctl           = I915_READ(GTFIFOCTL);
1247         s->gtlc_wake_ctrl       = I915_READ(VLV_GTLC_WAKE_CTRL);
1248         s->gtlc_survive         = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
1249         s->pmwgicz              = I915_READ(VLV_PMWGICZ);
1250
1251         /* Gunit-Display CZ domain, 0x182028-0x1821CF */
1252         s->gu_ctl0              = I915_READ(VLV_GU_CTL0);
1253         s->gu_ctl1              = I915_READ(VLV_GU_CTL1);
1254         s->pcbr                 = I915_READ(VLV_PCBR);
1255         s->clock_gate_dis2      = I915_READ(VLV_GUNIT_CLOCK_GATE2);
1256
1257         /*
1258          * Not saving any of:
1259          * DFT,         0x9800-0x9EC0
1260          * SARB,        0xB000-0xB1FC
1261          * GAC,         0x5208-0x524C, 0x14000-0x14C000
1262          * PCI CFG
1263          */
1264 }
1265
1266 static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv)
1267 {
1268         struct vlv_s0ix_state *s = &dev_priv->vlv_s0ix_state;
1269         u32 val;
1270         int i;
1271
1272         /* GAM 0x4000-0x4770 */
1273         I915_WRITE(GEN7_WR_WATERMARK,   s->wr_watermark);
1274         I915_WRITE(GEN7_GFX_PRIO_CTRL,  s->gfx_prio_ctrl);
1275         I915_WRITE(ARB_MODE,            s->arb_mode | (0xffff << 16));
1276         I915_WRITE(GEN7_GFX_PEND_TLB0,  s->gfx_pend_tlb0);
1277         I915_WRITE(GEN7_GFX_PEND_TLB1,  s->gfx_pend_tlb1);
1278
1279         for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++)
1280                 I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]);
1281
1282         I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count);
1283         I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count);
1284
1285         I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp);
1286         I915_WRITE(GAM_ECOCHK,          s->ecochk);
1287         I915_WRITE(BSD_HWS_PGA_GEN7,    s->bsd_hwsp);
1288         I915_WRITE(BLT_HWS_PGA_GEN7,    s->blt_hwsp);
1289
1290         I915_WRITE(GEN7_TLB_RD_ADDR,    s->tlb_rd_addr);
1291
1292         /* MBC 0x9024-0x91D0, 0x8500 */
1293         I915_WRITE(VLV_G3DCTL,          s->g3dctl);
1294         I915_WRITE(VLV_GSCKGCTL,        s->gsckgctl);
1295         I915_WRITE(GEN6_MBCTL,          s->mbctl);
1296
1297         /* GCP 0x9400-0x9424, 0x8100-0x810C */
1298         I915_WRITE(GEN6_UCGCTL1,        s->ucgctl1);
1299         I915_WRITE(GEN6_UCGCTL3,        s->ucgctl3);
1300         I915_WRITE(GEN6_RCGCTL1,        s->rcgctl1);
1301         I915_WRITE(GEN6_RCGCTL2,        s->rcgctl2);
1302         I915_WRITE(GEN6_RSTCTL,         s->rstctl);
1303         I915_WRITE(GEN7_MISCCPCTL,      s->misccpctl);
1304
1305         /* GPM 0xA000-0xAA84, 0x8000-0x80FC */
1306         I915_WRITE(GEN6_GFXPAUSE,       s->gfxpause);
1307         I915_WRITE(GEN6_RPDEUHWTC,      s->rpdeuhwtc);
1308         I915_WRITE(GEN6_RPDEUC,         s->rpdeuc);
1309         I915_WRITE(ECOBUS,              s->ecobus);
1310         I915_WRITE(VLV_PWRDWNUPCTL,     s->pwrdwnupctl);
1311         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout);
1312         I915_WRITE(GEN6_RPDEUCSW,       s->rp_deucsw);
1313         I915_WRITE(GEN6_RCUBMABDTMR,    s->rcubmabdtmr);
1314         I915_WRITE(VLV_RCEDATA,         s->rcedata);
1315         I915_WRITE(VLV_SPAREG2H,        s->spare2gh);
1316
1317         /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */
1318         I915_WRITE(GTIMR,               s->gt_imr);
1319         I915_WRITE(GTIER,               s->gt_ier);
1320         I915_WRITE(GEN6_PMIMR,          s->pm_imr);
1321         I915_WRITE(GEN6_PMIER,          s->pm_ier);
1322
1323         for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++)
1324                 I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]);
1325
1326         /* GT SA CZ domain, 0x100000-0x138124 */
1327         I915_WRITE(TILECTL,                     s->tilectl);
1328         I915_WRITE(GTFIFOCTL,                   s->gt_fifoctl);
1329         /*
1330          * Preserve the GT allow wake and GFX force clock bit, they are not
1331          * be restored, as they are used to control the s0ix suspend/resume
1332          * sequence by the caller.
1333          */
1334         val = I915_READ(VLV_GTLC_WAKE_CTRL);
1335         val &= VLV_GTLC_ALLOWWAKEREQ;
1336         val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ;
1337         I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
1338
1339         val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
1340         val &= VLV_GFX_CLK_FORCE_ON_BIT;
1341         val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT;
1342         I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
1343
1344         I915_WRITE(VLV_PMWGICZ,                 s->pmwgicz);
1345
1346         /* Gunit-Display CZ domain, 0x182028-0x1821CF */
1347         I915_WRITE(VLV_GU_CTL0,                 s->gu_ctl0);
1348         I915_WRITE(VLV_GU_CTL1,                 s->gu_ctl1);
1349         I915_WRITE(VLV_PCBR,                    s->pcbr);
1350         I915_WRITE(VLV_GUNIT_CLOCK_GATE2,       s->clock_gate_dis2);
1351 }
1352
1353 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
1354 {
1355         u32 val;
1356         int err;
1357
1358 #define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT)
1359
1360         val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
1361         val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
1362         if (force_on)
1363                 val |= VLV_GFX_CLK_FORCE_ON_BIT;
1364         I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val);
1365
1366         if (!force_on)
1367                 return 0;
1368
1369         err = wait_for(COND, 20);
1370         if (err)
1371                 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
1372                           I915_READ(VLV_GTLC_SURVIVABILITY_REG));
1373
1374         return err;
1375 #undef COND
1376 }
1377
1378 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
1379 {
1380         u32 val;
1381         int err = 0;
1382
1383         val = I915_READ(VLV_GTLC_WAKE_CTRL);
1384         val &= ~VLV_GTLC_ALLOWWAKEREQ;
1385         if (allow)
1386                 val |= VLV_GTLC_ALLOWWAKEREQ;
1387         I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
1388         POSTING_READ(VLV_GTLC_WAKE_CTRL);
1389
1390 #define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \
1391               allow)
1392         err = wait_for(COND, 1);
1393         if (err)
1394                 DRM_ERROR("timeout disabling GT waking\n");
1395         return err;
1396 #undef COND
1397 }
1398
1399 static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
1400                                  bool wait_for_on)
1401 {
1402         u32 mask;
1403         u32 val;
1404         int err;
1405
1406         mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
1407         val = wait_for_on ? mask : 0;
1408 #define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
1409         if (COND)
1410                 return 0;
1411
1412         DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
1413                         wait_for_on ? "on" : "off",
1414                         I915_READ(VLV_GTLC_PW_STATUS));
1415
1416         /*
1417          * RC6 transitioning can be delayed up to 2 msec (see
1418          * valleyview_enable_rps), use 3 msec for safety.
1419          */
1420         err = wait_for(COND, 3);
1421         if (err)
1422                 DRM_ERROR("timeout waiting for GT wells to go %s\n",
1423                           wait_for_on ? "on" : "off");
1424
1425         return err;
1426 #undef COND
1427 }
1428
1429 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
1430 {
1431         if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR))
1432                 return;
1433
1434         DRM_ERROR("GT register access while GT waking disabled\n");
1435         I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
1436 }
1437
1438 static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
1439 {
1440         u32 mask;
1441         int err;
1442
1443         /*
1444          * Bspec defines the following GT well on flags as debug only, so
1445          * don't treat them as hard failures.
1446          */
1447         (void)vlv_wait_for_gt_wells(dev_priv, false);
1448
1449         mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS;
1450         WARN_ON((I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask);
1451
1452         vlv_check_no_gt_access(dev_priv);
1453
1454         err = vlv_force_gfx_clock(dev_priv, true);
1455         if (err)
1456                 goto err1;
1457
1458         err = vlv_allow_gt_wake(dev_priv, false);
1459         if (err)
1460                 goto err2;
1461
1462         if (!IS_CHERRYVIEW(dev_priv->dev))
1463                 vlv_save_gunit_s0ix_state(dev_priv);
1464
1465         err = vlv_force_gfx_clock(dev_priv, false);
1466         if (err)
1467                 goto err2;
1468
1469         return 0;
1470
1471 err2:
1472         /* For safety always re-enable waking and disable gfx clock forcing */
1473         vlv_allow_gt_wake(dev_priv, true);
1474 err1:
1475         vlv_force_gfx_clock(dev_priv, false);
1476
1477         return err;
1478 }
1479
1480 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
1481                                 bool rpm_resume)
1482 {
1483         struct drm_device *dev = dev_priv->dev;
1484         int err;
1485         int ret;
1486
1487         /*
1488          * If any of the steps fail just try to continue, that's the best we
1489          * can do at this point. Return the first error code (which will also
1490          * leave RPM permanently disabled).
1491          */
1492         ret = vlv_force_gfx_clock(dev_priv, true);
1493
1494         if (!IS_CHERRYVIEW(dev_priv->dev))
1495                 vlv_restore_gunit_s0ix_state(dev_priv);
1496
1497         err = vlv_allow_gt_wake(dev_priv, true);
1498         if (!ret)
1499                 ret = err;
1500
1501         err = vlv_force_gfx_clock(dev_priv, false);
1502         if (!ret)
1503                 ret = err;
1504
1505         vlv_check_no_gt_access(dev_priv);
1506
1507         if (rpm_resume) {
1508                 intel_init_clock_gating(dev);
1509                 i915_gem_restore_fences(dev);
1510         }
1511
1512         return ret;
1513 }
1514
1515 #if 0
1516 static int intel_runtime_suspend(struct device *device)
1517 {
1518         struct pci_dev *pdev = to_pci_dev(device);
1519         struct drm_device *dev = pci_get_drvdata(pdev);
1520         struct drm_i915_private *dev_priv = dev->dev_private;
1521         int ret;
1522
1523         if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev))))
1524                 return -ENODEV;
1525
1526         if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
1527                 return -ENODEV;
1528
1529         DRM_DEBUG_KMS("Suspending device\n");
1530
1531         /*
1532          * We could deadlock here in case another thread holding struct_mutex
1533          * calls RPM suspend concurrently, since the RPM suspend will wait
1534          * first for this RPM suspend to finish. In this case the concurrent
1535          * RPM resume will be followed by its RPM suspend counterpart. Still
1536          * for consistency return -EAGAIN, which will reschedule this suspend.
1537          */
1538         if (!mutex_trylock(&dev->struct_mutex)) {
1539                 DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
1540                 /*
1541                  * Bump the expiration timestamp, otherwise the suspend won't
1542                  * be rescheduled.
1543                  */
1544                 pm_runtime_mark_last_busy(device);
1545
1546                 return -EAGAIN;
1547         }
1548         /*
1549          * We are safe here against re-faults, since the fault handler takes
1550          * an RPM reference.
1551          */
1552         i915_gem_release_all_mmaps(dev_priv);
1553         mutex_unlock(&dev->struct_mutex);
1554
1555         intel_guc_suspend(dev);
1556
1557         intel_suspend_gt_powersave(dev);
1558         intel_runtime_pm_disable_interrupts(dev_priv);
1559
1560         ret = intel_suspend_complete(dev_priv);
1561         if (ret) {
1562                 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
1563                 intel_runtime_pm_enable_interrupts(dev_priv);
1564
1565                 return ret;
1566         }
1567
1568         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
1569         intel_uncore_forcewake_reset(dev, false);
1570         dev_priv->pm.suspended = true;
1571
1572         /*
1573          * FIXME: We really should find a document that references the arguments
1574          * used below!
1575          */
1576         if (IS_BROADWELL(dev)) {
1577                 /*
1578                  * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
1579                  * being detected, and the call we do at intel_runtime_resume()
1580                  * won't be able to restore them. Since PCI_D3hot matches the
1581                  * actual specification and appears to be working, use it.
1582                  */
1583                 intel_opregion_notify_adapter(dev, PCI_D3hot);
1584         } else {
1585                 /*
1586                  * current versions of firmware which depend on this opregion
1587                  * notification have repurposed the D1 definition to mean
1588                  * "runtime suspended" vs. what you would normally expect (D3)
1589                  * to distinguish it from notifications that might be sent via
1590                  * the suspend path.
1591                  */
1592                 intel_opregion_notify_adapter(dev, PCI_D1);
1593         }
1594
1595         assert_forcewakes_inactive(dev_priv);
1596
1597         DRM_DEBUG_KMS("Device suspended\n");
1598         return 0;
1599 }
1600
1601 static int intel_runtime_resume(struct device *device)
1602 {
1603         struct pci_dev *pdev = to_pci_dev(device);
1604         struct drm_device *dev = pci_get_drvdata(pdev);
1605         struct drm_i915_private *dev_priv = dev->dev_private;
1606         int ret = 0;
1607
1608         if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
1609                 return -ENODEV;
1610
1611         DRM_DEBUG_KMS("Resuming device\n");
1612
1613         intel_opregion_notify_adapter(dev, PCI_D0);
1614         dev_priv->pm.suspended = false;
1615
1616         intel_guc_resume(dev);
1617
1618         if (IS_GEN6(dev_priv))
1619                 intel_init_pch_refclk(dev);
1620
1621         if (IS_BROXTON(dev))
1622                 ret = bxt_resume_prepare(dev_priv);
1623         else if (IS_SKYLAKE(dev))
1624                 ret = skl_resume_prepare(dev_priv);
1625         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
1626                 hsw_disable_pc8(dev_priv);
1627         else if (IS_VALLEYVIEW(dev_priv))
1628                 ret = vlv_resume_prepare(dev_priv, true);
1629
1630         /*
1631          * No point of rolling back things in case of an error, as the best
1632          * we can do is to hope that things will still work (and disable RPM).
1633          */
1634         i915_gem_init_swizzling(dev);
1635         gen6_update_ring_freq(dev);
1636
1637         intel_runtime_pm_enable_interrupts(dev_priv);
1638
1639         /*
1640          * On VLV/CHV display interrupts are part of the display
1641          * power well, so hpd is reinitialized from there. For
1642          * everyone else do it here.
1643          */
1644         if (!IS_VALLEYVIEW(dev_priv))
1645                 intel_hpd_init(dev_priv);
1646
1647         intel_enable_gt_powersave(dev);
1648
1649         if (ret)
1650                 DRM_ERROR("Runtime resume failed, disabling it (%d)\n", ret);
1651         else
1652                 DRM_DEBUG_KMS("Device resumed\n");
1653
1654         return ret;
1655 }
1656 #endif
1657
1658 /*
1659  * This function implements common functionality of runtime and system
1660  * suspend sequence.
1661  */
1662 static int intel_suspend_complete(struct drm_i915_private *dev_priv)
1663 {
1664         int ret;
1665
1666         if (IS_BROXTON(dev_priv))
1667                 ret = bxt_suspend_complete(dev_priv);
1668         else if (IS_SKYLAKE(dev_priv))
1669                 ret = skl_suspend_complete(dev_priv);
1670         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
1671                 ret = hsw_suspend_complete(dev_priv);
1672         else if (IS_VALLEYVIEW(dev_priv))
1673                 ret = vlv_suspend_complete(dev_priv);
1674         else
1675                 ret = 0;
1676
1677         return ret;
1678 }
1679
1680 #if 0
1681 static const struct dev_pm_ops i915_pm_ops = {
1682         /*
1683          * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
1684          * PMSG_RESUME]
1685          */
1686         .suspend = i915_pm_suspend,
1687         .suspend_late = i915_pm_suspend_late,
1688         .resume_early = i915_pm_resume_early,
1689         .resume = i915_pm_resume,
1690
1691         /*
1692          * S4 event handlers
1693          * @freeze, @freeze_late    : called (1) before creating the
1694          *                            hibernation image [PMSG_FREEZE] and
1695          *                            (2) after rebooting, before restoring
1696          *                            the image [PMSG_QUIESCE]
1697          * @thaw, @thaw_early       : called (1) after creating the hibernation
1698          *                            image, before writing it [PMSG_THAW]
1699          *                            and (2) after failing to create or
1700          *                            restore the image [PMSG_RECOVER]
1701          * @poweroff, @poweroff_late: called after writing the hibernation
1702          *                            image, before rebooting [PMSG_HIBERNATE]
1703          * @restore, @restore_early : called after rebooting and restoring the
1704          *                            hibernation image [PMSG_RESTORE]
1705          */
1706         .freeze = i915_pm_suspend,
1707         .freeze_late = i915_pm_suspend_late,
1708         .thaw_early = i915_pm_resume_early,
1709         .thaw = i915_pm_resume,
1710         .poweroff = i915_pm_suspend,
1711         .poweroff_late = i915_pm_poweroff_late,
1712         .restore_early = i915_pm_resume_early,
1713         .restore = i915_pm_resume,
1714
1715         /* S0ix (via runtime suspend) event handlers */
1716         .runtime_suspend = intel_runtime_suspend,
1717         .runtime_resume = intel_runtime_resume,
1718 };
1719
1720 static const struct vm_operations_struct i915_gem_vm_ops = {
1721         .fault = i915_gem_fault,
1722         .open = drm_gem_vm_open,
1723         .close = drm_gem_vm_close,
1724 };
1725
1726 static const struct file_operations i915_driver_fops = {
1727         .owner = THIS_MODULE,
1728         .open = drm_open,
1729         .release = drm_release,
1730         .unlocked_ioctl = drm_ioctl,
1731         .mmap = drm_gem_mmap,
1732         .poll = drm_poll,
1733         .read = drm_read,
1734 #ifdef CONFIG_COMPAT
1735         .compat_ioctl = i915_compat_ioctl,
1736 #endif
1737         .llseek = noop_llseek,
1738 };
1739 #endif
1740
1741 static struct cdev_pager_ops i915_gem_vm_ops = {
1742         .cdev_pg_fault  = i915_gem_fault,
1743         .cdev_pg_ctor   = i915_gem_pager_ctor,
1744         .cdev_pg_dtor   = i915_gem_pager_dtor
1745 };
1746
1747 static struct drm_driver driver = {
1748         /* Don't use MTRRs here; the Xserver or userspace app should
1749          * deal with them for Intel hardware.
1750          */
1751         .driver_features =
1752             DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM |
1753             DRIVER_RENDER | DRIVER_MODESET,
1754         .load = i915_driver_load,
1755         .unload = i915_driver_unload,
1756         .open = i915_driver_open,
1757         .lastclose = i915_driver_lastclose,
1758         .preclose = i915_driver_preclose,
1759         .postclose = i915_driver_postclose,
1760
1761 #if defined(CONFIG_DEBUG_FS)
1762         .debugfs_init = i915_debugfs_init,
1763         .debugfs_cleanup = i915_debugfs_cleanup,
1764 #endif
1765         .gem_free_object = i915_gem_free_object,
1766         .gem_pager_ops = &i915_gem_vm_ops,
1767
1768         .dumb_create = i915_gem_dumb_create,
1769         .dumb_map_offset = i915_gem_mmap_gtt,
1770         .dumb_destroy = drm_gem_dumb_destroy,
1771         .ioctls = i915_ioctls,
1772         .name = DRIVER_NAME,
1773         .desc = DRIVER_DESC,
1774         .date = DRIVER_DATE,
1775         .major = DRIVER_MAJOR,
1776         .minor = DRIVER_MINOR,
1777         .patchlevel = DRIVER_PATCHLEVEL,
1778 };
1779
1780 static int __init i915_init(void);
1781
1782 static int
1783 i915_attach(device_t kdev)
1784 {
1785         struct drm_device *dev = device_get_softc(kdev);
1786         int error;
1787         int dummy;
1788
1789         i915_init();
1790
1791         dev->driver = &driver;
1792         error = drm_attach(kdev, i915_attach_list);
1793
1794         /*
1795          * XXX hack - give the kvm_console time to come up before X starts
1796          * messing with everything, avoiding at least one deadlock.
1797          */
1798         tsleep(&dummy, 0, "i915_attach", hz*2);
1799
1800         return error;
1801 }
1802
1803 static device_method_t i915_methods[] = {
1804         /* Device interface */
1805         DEVMETHOD(device_probe,         i915_pci_probe),
1806         DEVMETHOD(device_attach,        i915_attach),
1807         DEVMETHOD(device_suspend,       i915_suspend_switcheroo),
1808         DEVMETHOD(device_resume,        i915_resume_switcheroo),
1809         DEVMETHOD(device_detach,        drm_release),
1810         DEVMETHOD_END
1811 };
1812
1813 static driver_t i915_driver = {
1814         "drm",
1815         i915_methods,
1816         sizeof(struct drm_device)
1817 };
1818
1819 static int __init i915_init(void)
1820 {
1821         driver.num_ioctls = i915_max_ioctl;
1822
1823         /*
1824          * Enable KMS by default, unless explicitly overriden by
1825          * either the i915.modeset prarameter or by the
1826          * vga_text_mode_force boot option.
1827          */
1828
1829         if (i915.modeset == 0)
1830                 driver.driver_features &= ~DRIVER_MODESET;
1831
1832 #ifdef CONFIG_VGA_CONSOLE
1833         if (vgacon_text_force() && i915.modeset == -1)
1834                 driver.driver_features &= ~DRIVER_MODESET;
1835 #endif
1836
1837         if (!(driver.driver_features & DRIVER_MODESET)) {
1838                 /* Silently fail loading to not upset userspace. */
1839                 DRM_DEBUG_DRIVER("KMS and UMS disabled.\n");
1840                 return 0;
1841         }
1842
1843         if (i915.nuclear_pageflip)
1844                 driver.driver_features |= DRIVER_ATOMIC;
1845
1846 #if 0
1847         return drm_pci_init(&driver, &i915_pci_driver);
1848 #else
1849         return 1;
1850 #endif
1851 }
1852
1853 #if 0
1854 static void __exit i915_exit(void)
1855 {
1856         if (!(driver.driver_features & DRIVER_MODESET))
1857                 return; /* Never loaded a driver. */
1858
1859         drm_pci_exit(&driver, &i915_pci_driver);
1860 }
1861 #endif
1862
1863 DRIVER_MODULE_ORDERED(i915, vgapci, i915_driver, drm_devclass, NULL, NULL, SI_ORDER_ANY);
1864 MODULE_DEPEND(i915, drm, 1, 1, 1);
1865 MODULE_DEPEND(i915, iicbus, 1, 1, 1);
1866 MODULE_DEPEND(i915, iic, 1, 1, 1);
1867 MODULE_DEPEND(i915, iicbb, 1, 1, 1);
1868 #ifdef CONFIG_ACPI
1869 MODULE_DEPEND(i915, acpi, 1, 1, 1);
1870 #endif