drm/i915: Update to Linux 3.9.11
authorFrançois Tigeot <ftigeot@wolfpond.org>
Fri, 2 Jan 2015 12:51:10 +0000 (13:51 +0100)
committerFrançois Tigeot <ftigeot@wolfpond.org>
Sun, 4 Jan 2015 17:17:02 +0000 (18:17 +0100)
* Various bug fixes and small improvements

* Better handling of GPU hangs, improved driver robustness

* Support for the display power well on Haswell GPUs

* Support for more Haswell GPU (PCI ID) models

46 files changed:
sys/dev/drm/drm_crtc.c
sys/dev/drm/drm_drv.c
sys/dev/drm/drm_edid.c
sys/dev/drm/i915/Makefile
sys/dev/drm/i915/i915_debugfs.c
sys/dev/drm/i915/i915_dma.c
sys/dev/drm/i915/i915_drv.c
sys/dev/drm/i915/i915_drv.h
sys/dev/drm/i915/i915_gem.c
sys/dev/drm/i915/i915_gem_context.c
sys/dev/drm/i915/i915_gem_evict.c
sys/dev/drm/i915/i915_gem_execbuffer.c
sys/dev/drm/i915/i915_gem_gtt.c
sys/dev/drm/i915/i915_gem_stolen.c [new file with mode: 0644]
sys/dev/drm/i915/i915_gem_tiling.c
sys/dev/drm/i915/i915_irq.c
sys/dev/drm/i915/i915_reg.h
sys/dev/drm/i915/i915_suspend.c
sys/dev/drm/i915/i915_trace.h [new file with mode: 0644]
sys/dev/drm/i915/i915_ums.c [copied from sys/dev/drm/i915/i915_suspend.c with 52% similarity]
sys/dev/drm/i915/intel_crt.c
sys/dev/drm/i915/intel_ddi.c
sys/dev/drm/i915/intel_display.c
sys/dev/drm/i915/intel_dp.c
sys/dev/drm/i915/intel_drv.h
sys/dev/drm/i915/intel_fb.c
sys/dev/drm/i915/intel_hdmi.c
sys/dev/drm/i915/intel_i2c.c
sys/dev/drm/i915/intel_lvds.c
sys/dev/drm/i915/intel_modes.c
sys/dev/drm/i915/intel_opregion.c
sys/dev/drm/i915/intel_overlay.c
sys/dev/drm/i915/intel_panel.c
sys/dev/drm/i915/intel_pm.c
sys/dev/drm/i915/intel_ringbuffer.c
sys/dev/drm/i915/intel_ringbuffer.h
sys/dev/drm/i915/intel_sdvo.c
sys/dev/drm/i915/intel_sprite.c
sys/dev/drm/i915/intel_tv.c
sys/dev/drm/include/drm/drmP.h
sys/dev/drm/include/drm/drm_crtc.h
sys/dev/drm/include/drm/drm_edid.h
sys/dev/drm/include/drm/drm_global.h
sys/dev/drm/include/uapi_drm/drm.h
sys/dev/drm/include/uapi_drm/drm_mode.h
sys/dev/drm/include/uapi_drm/i915_drm.h

index 0554e6a..3353cf1 100644 (file)
 #include <drm/drm_edid.h>
 #include <uapi_drm/drm_fourcc.h>
 
+/**
+ * drm_modeset_lock_all - take all modeset locks
+ * @dev: drm device
+ *
+ * This function takes all modeset locks, suitable where a more fine-grained
+ * scheme isn't (yet) implemented.
+ */
+void drm_modeset_lock_all(struct drm_device *dev)
+{
+       struct drm_crtc *crtc;
+
+       lockmgr(&dev->mode_config.mutex, LK_EXCLUSIVE);
+
+       list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+//             mutex_lock_nest_lock(&crtc->mutex, &dev->mode_config.mutex);
+               lockmgr(&crtc->mutex, LK_EXCLUSIVE);
+}
+EXPORT_SYMBOL(drm_modeset_lock_all);
+
+/**
+ * drm_modeset_unlock_all - drop all modeset locks
+ * @dev: device
+ */
+void drm_modeset_unlock_all(struct drm_device *dev)
+{
+       struct drm_crtc *crtc;
+
+       list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+               lockmgr(&crtc->mutex, LK_RELEASE);
+
+       lockmgr(&dev->mode_config.mutex, LK_RELEASE);
+}
+EXPORT_SYMBOL(drm_modeset_unlock_all);
+
 /* Avoid boilerplate.  I'm tired of typing. */
 #define DRM_ENUM_NAME_FN(fnname, list)                         \
        char *fnname(int val)                                   \
@@ -339,6 +373,50 @@ void drm_framebuffer_reference(struct drm_framebuffer *fb)
 }
 EXPORT_SYMBOL(drm_framebuffer_reference);
 
+static void drm_framebuffer_free_bug(struct kref *kref)
+{
+       BUG();
+}
+
+static void __drm_framebuffer_unreference(struct drm_framebuffer *fb)
+{
+       DRM_DEBUG("FB ID: %d\n", fb->base.id);
+       kref_put(&fb->refcount, drm_framebuffer_free_bug);
+}
+
+/* dev->mode_config.fb_lock must be held! */
+static void __drm_framebuffer_unregister(struct drm_device *dev,
+                                        struct drm_framebuffer *fb)
+{
+       mutex_lock(&dev->mode_config.idr_mutex);
+       idr_remove(&dev->mode_config.crtc_idr, fb->base.id);
+       mutex_unlock(&dev->mode_config.idr_mutex);
+
+       fb->base.id = 0;
+
+       __drm_framebuffer_unreference(fb);
+}
+
+/**
+ * drm_framebuffer_unregister_private - unregister a private fb from the lookup idr
+ * @fb: fb to unregister
+ *
+ * Drivers need to call this when cleaning up driver-private framebuffers, e.g.
+ * those used for fbdev. Note that the caller must hold a reference of it's own,
+ * i.e. the object may not be destroyed through this call (since it'll lead to a
+ * locking inversion).
+ */
+void drm_framebuffer_unregister_private(struct drm_framebuffer *fb)
+{
+       struct drm_device *dev = fb->dev;
+
+       mutex_lock(&dev->mode_config.fb_lock);
+       /* Mark fb as reaped and drop idr ref. */
+       __drm_framebuffer_unregister(dev, fb);
+       mutex_unlock(&dev->mode_config.fb_lock);
+}
+EXPORT_SYMBOL(drm_framebuffer_unregister_private);
+
 /**
  * drm_framebuffer_cleanup - remove a framebuffer object
  * @fb: framebuffer to remove
@@ -518,10 +596,7 @@ EXPORT_SYMBOL(drm_mode_remove);
  * @dev: DRM device
  * @connector: the connector to init
  * @funcs: callbacks for this connector
- * @name: user visible name of the connector
- *
- * LOCKING:
- * Takes mode config lock.
+ * @connector_type: user visible type of the connector
  *
  * Initialises a preallocated connector. Connectors should be
  * subclassed as part of driver connector objects.
@@ -548,7 +623,6 @@ int drm_connector_init(struct drm_device *dev,
        connector->connector_type = connector_type;
        connector->connector_type_id =
                ++drm_connector_enum_list[connector_type].count; /* TODO */
-       INIT_LIST_HEAD(&connector->user_modes);
        INIT_LIST_HEAD(&connector->probed_modes);
        INIT_LIST_HEAD(&connector->modes);
        connector->edid_blob_ptr = NULL;
@@ -576,9 +650,6 @@ EXPORT_SYMBOL(drm_connector_init);
  * drm_connector_cleanup - cleans up an initialised connector
  * @connector: connector to cleanup
  *
- * LOCKING:
- * Takes mode config lock.
- *
  * Cleans up the connector but doesn't free the object.
  */
 void drm_connector_cleanup(struct drm_connector *connector)
@@ -592,9 +663,6 @@ void drm_connector_cleanup(struct drm_connector *connector)
        list_for_each_entry_safe(mode, t, &connector->modes, head)
                drm_mode_remove(connector, mode);
 
-       list_for_each_entry_safe(mode, t, &connector->user_modes, head)
-               drm_mode_remove(connector, mode);
-
        lockmgr(&dev->mode_config.mutex, LK_EXCLUSIVE);
        drm_mode_object_put(dev, &connector->base);
        list_del(&connector->head);
@@ -2595,194 +2663,6 @@ void drm_fb_release(struct drm_file *priv)
        lockmgr(&dev->mode_config.mutex, LK_RELEASE);
 }
 
-/**
- * drm_mode_attachmode - add a mode to the user mode list
- * @dev: DRM device
- * @connector: connector to add the mode to
- * @mode: mode to add
- *
- * Add @mode to @connector's user mode list.
- */
-static void drm_mode_attachmode(struct drm_device *dev,
-                               struct drm_connector *connector,
-                               struct drm_display_mode *mode)
-{
-       list_add_tail(&mode->head, &connector->user_modes);
-}
-
-int drm_mode_attachmode_crtc(struct drm_device *dev, struct drm_crtc *crtc,
-                            const struct drm_display_mode *mode)
-{
-       struct drm_connector *connector;
-       int ret = 0;
-       struct drm_display_mode *dup_mode, *next;
-       LINUX_LIST_HEAD(list);
-
-       list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-               if (!connector->encoder)
-                       continue;
-               if (connector->encoder->crtc == crtc) {
-                       dup_mode = drm_mode_duplicate(dev, mode);
-                       if (!dup_mode) {
-                               ret = -ENOMEM;
-                               goto out;
-                       }
-                       list_add_tail(&dup_mode->head, &list);
-               }
-       }
-
-       list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-               if (!connector->encoder)
-                       continue;
-               if (connector->encoder->crtc == crtc)
-                       list_move_tail(list.next, &connector->user_modes);
-       }
-
-       WARN_ON(!list_empty(&list));
-
- out:
-       list_for_each_entry_safe(dup_mode, next, &list, head)
-               drm_mode_destroy(dev, dup_mode);
-
-       return ret;
-}
-EXPORT_SYMBOL(drm_mode_attachmode_crtc);
-
-static int drm_mode_detachmode(struct drm_device *dev,
-                              struct drm_connector *connector,
-                              struct drm_display_mode *mode)
-{
-       int found = 0;
-       int ret = 0;
-       struct drm_display_mode *match_mode, *t;
-
-       list_for_each_entry_safe(match_mode, t, &connector->user_modes, head) {
-               if (drm_mode_equal(match_mode, mode)) {
-                       list_del(&match_mode->head);
-                       drm_mode_destroy(dev, match_mode);
-                       found = 1;
-                       break;
-               }
-       }
-
-       if (!found)
-               ret = -EINVAL;
-
-       return ret;
-}
-
-int drm_mode_detachmode_crtc(struct drm_device *dev, struct drm_display_mode *mode)
-{
-       struct drm_connector *connector;
-
-       list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-               drm_mode_detachmode(dev, connector, mode);
-       }
-       return 0;
-}
-EXPORT_SYMBOL(drm_mode_detachmode_crtc);
-
-/**
- * drm_fb_attachmode - Attach a user mode to an connector
- * @inode: inode from the ioctl
- * @filp: file * from the ioctl
- * @cmd: cmd from ioctl
- * @arg: arg from ioctl
- *
- * This attaches a user specified mode to an connector.
- * Called by the user via ioctl.
- *
- * RETURNS:
- * Zero on success, errno on failure.
- */
-int drm_mode_attachmode_ioctl(struct drm_device *dev,
-                             void *data, struct drm_file *file_priv)
-{
-       struct drm_mode_mode_cmd *mode_cmd = data;
-       struct drm_connector *connector;
-       struct drm_display_mode *mode;
-       struct drm_mode_object *obj;
-       struct drm_mode_modeinfo *umode = &mode_cmd->mode;
-       int ret;
-
-       if (!drm_core_check_feature(dev, DRIVER_MODESET))
-               return -EINVAL;
-
-       lockmgr(&dev->mode_config.mutex, LK_EXCLUSIVE);
-
-       obj = drm_mode_object_find(dev, mode_cmd->connector_id, DRM_MODE_OBJECT_CONNECTOR);
-       if (!obj) {
-               ret = -EINVAL;
-               goto out;
-       }
-       connector = obj_to_connector(obj);
-
-       mode = drm_mode_create(dev);
-       if (!mode) {
-               ret = -ENOMEM;
-               goto out;
-       }
-
-       ret = drm_crtc_convert_umode(mode, umode);
-       if (ret) {
-               DRM_DEBUG_KMS("Invalid mode\n");
-               drm_mode_destroy(dev, mode);
-               goto out;
-       }
-
-       drm_mode_attachmode(dev, connector, mode);
-out:
-       lockmgr(&dev->mode_config.mutex, LK_RELEASE);
-       return ret;
-}
-
-
-/**
- * drm_fb_detachmode - Detach a user specified mode from an connector
- * @inode: inode from the ioctl
- * @filp: file * from the ioctl
- * @cmd: cmd from ioctl
- * @arg: arg from ioctl
- *
- * Called by the user via ioctl.
- *
- * RETURNS:
- * Zero on success, errno on failure.
- */
-int drm_mode_detachmode_ioctl(struct drm_device *dev,
-                             void *data, struct drm_file *file_priv)
-{
-       struct drm_mode_object *obj;
-       struct drm_mode_mode_cmd *mode_cmd = data;
-       struct drm_connector *connector;
-       struct drm_display_mode mode;
-       struct drm_mode_modeinfo *umode = &mode_cmd->mode;
-       int ret;
-
-       if (!drm_core_check_feature(dev, DRIVER_MODESET))
-               return -EINVAL;
-
-       lockmgr(&dev->mode_config.mutex, LK_EXCLUSIVE);
-
-       obj = drm_mode_object_find(dev, mode_cmd->connector_id, DRM_MODE_OBJECT_CONNECTOR);
-       if (!obj) {
-               ret = -EINVAL;
-               goto out;
-       }
-       connector = obj_to_connector(obj);
-
-       ret = drm_crtc_convert_umode(&mode, umode);
-       if (ret) {
-               DRM_DEBUG_KMS("Invalid mode\n");
-               goto out;
-       }
-
-       ret = drm_mode_detachmode(dev, connector, &mode);
-out:
-       lockmgr(&dev->mode_config.mutex, LK_RELEASE);
-       return ret;
-}
-
 struct drm_property *drm_property_create(struct drm_device *dev, int flags,
                                         const char *name, int num_values)
 {
index e5861f8..ba122d7 100644 (file)
@@ -164,8 +164,8 @@ static drm_ioctl_desc_t               drm_ioctls[256] = {
        DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETGAMMA, drm_mode_gamma_set_ioctl, DRM_MASTER|DRM_UNLOCKED),
        DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETENCODER, drm_mode_getencoder, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
        DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCONNECTOR, drm_mode_getconnector, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-       DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATTACHMODE, drm_mode_attachmode_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-       DRM_IOCTL_DEF(DRM_IOCTL_MODE_DETACHMODE, drm_mode_detachmode_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+       DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATTACHMODE, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+       DRM_IOCTL_DEF(DRM_IOCTL_MODE_DETACHMODE, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
        DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPERTY, drm_mode_getproperty_ioctl, DRM_MASTER | DRM_CONTROL_ALLOW|DRM_UNLOCKED),
        DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPROPERTY, drm_mode_connector_property_set_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
        DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPBLOB, drm_mode_getblob_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
@@ -308,7 +308,7 @@ int drm_attach(device_t kdev, drm_pci_id_list_t *idlist)
        lwkt_serialize_init(&dev->irq_lock);
        lockinit(&dev->vbl_lock, "drmvbl", 0, LK_CANRECURSE);
        lockinit(&dev->event_lock, "drmev", 0, LK_CANRECURSE);
-       lockinit(&dev->dev_struct_lock, "drmslk", 0, LK_CANRECURSE);
+       lockinit(&dev->struct_mutex, "drmslk", 0, LK_CANRECURSE);
 
        error = drm_load(dev);
        if (error)
@@ -553,7 +553,7 @@ error:
        lockuninit(&dev->vbl_lock);
        lockuninit(&dev->dev_lock);
        lockuninit(&dev->event_lock);
-       lockuninit(&dev->dev_struct_lock);
+       lockuninit(&dev->struct_mutex);
 
        return retcode;
 }
@@ -619,7 +619,7 @@ static void drm_unload(struct drm_device *dev)
        lockuninit(&dev->vbl_lock);
        lockuninit(&dev->dev_lock);
        lockuninit(&dev->event_lock);
-       lockuninit(&dev->dev_struct_lock);
+       lockuninit(&dev->struct_mutex);
 }
 
 int drm_version(struct drm_device *dev, void *data, struct drm_file *file_priv)
index b3bec1b..a71be1c 100644 (file)
@@ -26,8 +26,6 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
- *
- * $FreeBSD: head/sys/dev/drm2/drm_edid.c 249041 2013-04-03 08:27:35Z dumbbell $
  */
 
 #include <linux/export.h>
@@ -1497,9 +1495,11 @@ add_detailed_modes(struct drm_connector *connector, struct edid *edid,
 #define VIDEO_BLOCK     0x02
 #define VENDOR_BLOCK    0x03
 #define SPEAKER_BLOCK  0x04
+#define VIDEO_CAPABILITY_BLOCK 0x07
 #define EDID_BASIC_AUDIO       (1 << 6)
 #define EDID_CEA_YCRCB444      (1 << 5)
 #define EDID_CEA_YCRCB422      (1 << 4)
+#define EDID_CEA_VCDB_QS       (1 << 6)
 
 /**
  * Search EDID for CEA extension block.
@@ -1531,7 +1531,7 @@ EXPORT_SYMBOL(drm_find_cea_extension);
  * Looks for a CEA mode matching given drm_display_mode.
  * Returns its CEA Video ID code, or 0 if not found.
  */
-u8 drm_match_cea_mode(struct drm_display_mode *to_match)
+u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 {
        u8 mode;
 
@@ -1914,6 +1914,37 @@ end:
 }
 EXPORT_SYMBOL(drm_detect_monitor_audio);
 
+/**
+ * drm_rgb_quant_range_selectable - is RGB quantization range selectable?
+ *
+ * Check whether the monitor reports the RGB quantization range selection
+ * as supported. The AVI infoframe can then be used to inform the monitor
+ * which quantization range (full or limited) is used.
+ */
+bool drm_rgb_quant_range_selectable(struct edid *edid)
+{
+       u8 *edid_ext;
+       int i, start, end;
+
+       edid_ext = drm_find_cea_extension(edid);
+       if (!edid_ext)
+               return false;
+
+       if (cea_db_offsets(edid_ext, &start, &end))
+               return false;
+
+       for_each_cea_db(edid_ext, i, start, end) {
+               if (cea_db_tag(&edid_ext[i]) == VIDEO_CAPABILITY_BLOCK &&
+                   cea_db_payload_len(&edid_ext[i]) == 2) {
+                       DRM_DEBUG_KMS("CEA VCDB 0x%02x\n", edid_ext[i + 2]);
+                       return edid_ext[i + 2] & EDID_CEA_VCDB_QS;
+               }
+       }
+
+       return false;
+}
+EXPORT_SYMBOL(drm_rgb_quant_range_selectable);
+
 /**
  * drm_add_display_info - pull display info out if present
  * @edid: EDID data
@@ -2092,22 +2123,3 @@ int drm_add_modes_noedid(struct drm_connector *connector,
        return num_modes;
 }
 EXPORT_SYMBOL(drm_add_modes_noedid);
-
-/**
- * drm_mode_cea_vic - return the CEA-861 VIC of a given mode
- * @mode: mode
- *
- * RETURNS:
- * The VIC number, 0 in case it's not a CEA-861 mode.
- */
-uint8_t drm_mode_cea_vic(const struct drm_display_mode *mode)
-{
-       uint8_t i;
-
-       for (i = 0; i < drm_num_cea_modes; i++)
-               if (drm_mode_equal(mode, &edid_cea_modes[i]))
-                       return i + 1;
-
-       return 0;
-}
-EXPORT_SYMBOL(drm_mode_cea_vic);
index a6e972c..b4821d5 100644 (file)
@@ -8,9 +8,11 @@ SRCS   = \
        i915_gem_execbuffer.c \
        i915_gem_evict.c \
        i915_gem_gtt.c \
+       i915_gem_stolen.c \
        i915_gem_tiling.c \
        i915_irq.c \
        i915_suspend.c \
+       i915_ums.c \
        intel_bios.c \
        intel_crt.c \
        intel_ddi.c \
index c415636..4e5d70a 100644 (file)
@@ -99,7 +99,7 @@ static const char *cache_level_str(int type)
 static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
-       seq_printf(m, "%pK: %s%s %8zdKiB %04x %04x %d %d %d%s%s%s",
+       seq_printf(m, "%pK: %s%s %8zdKiB %02x %02x %d %d %d%s%s%s",
                   &obj->base,
                   get_pin_flag(obj),
                   get_tiling_flag(obj),
@@ -121,6 +121,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
        if (obj->gtt_space != NULL)
                seq_printf(m, " (gtt offset: %08x, size: %08x)",
                           obj->gtt_offset, (unsigned int)obj->gtt_space->size);
+       if (obj->stolen)
+               seq_printf(m, " (stolen: %08lx)", obj->stolen->start);
        if (obj->pin_mappable || obj->fault_mappable) {
                char s[3], *t = s;
                if (obj->pin_mappable)
@@ -253,8 +255,9 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
        seq_printf(m, "%u fault mappable objects, %zu bytes\n",
                   count, size);
 
-       seq_printf(m, "%zu [%zu] gtt total\n",
-                  dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total);
+       seq_printf(m, "%zu [%lu] gtt total\n",
+                  dev_priv->gtt.total,
+                  dev_priv->gtt.mappable_end - dev_priv->gtt.start);
 
        mutex_unlock(&dev->struct_mutex);
 
@@ -384,7 +387,7 @@ static void i915_ring_seqno_info(struct seq_file *m,
                                 struct intel_ring_buffer *ring)
 {
        if (ring->get_seqno) {
-               seq_printf(m, "Current sequence (%s): %d\n",
+               seq_printf(m, "Current sequence (%s): %u\n",
                           ring->name, ring->get_seqno(ring, false));
        }
 }
@@ -541,11 +544,11 @@ static int i915_hws_info(struct seq_file *m, void *data)
        struct drm_device *dev = node->minor->dev;
        drm_i915_private_t *dev_priv = dev->dev_private;
        struct intel_ring_buffer *ring;
-       const volatile u32 __iomem *hws;
+       const u32 *hws;
        int i;
 
        ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
-       hws = (volatile u32 __iomem *)ring->status_page.page_addr;
+       hws = ring->status_page.page_addr;
        if (hws == NULL)
                return 0;
 
@@ -605,7 +608,7 @@ static void print_error_buffers(struct seq_file *m,
        seq_printf(m, "%s [%d]:\n", name, count);
 
        while (count--) {
-               seq_printf(m, "  %08x %8u %04x %04x %x %x%s%s%s%s%s%s%s",
+               seq_printf(m, "  %08x %8u %02x %02x %x %x%s%s%s%s%s%s%s",
                           err->gtt_offset,
                           err->size,
                           err->read_domains,
@@ -812,11 +815,11 @@ static int i915_error_state_open(struct inode *inode, struct file *file)
 
        error_priv->dev = dev;
 
-       spin_lock_irqsave(&dev_priv->error_lock, flags);
-       error_priv->error = dev_priv->first_error;
+       spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
+       error_priv->error = dev_priv->gpu_error.first_error;
        if (error_priv->error)
                kref_get(&error_priv->error->ref);
-       spin_unlock_irqrestore(&dev_priv->error_lock, flags);
+       spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
 
        return single_open(file, i915_error_state, error_priv);
 }
@@ -842,6 +845,77 @@ static const struct file_operations i915_error_state_fops = {
        .release = i915_error_state_release,
 };
 
+static ssize_t
+i915_next_seqno_read(struct file *filp,
+                char __user *ubuf,
+                size_t max,
+                loff_t *ppos)
+{
+       struct drm_device *dev = filp->private_data;
+       drm_i915_private_t *dev_priv = dev->dev_private;
+       char buf[80];
+       int len;
+       int ret;
+
+       ret = mutex_lock_interruptible(&dev->struct_mutex);
+       if (ret)
+               return ret;
+
+       len = snprintf(buf, sizeof(buf),
+                      "next_seqno :  0x%x\n",
+                      dev_priv->next_seqno);
+
+       mutex_unlock(&dev->struct_mutex);
+
+       if (len > sizeof(buf))
+               len = sizeof(buf);
+
+       return simple_read_from_buffer(ubuf, max, ppos, buf, len);
+}
+
+static ssize_t
+i915_next_seqno_write(struct file *filp,
+                     const char __user *ubuf,
+                     size_t cnt,
+                     loff_t *ppos)
+{
+       struct drm_device *dev = filp->private_data;
+       char buf[20];
+       u32 val = 1;
+       int ret;
+
+       if (cnt > 0) {
+               if (cnt > sizeof(buf) - 1)
+                       return -EINVAL;
+
+               if (copy_from_user(buf, ubuf, cnt))
+                       return -EFAULT;
+               buf[cnt] = 0;
+
+               ret = kstrtouint(buf, 0, &val);
+               if (ret < 0)
+                       return ret;
+       }
+
+       ret = mutex_lock_interruptible(&dev->struct_mutex);
+       if (ret)
+               return ret;
+
+       ret = i915_gem_set_seqno(dev, val);
+
+       mutex_unlock(&dev->struct_mutex);
+
+       return ret ?: cnt;
+}
+
+static const struct file_operations i915_next_seqno_fops = {
+       .owner = THIS_MODULE,
+       .open = simple_open,
+       .read = i915_next_seqno_read,
+       .write = i915_next_seqno_write,
+       .llseek = default_llseek,
+};
+
 static int i915_rstdby_delays(struct seq_file *m, void *unused)
 {
        struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -1372,28 +1446,31 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
        ifbdev = dev_priv->fbdev;
        fb = to_intel_framebuffer(ifbdev->helper.fb);
 
-       seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, obj ",
+       seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, refcount %d, obj ",
                   fb->base.width,
                   fb->base.height,
                   fb->base.depth,
-                  fb->base.bits_per_pixel);
+                  fb->base.bits_per_pixel,
+                  atomic_read(&fb->base.refcount.refcount));
        describe_obj(m, fb->obj);
        seq_printf(m, "\n");
+       mutex_unlock(&dev->mode_config.mutex);
 
+       mutex_lock(&dev->mode_config.fb_lock);
        list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) {
                if (&fb->base == ifbdev->helper.fb)
                        continue;
 
-               seq_printf(m, "user size: %d x %d, depth %d, %d bpp, obj ",
+               seq_printf(m, "user size: %d x %d, depth %d, %d bpp, refcount %d, obj ",
                           fb->base.width,
                           fb->base.height,
                           fb->base.depth,
-                          fb->base.bits_per_pixel);
+                          fb->base.bits_per_pixel,
+                          atomic_read(&fb->base.refcount.refcount));
                describe_obj(m, fb->obj);
                seq_printf(m, "\n");
        }
-
-       mutex_unlock(&dev->mode_config.mutex);
+       mutex_unlock(&dev->mode_config.fb_lock);
 
        return 0;
 }
@@ -1403,7 +1480,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
        struct drm_info_node *node = (struct drm_info_node *) m->private;
        struct drm_device *dev = node->minor->dev;
        drm_i915_private_t *dev_priv = dev->dev_private;
-       int ret;
+       struct intel_ring_buffer *ring;
+       int ret, i;
 
        ret = mutex_lock_interruptible(&dev->mode_config.mutex);
        if (ret)
@@ -1421,6 +1499,14 @@ static int i915_context_status(struct seq_file *m, void *unused)
                seq_printf(m, "\n");
        }
 
+       for_each_ring(ring, dev_priv, i) {
+               if (ring->default_context) {
+                       seq_printf(m, "HW default context %s ring ", ring->name);
+                       describe_obj(m, ring->default_context->obj);
+                       seq_printf(m, "\n");
+               }
+       }
+
        mutex_unlock(&dev->mode_config.mutex);
 
        return 0;
@@ -1460,7 +1546,7 @@ static const char *swizzle_string(unsigned swizzle)
        case I915_BIT_6_SWIZZLE_9_10_17:
                return "bit9/bit10/bit17";
        case I915_BIT_6_SWIZZLE_UNKNOWN:
-               return "unkown";
+               return "unknown";
        }
 
        return "bug";
@@ -1556,7 +1642,7 @@ static int i915_dpio_info(struct seq_file *m, void *data)
                return 0;
        }
 
-       ret = mutex_lock_interruptible(&dev->mode_config.mutex);
+       ret = mutex_lock_interruptible(&dev_priv->dpio_lock);
        if (ret)
                return ret;
 
@@ -1585,7 +1671,7 @@ static int i915_dpio_info(struct seq_file *m, void *data)
        seq_printf(m, "DPIO_FASTCLK_DISABLE: 0x%08x\n",
                   intel_dpio_read(dev_priv, DPIO_FASTCLK_DISABLE));
 
-       mutex_unlock(&dev->mode_config.mutex);
+       mutex_unlock(&dev_priv->dpio_lock);
 
        return 0;
 }
@@ -1603,7 +1689,7 @@ i915_wedged_read(struct file *filp,
 
        len = snprintf(buf, sizeof(buf),
                       "wedged :  %d\n",
-                      atomic_read(&dev_priv->mm.wedged));
+                      atomic_read(&dev_priv->gpu_error.reset_counter));
 
        if (len > sizeof(buf))
                len = sizeof(buf);
@@ -1658,7 +1744,7 @@ i915_ring_stop_read(struct file *filp,
        int len;
 
        len = snprintf(buf, sizeof(buf),
-                      "0x%08x\n", dev_priv->stop_rings);
+                      "0x%08x\n", dev_priv->gpu_error.stop_rings);
 
        if (len > sizeof(buf))
                len = sizeof(buf);
@@ -1694,7 +1780,7 @@ i915_ring_stop_write(struct file *filp,
        if (ret)
                return ret;
 
-       dev_priv->stop_rings = val;
+       dev_priv->gpu_error.stop_rings = val;
        mutex_unlock(&dev->struct_mutex);
 
        return cnt;
@@ -1708,6 +1794,102 @@ static const struct file_operations i915_ring_stop_fops = {
        .llseek = default_llseek,
 };
 
+#define DROP_UNBOUND 0x1
+#define DROP_BOUND 0x2
+#define DROP_RETIRE 0x4
+#define DROP_ACTIVE 0x8
+#define DROP_ALL (DROP_UNBOUND | \
+                 DROP_BOUND | \
+                 DROP_RETIRE | \
+                 DROP_ACTIVE)
+static ssize_t
+i915_drop_caches_read(struct file *filp,
+                     char __user *ubuf,
+                     size_t max,
+                     loff_t *ppos)
+{
+       char buf[20];
+       int len;
+
+       len = snprintf(buf, sizeof(buf), "0x%08x\n", DROP_ALL);
+       if (len > sizeof(buf))
+               len = sizeof(buf);
+
+       return simple_read_from_buffer(ubuf, max, ppos, buf, len);
+}
+
+static ssize_t
+i915_drop_caches_write(struct file *filp,
+                      const char __user *ubuf,
+                      size_t cnt,
+                      loff_t *ppos)
+{
+       struct drm_device *dev = filp->private_data;
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct drm_i915_gem_object *obj, *next;
+       char buf[20];
+       int val = 0, ret;
+
+       if (cnt > 0) {
+               if (cnt > sizeof(buf) - 1)
+                       return -EINVAL;
+
+               if (copy_from_user(buf, ubuf, cnt))
+                       return -EFAULT;
+               buf[cnt] = 0;
+
+               val = simple_strtoul(buf, NULL, 0);
+       }
+
+       DRM_DEBUG_DRIVER("Dropping caches: 0x%08x\n", val);
+
+       /* No need to check and wait for gpu resets, only libdrm auto-restarts
+        * on ioctls on -EAGAIN. */
+       ret = mutex_lock_interruptible(&dev->struct_mutex);
+       if (ret)
+               return ret;
+
+       if (val & DROP_ACTIVE) {
+               ret = i915_gpu_idle(dev);
+               if (ret)
+                       goto unlock;
+       }
+
+       if (val & (DROP_RETIRE | DROP_ACTIVE))
+               i915_gem_retire_requests(dev);
+
+       if (val & DROP_BOUND) {
+               list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, mm_list)
+                       if (obj->pin_count == 0) {
+                               ret = i915_gem_object_unbind(obj);
+                               if (ret)
+                                       goto unlock;
+                       }
+       }
+
+       if (val & DROP_UNBOUND) {
+               list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
+                       if (obj->pages_pin_count == 0) {
+                               ret = i915_gem_object_put_pages(obj);
+                               if (ret)
+                                       goto unlock;
+                       }
+       }
+
+unlock:
+       mutex_unlock(&dev->struct_mutex);
+
+       return ret ?: cnt;
+}
+
+static const struct file_operations i915_drop_caches_fops = {
+       .owner = THIS_MODULE,
+       .open = simple_open,
+       .read = i915_drop_caches_read,
+       .write = i915_drop_caches_write,
+       .llseek = default_llseek,
+};
+
 static ssize_t
 i915_max_freq_read(struct file *filp,
                   char __user *ubuf,
@@ -2104,12 +2286,24 @@ int i915_debugfs_init(struct drm_minor *minor)
        if (ret)
                return ret;
 
+       ret = i915_debugfs_create(minor->debugfs_root, minor,
+                                 "i915_gem_drop_caches",
+                                 &i915_drop_caches_fops);
+       if (ret)
+               return ret;
+
        ret = i915_debugfs_create(minor->debugfs_root, minor,
                                  "i915_error_state",
                                  &i915_error_state_fops);
        if (ret)
                return ret;
 
+       ret = i915_debugfs_create(minor->debugfs_root, minor,
+                                "i915_next_seqno",
+                                &i915_next_seqno_fops);
+       if (ret)
+               return ret;
+
        return drm_debugfs_create_files(i915_debugfs_list,
                                        I915_DEBUGFS_ENTRIES,
                                        minor->debugfs_root, minor);
@@ -2129,10 +2323,14 @@ void i915_debugfs_cleanup(struct drm_minor *minor)
                                 1, minor);
        drm_debugfs_remove_files((struct drm_info_list *) &i915_cache_sharing_fops,
                                 1, minor);
+       drm_debugfs_remove_files((struct drm_info_list *) &i915_drop_caches_fops,
+                                1, minor);
        drm_debugfs_remove_files((struct drm_info_list *) &i915_ring_stop_fops,
                                 1, minor);
        drm_debugfs_remove_files((struct drm_info_list *) &i915_error_state_fops,
                                 1, minor);
+       drm_debugfs_remove_files((struct drm_info_list *) &i915_next_seqno_fops,
+                                1, minor);
 }
 
 #endif /* CONFIG_DEBUG_FS */
index 73d76e6..c696eea 100644 (file)
@@ -162,10 +162,10 @@ static int i915_dma_cleanup(struct drm_device * dev)
        if (dev->irq_enabled)
                drm_irq_uninstall(dev);
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        for (i = 0; i < I915_NUM_RINGS; i++)
                intel_cleanup_ring_buffer(&dev_priv->ring[i]);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        /* Clear the HWS virtual address at teardown */
        if (I915_NEED_GFX_HWS(dev))
@@ -601,9 +601,9 @@ static int i915_flush_ioctl(struct drm_device *dev, void *data,
 
        RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        ret = i915_quiescent(dev);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        return ret;
 }
@@ -649,9 +649,9 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
                }
        }
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        ret = i915_dispatch_batchbuffer(dev, batch, cliprects);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        if (sarea_priv)
                sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
@@ -710,9 +710,9 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
                }
        }
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        if (ret) {
                DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
                goto fail_clip_free;
@@ -824,9 +824,9 @@ static int i915_irq_emit(struct drm_device *dev, void *data,
 
        RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        result = i915_emit_irq(dev);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
                DRM_ERROR("copy_to_user\n");
@@ -909,9 +909,9 @@ static int i915_flip_bufs(struct drm_device *dev, void *data,
 
        RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        ret = i915_dispatch_flip(dev);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        return ret;
 }
@@ -993,6 +993,12 @@ static int i915_getparam(struct drm_device *dev, void *data,
        case I915_PARAM_HAS_PINNED_BATCHES:
                value = 1;
                break;
+       case I915_PARAM_HAS_EXEC_NO_RELOC:
+               value = 1;
+               break;
+       case I915_PARAM_HAS_EXEC_HANDLE_LUT:
+               value = 1;
+               break;
        default:
                DRM_DEBUG_DRIVER("Unknown parameter %d\n",
                                 param->param);
@@ -1024,7 +1030,7 @@ static int i915_setparam(struct drm_device *dev, void *data,
        case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
                break;
        case I915_SETPARAM_ALLOW_BATCHBUFFER:
-               dev_priv->dri1.allow_batchbuffer = param->value;
+               dev_priv->dri1.allow_batchbuffer = param->value ? 1 : 0;
                break;
        case I915_SETPARAM_NUM_USED_FENCES:
                if (param->value > dev_priv->num_fence_regs ||
@@ -1072,7 +1078,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
        ring->status_page.gfx_addr = hws->addr & (0x1ffff<<12);
 
        dev_priv->dri1.gfx_hws_cpu_addr =
-               ioremap_wc(dev_priv->mm.gtt_base_addr + hws->addr, 4096);
+               ioremap_wc(dev_priv->gtt.mappable_base + hws->addr, 4096);
        if (dev_priv->dri1.gfx_hws_cpu_addr == NULL) {
                i915_dma_cleanup(dev);
                ring->status_page.gfx_addr = 0;
@@ -1260,17 +1266,23 @@ static int i915_load_modeset_init(struct drm_device *dev)
                goto cleanup_vga_switcheroo;
 #endif
 
+       ret = drm_irq_install(dev);
+       if (ret)
+               goto cleanup_gem_stolen;
+
+       /* Important: The output setup functions called by modeset_init need
+        * working irqs for e.g. gmbus and dp aux transfers. */
        intel_modeset_init(dev);
 
        ret = i915_gem_init(dev);
        if (ret)
-               goto cleanup_gem_stolen;
+               goto cleanup_irq;
 
-       intel_modeset_gem_init(dev);
+#if 0
+       INIT_WORK(&dev_priv->console_resume_work, intel_console_resume);
+#endif
 
-       ret = drm_irq_install(dev);
-       if (ret)
-               goto cleanup_gem;
+       intel_modeset_gem_init(dev);
 
        /* Always safe in the mode setting case. */
        /* FIXME: do pre/post-mode set stuff in core KMS code */
@@ -1278,7 +1290,25 @@ static int i915_load_modeset_init(struct drm_device *dev)
 
        ret = intel_fbdev_init(dev);
        if (ret)
-               goto cleanup_irq;
+               goto cleanup_gem;
+
+       /* Only enable hotplug handling once the fbdev is fully set up. */
+       intel_hpd_init(dev);
+
+       /*
+        * Some ports require correctly set-up hpd registers for detection to
+        * work properly (leading to ghost connected connector status), e.g. VGA
+        * on gm45.  Hence we can only set up the initial fbdev config after hpd
+        * irqs are fully enabled. Now we should scan for the initial config
+        * only once hotplug handling is enabled, but due to screwed-up locking
+        * around kms/fbdev init we can't protect the fdbev initial config
+        * scanning against hotplug events. Hence do this first and ignore the
+        * tiny window where we will loose hotplug notifactions.
+        */
+       intel_fbdev_initial_config(dev);
+
+       /* Only enable hotplug handling once the fbdev is fully set up. */
+       dev_priv->enable_hotplug_processing = true;
 
        drm_kms_helper_poll_init(dev);
 
@@ -1287,13 +1317,13 @@ static int i915_load_modeset_init(struct drm_device *dev)
 
        return 0;
 
-cleanup_irq:
-       drm_irq_uninstall(dev);
 cleanup_gem:
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        i915_gem_cleanup_ringbuffer(dev);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        i915_gem_cleanup_aliasing_ppgtt(dev);
+cleanup_irq:
+       drm_irq_uninstall(dev);
 cleanup_gem_stolen:
 #if 0
        i915_gem_cleanup_stolen(dev);
@@ -1306,6 +1336,29 @@ out:
        return ret;
 }
 
+#if 0
+static void i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
+{
+       struct apertures_struct *ap;
+       struct pci_dev *pdev = dev_priv->dev->pdev;
+       bool primary;
+
+       ap = alloc_apertures(1);
+       if (!ap)
+               return;
+
+       ap->ranges[0].base = dev_priv->gtt.mappable_base;
+       ap->ranges[0].size = dev_priv->gtt.mappable_end - dev_priv->gtt.start;
+
+       primary =
+               pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+
+       remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
+
+       kfree(ap);
+}
+#endif
+
 /**
  * i915_driver_load - setup chip and create an initial config
  * @dev: DRM device
@@ -1321,11 +1374,9 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        unsigned long base, size;
-       int mmio_bar, ret;
+       int ret = 0, mmio_bar;
        static struct pci_dev i915_pdev;
 
-       ret = 0;
-
        /* XXX: struct pci_dev */
        i915_pdev.dev = dev->dev;
        dev->pdev = &i915_pdev;
@@ -1347,16 +1398,71 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
        dev_priv->info = i915_get_device_id(dev->pci_device);
 
        if (i915_get_bridge_dev(dev)) {
-               drm_free(dev_priv, M_DRM);
-               return (-EIO);
+               ret = -EIO;
+               goto free_priv;
        }
 
        ret = i915_gem_gtt_init(dev);
        if (ret)
                goto put_bridge;
 
-       /* Add register map (needed for suspend/resume) */
+#if 0
+       if (drm_core_check_feature(dev, DRIVER_MODESET))
+               i915_kick_out_firmware_fb(dev_priv);
+
+       pci_set_master(dev->pdev);
+
+       /* overlay on gen2 is broken and can't address above 1G */
+       if (IS_GEN2(dev))
+               dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
+
+       /* 965GM sometimes incorrectly writes to hardware status page (HWS)
+        * using 32bit addressing, overwriting memory if HWS is located
+        * above 4GB.
+        *
+        * The documentation also mentions an issue with undefined
+        * behaviour if any general state is accessed within a page above 4GB,
+        * which also needs to be handled carefully.
+        */
+       if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
+               dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
+#endif
+
        mmio_bar = IS_GEN2(dev) ? 1 : 0;
+       /* Before gen4, the registers and the GTT are behind different BARs.
+        * However, from gen4 onwards, the registers and the GTT are shared
+        * in the same BAR, so we want to restrict this ioremap from
+        * clobbering the GTT which we want ioremap_wc instead. Fortunately,
+        * the register BAR remains the same size for all the earlier
+        * generations up to Ironlake.
+        */
+#if 0
+       if (info->gen < 5)
+               mmio_size = 512*1024;
+       else
+               mmio_size = 2*1024*1024;
+
+       dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
+       if (!dev_priv->regs) {
+               DRM_ERROR("failed to map registers\n");
+               ret = -EIO;
+               goto put_gmch;
+       }
+
+       aperture_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
+
+       dev_priv->gtt.mappable =
+               io_mapping_create_wc(dev_priv->gtt.mappable_base,
+                                    aperture_size);
+       if (dev_priv->gtt.mappable == NULL) {
+               ret = -EIO;
+               goto out_rmmap;
+       }
+
+       i915_mtrr_setup(dev_priv, dev_priv->gtt.mappable_base,
+                       aperture_size);
+#endif
+
        base = drm_get_resource_start(dev, mmio_bar);
        size = drm_get_resource_len(dev, mmio_bar);
 
@@ -1409,13 +1515,18 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         * be lost or delayed, but we use them anyways to avoid
         * stuck interrupts on some machines.
         */
+#if 0
+       if (!IS_I945G(dev) && !IS_I945GM(dev))
+               pci_enable_msi(dev->pdev);
+#endif
 
        lockinit(&dev_priv->irq_lock, "userirq", 0, LK_CANRECURSE);
-       lockinit(&dev_priv->error_lock, "915err", 0, LK_CANRECURSE);
+       lockinit(&dev_priv->gpu_error.lock, "915err", 0, LK_CANRECURSE);
        spin_init(&dev_priv->rps.lock, "i915initrps");
-       spin_init(&dev_priv->dpio_lock, "i915initdpio");
+       lockinit(&dev_priv->dpio_lock, "i915dpio", 0, LK_CANRECURSE);
 
        lockinit(&dev_priv->rps.hw_lock, "i915 rps.hw_lock", 0, LK_CANRECURSE);
+       lockinit(&dev_priv->modeset_restore_lock, "i915mrl", 0, LK_CANRECURSE);
 
        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
                dev_priv->num_pipe = 3;
@@ -1439,11 +1550,15 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
                }
        }
 
+#if 0
+       i915_setup_sysfs(dev);
+#endif
+
        /* Must be done after probing outputs */
        intel_opregion_init(dev);
-
-       setup_timer(&dev_priv->hangcheck_timer, i915_hangcheck_elapsed,
-                   (unsigned long) dev);
+#if 0
+       acpi_video_register();
+#endif
 
        if (IS_GEN5(dev))
                intel_gpu_ips_init(dev_priv);
@@ -1451,11 +1566,14 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
        return 0;
 
 out_gem_unload:
+
        intel_teardown_gmbus(dev);
        intel_teardown_mchbar(dev);
        destroy_workqueue(dev_priv->wq);
 out_mtrrfree:
 put_bridge:
+free_priv:
+       kfree(dev_priv, M_DRM);
        return ret;
 }
 
@@ -1466,28 +1584,57 @@ int i915_driver_unload(struct drm_device *dev)
 
        intel_gpu_ips_teardown();
 
-       DRM_LOCK(dev);
+#if 0
+       i915_teardown_sysfs(dev);
+
+       if (dev_priv->mm.inactive_shrinker.shrink)
+               unregister_shrinker(&dev_priv->mm.inactive_shrinker);
+#endif
+
+       mutex_lock(&dev->struct_mutex);
        ret = i915_gpu_idle(dev);
        if (ret)
                DRM_ERROR("failed to idle hardware: %d\n", ret);
        i915_gem_retire_requests(dev);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        /* Cancel the retire work handler, which should be idle now. */
        cancel_delayed_work_sync(&dev_priv->mm.retire_work);
 
-       i915_free_hws(dev);
+#if 0
+       io_mapping_free(dev_priv->gtt.mappable);
+       if (dev_priv->mm.gtt_mtrr >= 0) {
+               mtrr_del(dev_priv->mm.gtt_mtrr,
+                        dev_priv->gtt.mappable_base,
+                        dev_priv->mm.gtt->gtt_mappable_entries * PAGE_SIZE);
+               dev_priv->mm.gtt_mtrr = -1;
+       }
 
-       intel_teardown_mchbar(dev);
+       acpi_video_unregister();
+#endif
 
        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                intel_fbdev_fini(dev);
                intel_modeset_cleanup(dev);
+#if 0
+               cancel_work_sync(&dev_priv->console_resume_work);
+#endif
+
+               /*
+                * free the memory space allocated for the child device
+                * config parsed from VBT
+                */
+               if (dev_priv->child_dev && dev_priv->child_dev_num) {
+                       kfree(dev_priv->child_dev, M_DRM);
+                       dev_priv->child_dev = NULL;
+                       dev_priv->child_dev_num = 0;
+               }
+
        }
 
        /* Free error state after interrupts are fully disabled. */
-       del_timer_sync(&dev_priv->hangcheck_timer);
-       cancel_work_sync(&dev_priv->error_work);
+       del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
+       cancel_work_sync(&dev_priv->gpu_error.work);
        i915_destroy_error_state(dev);
 
        intel_opregion_fini(dev);
@@ -1496,28 +1643,36 @@ int i915_driver_unload(struct drm_device *dev)
                /* Flush any outstanding unpin_work. */
                flush_workqueue(dev_priv->wq);
 
-               DRM_LOCK(dev);
+               mutex_lock(&dev->struct_mutex);
                i915_gem_free_all_phys_object(dev);
                i915_gem_cleanup_ringbuffer(dev);
                i915_gem_context_fini(dev);
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                i915_gem_cleanup_aliasing_ppgtt(dev);
-               drm_mm_takedown(&dev_priv->mm.stolen);
-
-               intel_cleanup_overlay(dev);
+#if 0
+               i915_gem_cleanup_stolen(dev);
+#endif
 
                if (!I915_NEED_GFX_HWS(dev))
                        i915_free_hws(dev);
        }
 
-       i915_gem_unload(dev);
+#if 0
+       if (dev_priv->regs != NULL)
+               pci_iounmap(dev->pdev, dev_priv->regs);
+#endif
+
+       intel_teardown_gmbus(dev);
+       intel_teardown_mchbar(dev);
 
        bus_generic_detach(dev->dev);
        drm_rmmap(dev, dev_priv->mmio_map);
        intel_teardown_gmbus(dev);
 
        destroy_workqueue(dev_priv->wq);
+       pm_qos_remove_request(&dev_priv->pm_qos);
 
+       pci_dev_put(dev_priv->bridge_dev);
        drm_free(dev->dev_private, M_DRM);
 
        return 0;
index 04eaaaf..5f0aa60 100644 (file)
 int i915_lvds_channel_mode __read_mostly = 0;
 TUNABLE_INT("drm.i915.lvds_channel_mode", &i915_lvds_channel_mode);
 
+int i915_disable_power_well __read_mostly = 0;
+module_param_named(disable_power_well, i915_disable_power_well, int, 0600);
+MODULE_PARM_DESC(disable_power_well,
+                "Disable the power well when possible (default: false)");
+
+bool i915_enable_hangcheck __read_mostly = true;
+module_param_named(enable_hangcheck, i915_enable_hangcheck, bool, 0644);
+MODULE_PARM_DESC(enable_hangcheck,
+               "Periodically check GPU activity for detecting hangs. "
+               "WARNING: Disabling this can cause system wide hangs. "
+               "(default: true)");
+
 static struct drm_driver driver;
 
 #define INTEL_VGA_DEVICE(id, info_) {          \
@@ -184,6 +196,7 @@ static const struct intel_device_info intel_valleyview_m_info = {
        .has_bsd_ring = 1,
        .has_blt_ring = 1,
        .is_valleyview = 1,
+       .display_mmio_offset = VLV_DISPLAY_BASE,
 };
 
 static const struct intel_device_info intel_valleyview_d_info = {
@@ -193,6 +206,7 @@ static const struct intel_device_info intel_valleyview_d_info = {
        .has_bsd_ring = 1,
        .has_blt_ring = 1,
        .is_valleyview = 1,
+       .display_mmio_offset = VLV_DISPLAY_BASE,
 };
 
 static const struct intel_device_info intel_haswell_d_info = {
@@ -263,40 +277,64 @@ static const struct intel_gfx_device_id {
        INTEL_VGA_DEVICE(0x016a, &intel_ivybridge_d_info), /* GT2 server */
        INTEL_VGA_DEVICE(0x0402, &intel_haswell_d_info), /* GT1 desktop */
        INTEL_VGA_DEVICE(0x0412, &intel_haswell_d_info), /* GT2 desktop */
-       INTEL_VGA_DEVICE(0x0422, &intel_haswell_d_info), /* GT2 desktop */
+       INTEL_VGA_DEVICE(0x0422, &intel_haswell_d_info), /* GT3 desktop */
        INTEL_VGA_DEVICE(0x040a, &intel_haswell_d_info), /* GT1 server */
        INTEL_VGA_DEVICE(0x041a, &intel_haswell_d_info), /* GT2 server */
-       INTEL_VGA_DEVICE(0x042a, &intel_haswell_d_info), /* GT2 server */
+       INTEL_VGA_DEVICE(0x042a, &intel_haswell_d_info), /* GT3 server */
        INTEL_VGA_DEVICE(0x0406, &intel_haswell_m_info), /* GT1 mobile */
        INTEL_VGA_DEVICE(0x0416, &intel_haswell_m_info), /* GT2 mobile */
        INTEL_VGA_DEVICE(0x0426, &intel_haswell_m_info), /* GT2 mobile */
+       INTEL_VGA_DEVICE(0x040B, &intel_haswell_d_info), /* GT1 reserved */
+       INTEL_VGA_DEVICE(0x041B, &intel_haswell_d_info), /* GT2 reserved */
+       INTEL_VGA_DEVICE(0x042B, &intel_haswell_d_info), /* GT3 reserved */
+       INTEL_VGA_DEVICE(0x040E, &intel_haswell_d_info), /* GT1 reserved */
+       INTEL_VGA_DEVICE(0x041E, &intel_haswell_d_info), /* GT2 reserved */
+       INTEL_VGA_DEVICE(0x042E, &intel_haswell_d_info), /* GT3 reserved */
        INTEL_VGA_DEVICE(0x0C02, &intel_haswell_d_info), /* SDV GT1 desktop */
        INTEL_VGA_DEVICE(0x0C12, &intel_haswell_d_info), /* SDV GT2 desktop */
-       INTEL_VGA_DEVICE(0x0C22, &intel_haswell_d_info), /* SDV GT2 desktop */
+       INTEL_VGA_DEVICE(0x0C22, &intel_haswell_d_info), /* SDV GT3 desktop */
        INTEL_VGA_DEVICE(0x0C0A, &intel_haswell_d_info), /* SDV GT1 server */
        INTEL_VGA_DEVICE(0x0C1A, &intel_haswell_d_info), /* SDV GT2 server */
-       INTEL_VGA_DEVICE(0x0C2A, &intel_haswell_d_info), /* SDV GT2 server */
+       INTEL_VGA_DEVICE(0x0C2A, &intel_haswell_d_info), /* SDV GT3 server */
        INTEL_VGA_DEVICE(0x0C06, &intel_haswell_m_info), /* SDV GT1 mobile */
        INTEL_VGA_DEVICE(0x0C16, &intel_haswell_m_info), /* SDV GT2 mobile */
-       INTEL_VGA_DEVICE(0x0C26, &intel_haswell_m_info), /* SDV GT2 mobile */
+       INTEL_VGA_DEVICE(0x0C26, &intel_haswell_m_info), /* SDV GT3 mobile */
+       INTEL_VGA_DEVICE(0x0C0B, &intel_haswell_d_info), /* SDV GT1 reserved */
+       INTEL_VGA_DEVICE(0x0C1B, &intel_haswell_d_info), /* SDV GT2 reserved */
+       INTEL_VGA_DEVICE(0x0C2B, &intel_haswell_d_info), /* SDV GT3 reserved */
+       INTEL_VGA_DEVICE(0x0C0E, &intel_haswell_d_info), /* SDV GT1 reserved */
+       INTEL_VGA_DEVICE(0x0C1E, &intel_haswell_d_info), /* SDV GT2 reserved */
+       INTEL_VGA_DEVICE(0x0C2E, &intel_haswell_d_info), /* SDV GT3 reserved */
        INTEL_VGA_DEVICE(0x0A02, &intel_haswell_d_info), /* ULT GT1 desktop */
        INTEL_VGA_DEVICE(0x0A12, &intel_haswell_d_info), /* ULT GT2 desktop */
-       INTEL_VGA_DEVICE(0x0A22, &intel_haswell_d_info), /* ULT GT2 desktop */
+       INTEL_VGA_DEVICE(0x0A22, &intel_haswell_d_info), /* ULT GT3 desktop */
        INTEL_VGA_DEVICE(0x0A0A, &intel_haswell_d_info), /* ULT GT1 server */
        INTEL_VGA_DEVICE(0x0A1A, &intel_haswell_d_info), /* ULT GT2 server */
-       INTEL_VGA_DEVICE(0x0A2A, &intel_haswell_d_info), /* ULT GT2 server */
+       INTEL_VGA_DEVICE(0x0A2A, &intel_haswell_d_info), /* ULT GT3 server */
        INTEL_VGA_DEVICE(0x0A06, &intel_haswell_m_info), /* ULT GT1 mobile */
        INTEL_VGA_DEVICE(0x0A16, &intel_haswell_m_info), /* ULT GT2 mobile */
-       INTEL_VGA_DEVICE(0x0A26, &intel_haswell_m_info), /* ULT GT2 mobile */
+       INTEL_VGA_DEVICE(0x0A26, &intel_haswell_m_info), /* ULT GT3 mobile */
+       INTEL_VGA_DEVICE(0x0A0B, &intel_haswell_d_info), /* ULT GT1 reserved */
+       INTEL_VGA_DEVICE(0x0A1B, &intel_haswell_d_info), /* ULT GT2 reserved */
+       INTEL_VGA_DEVICE(0x0A2B, &intel_haswell_d_info), /* ULT GT3 reserved */
+       INTEL_VGA_DEVICE(0x0A0E, &intel_haswell_m_info), /* ULT GT1 reserved */
+       INTEL_VGA_DEVICE(0x0A1E, &intel_haswell_m_info), /* ULT GT2 reserved */
+       INTEL_VGA_DEVICE(0x0A2E, &intel_haswell_m_info), /* ULT GT3 reserved */
        INTEL_VGA_DEVICE(0x0D02, &intel_haswell_d_info), /* CRW GT1 desktop */
        INTEL_VGA_DEVICE(0x0D12, &intel_haswell_d_info), /* CRW GT2 desktop */
-       INTEL_VGA_DEVICE(0x0D22, &intel_haswell_d_info), /* CRW GT2 desktop */
+       INTEL_VGA_DEVICE(0x0D22, &intel_haswell_d_info), /* CRW GT3 desktop */
        INTEL_VGA_DEVICE(0x0D0A, &intel_haswell_d_info), /* CRW GT1 server */
        INTEL_VGA_DEVICE(0x0D1A, &intel_haswell_d_info), /* CRW GT2 server */
-       INTEL_VGA_DEVICE(0x0D2A, &intel_haswell_d_info), /* CRW GT2 server */
+       INTEL_VGA_DEVICE(0x0D2A, &intel_haswell_d_info), /* CRW GT3 server */
        INTEL_VGA_DEVICE(0x0D06, &intel_haswell_m_info), /* CRW GT1 mobile */
        INTEL_VGA_DEVICE(0x0D16, &intel_haswell_m_info), /* CRW GT2 mobile */
-       INTEL_VGA_DEVICE(0x0D26, &intel_haswell_m_info), /* CRW GT2 mobile */
+       INTEL_VGA_DEVICE(0x0D26, &intel_haswell_m_info), /* CRW GT3 mobile */
+       INTEL_VGA_DEVICE(0x0D0B, &intel_haswell_d_info), /* CRW GT1 reserved */
+       INTEL_VGA_DEVICE(0x0D1B, &intel_haswell_d_info), /* CRW GT2 reserved */
+       INTEL_VGA_DEVICE(0x0D2B, &intel_haswell_d_info), /* CRW GT3 reserved */
+       INTEL_VGA_DEVICE(0x0D0E, &intel_haswell_d_info), /* CRW GT1 reserved */
+       INTEL_VGA_DEVICE(0x0D1E, &intel_haswell_d_info), /* CRW GT2 reserved */
+       INTEL_VGA_DEVICE(0x0D2E, &intel_haswell_d_info), /* CRW GT3 reserved */
        INTEL_VGA_DEVICE(0x0f30, &intel_valleyview_m_info),
        INTEL_VGA_DEVICE(0x0157, &intel_valleyview_m_info),
        INTEL_VGA_DEVICE(0x0155, &intel_valleyview_d_info),
@@ -379,6 +417,13 @@ static int i915_drm_freeze(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
 
+       /* ignore lid events during suspend */
+       mutex_lock(&dev_priv->modeset_restore_lock);
+       dev_priv->modeset_restore = MODESET_SUSPENDED;
+       mutex_unlock(&dev_priv->modeset_restore_lock);
+
+       intel_set_power_well(dev, true);
+
        drm_kms_helper_poll_disable(dev);
 
 #if 0
@@ -389,10 +434,11 @@ static int i915_drm_freeze(struct drm_device *dev)
        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                int error = i915_gem_idle(dev);
                if (error) {
-                       device_printf(dev->dev,
-                               "GEM idle failed, resume might fail");
+                       dev_err(dev->pdev->dev,
+                               "GEM idle failed, resume might fail\n");
                        return error;
                }
+
                cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
 
 #if 0
@@ -400,15 +446,13 @@ static int i915_drm_freeze(struct drm_device *dev)
 #endif
 
                drm_irq_uninstall(dev);
+               dev_priv->enable_hotplug_processing = false;
        }
 
        i915_save_state(dev);
 
        intel_opregion_fini(dev);
 
-       /* Modeset on resume, not lid events */
-       dev_priv->modeset_on_lid = 0;
-
        return 0;
 }
 
@@ -434,19 +478,11 @@ i915_suspend(device_t kdev)
        return (error);
 }
 
-static int i915_drm_thaw(struct drm_device *dev)
+static int __i915_drm_thaw(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        int error = 0;
 
-       intel_gt_reset(dev);
-
-       if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-               DRM_LOCK(dev);
-               i915_gem_restore_gtt_mappings(dev);
-               DRM_UNLOCK(dev);
-       }
-
        i915_restore_state(dev);
        intel_opregion_setup(dev);
 
@@ -454,26 +490,64 @@ static int i915_drm_thaw(struct drm_device *dev)
        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                intel_init_pch_refclk(dev);
 
-               DRM_LOCK(dev);
+               mutex_lock(&dev->struct_mutex);
                dev_priv->mm.suspended = 0;
 
                error = i915_gem_init_hw(dev);
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
+
+               /* We need working interrupts for modeset enabling ... */
+               drm_irq_install(dev);
 
                intel_modeset_init_hw(dev);
                intel_modeset_setup_hw_state(dev, false);
-               drm_irq_install(dev);
+
+               /*
+                * ... but also need to make sure that hotplug processing
+                * doesn't cause havoc. Like in the driver load code we don't
+                * bother with the tiny race here where we might loose hotplug
+                * notifications.
+                * */
+               intel_hpd_init(dev);
+               dev_priv->enable_hotplug_processing = true;
        }
 
        intel_opregion_init(dev);
 
-       dev_priv->modeset_on_lid = 0;
-
+       /*
+        * The console lock can be pretty contented on resume due
+        * to all the printk activity.  Try to keep it out of the hot
+        * path of resume if possible.
+        */
 #if 0
-       console_lock();
-       intel_fbdev_set_suspend(dev, 0);
-       console_unlock();
+       if (console_trylock()) {
+               intel_fbdev_set_suspend(dev, 0);
+               console_unlock();
+       } else {
+               schedule_work(&dev_priv->console_resume_work);
+       }
 #endif
+
+       mutex_lock(&dev_priv->modeset_restore_lock);
+       dev_priv->modeset_restore = MODESET_DONE;
+       mutex_unlock(&dev_priv->modeset_restore_lock);
+       return error;
+}
+
+static int i915_drm_thaw(struct drm_device *dev)
+{
+       int error = 0;
+
+       intel_gt_reset(dev);
+
+       if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+               mutex_lock(&dev->struct_mutex);
+               i915_gem_restore_gtt_mappings(dev);
+               mutex_unlock(&dev->struct_mutex);
+       }
+
+       __i915_drm_thaw(dev);
+
        return error;
 }
 
@@ -509,29 +583,6 @@ static drm_pci_id_list_t i915_attach_list[] = {
        {0, 0, 0, NULL}
 };
 
-static int
-i915_probe(device_t kdev)
-{
-       int device, i = 0;
-
-       if (pci_get_class(kdev) != PCIC_DISPLAY)
-               return ENXIO;
-
-       if (pci_get_vendor(kdev) != PCI_VENDOR_INTEL)
-               return ENXIO;
-
-       device = pci_get_device(kdev);
-
-       for (i = 0; pciidlist[i].device != 0; i++) {
-               if (pciidlist[i].device == device) {
-                       i915_attach_list[0].device = device;
-                       return 0;
-               }
-       }
-
-       return ENXIO;
-}
-
 int i915_modeset;
 
 /* static int __init i915_init(void) */
@@ -564,30 +615,7 @@ i915_get_device_id(int device)
        return (NULL);
 }
 
-static device_method_t i915_methods[] = {
-       /* Device interface */
-       DEVMETHOD(device_probe,         i915_probe),
-       DEVMETHOD(device_attach,        i915_attach),
-       DEVMETHOD(device_suspend,       i915_suspend),
-       DEVMETHOD(device_resume,        i915_resume),
-       DEVMETHOD(device_detach,        drm_detach),
-       DEVMETHOD_END
-};
-
-static driver_t i915_driver = {
-       "drm",
-       i915_methods,
-       sizeof(struct drm_device)
-};
-
 extern devclass_t drm_devclass;
-DRIVER_MODULE_ORDERED(i915kms, vgapci, i915_driver, drm_devclass, 0, 0,
-    SI_ORDER_ANY);
-MODULE_DEPEND(i915kms, drm, 1, 1, 1);
-MODULE_DEPEND(i915kms, agp, 1, 1, 1);
-MODULE_DEPEND(i915kms, iicbus, 1, 1, 1);
-MODULE_DEPEND(i915kms, iic, 1, 1, 1);
-MODULE_DEPEND(i915kms, iicbb, 1, 1, 1);
 
 int intel_iommu_enabled = 0;
 TUNABLE_INT("drm.i915.intel_iommu_enabled", &intel_iommu_enabled);
@@ -614,8 +642,6 @@ int i915_modeset = 1;
 TUNABLE_INT("drm.i915.modeset", &i915_modeset);
 int i915_enable_ppgtt = -1;
 TUNABLE_INT("drm.i915.enable_ppgtt", &i915_enable_ppgtt);
-int i915_enable_hangcheck = 1;
-TUNABLE_INT("drm.i915.enable_hangcheck", &i915_enable_hangcheck);
 
 static int i8xx_do_reset(struct drm_device *dev)
 {
@@ -759,9 +785,9 @@ int intel_gpu_reset(struct drm_device *dev)
        }
 
        /* Also reset the gpu hangman. */
-       if (dev_priv->stop_rings) {
+       if (dev_priv->gpu_error.stop_rings) {
                DRM_DEBUG("Simulated gpu hang, resetting stop_rings\n");
-               dev_priv->stop_rings = 0;
+               dev_priv->gpu_error.stop_rings = 0;
                if (ret == -ENODEV) {
                        DRM_ERROR("Reset not implemented, but ignoring "
                                  "error for simulated gpu hangs\n");
@@ -795,20 +821,20 @@ int i915_reset(struct drm_device *dev)
        if (!i915_try_reset)
                return 0;
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
 
        i915_gem_reset(dev);
 
        ret = -ENODEV;
-       if (time_uptime - dev_priv->last_gpu_reset < 5)
+       if (time_uptime - dev_priv->gpu_error.last_reset < 5)
                DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
        else
                ret = intel_gpu_reset(dev);
 
-       dev_priv->last_gpu_reset = time_uptime;
+       dev_priv->gpu_error.last_reset = time_uptime;
        if (ret) {
                DRM_ERROR("Failed to reset chip.\n");
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                return ret;
        }
 
@@ -847,17 +873,41 @@ int i915_reset(struct drm_device *dev)
                 * some unknown reason, this blows up my ilk, so don't.
                 */
 
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
 
                drm_irq_uninstall(dev);
                drm_irq_install(dev);
+               intel_hpd_init(dev);
        } else {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
        }
 
        return 0;
 }
 
+static int
+i915_pci_probe(device_t kdev)
+{
+       int device, i = 0;
+
+       if (pci_get_class(kdev) != PCIC_DISPLAY)
+               return ENXIO;
+
+       if (pci_get_vendor(kdev) != PCI_VENDOR_INTEL)
+               return ENXIO;
+
+       device = pci_get_device(kdev);
+
+       for (i = 0; pciidlist[i].device != 0; i++) {
+               if (pciidlist[i].device == device) {
+                       i915_attach_list[0].device = device;
+                       return 0;
+               }
+       }
+
+       return ENXIO;
+}
+
 static struct drm_driver driver = {
        .driver_features =   DRIVER_USE_AGP | DRIVER_REQUIRE_AGP |
            DRIVER_USE_MTRR | DRIVER_HAVE_IRQ | DRIVER_LOCKLESS_IRQ |
@@ -888,107 +938,35 @@ static struct drm_driver driver = {
        .patchlevel     = DRIVER_PATCHLEVEL,
 };
 
+static device_method_t i915_methods[] = {
+       /* Device interface */
+       DEVMETHOD(device_probe,         i915_pci_probe),
+       DEVMETHOD(device_attach,        i915_attach),
+       DEVMETHOD(device_suspend,       i915_suspend),
+       DEVMETHOD(device_resume,        i915_resume),
+       DEVMETHOD(device_detach,        drm_detach),
+       DEVMETHOD_END
+};
+
+static driver_t i915_driver = {
+       "drm",
+       i915_methods,
+       sizeof(struct drm_device)
+};
+
+DRIVER_MODULE_ORDERED(i915kms, vgapci, i915_driver, drm_devclass, 0, 0,
+    SI_ORDER_ANY);
+MODULE_DEPEND(i915kms, drm, 1, 1, 1);
+MODULE_DEPEND(i915kms, agp, 1, 1, 1);
+MODULE_DEPEND(i915kms, iicbus, 1, 1, 1);
+MODULE_DEPEND(i915kms, iic, 1, 1, 1);
+MODULE_DEPEND(i915kms, iicbb, 1, 1, 1);
+
 /* We give fast paths for the really cool registers */
 #define NEEDS_FORCE_WAKE(dev_priv, reg) \
        ((HAS_FORCE_WAKE((dev_priv)->dev)) && \
         ((reg) < 0x40000) &&            \
         ((reg) != FORCEWAKE))
-
-static bool IS_DISPLAYREG(u32 reg)
-{
-       /*
-        * This should make it easier to transition modules over to the
-        * new register block scheme, since we can do it incrementally.
-        */
-       if (reg >= VLV_DISPLAY_BASE)
-               return false;
-
-       if (reg >= RENDER_RING_BASE &&
-           reg < RENDER_RING_BASE + 0xff)
-               return false;
-       if (reg >= GEN6_BSD_RING_BASE &&
-           reg < GEN6_BSD_RING_BASE + 0xff)
-               return false;
-       if (reg >= BLT_RING_BASE &&
-           reg < BLT_RING_BASE + 0xff)
-               return false;
-
-       if (reg == PGTBL_ER)
-               return false;
-
-       if (reg >= IPEIR_I965 &&
-           reg < HWSTAM)
-               return false;
-
-       if (reg == MI_MODE)
-               return false;
-
-       if (reg == GFX_MODE_GEN7)
-               return false;
-
-       if (reg == RENDER_HWS_PGA_GEN7 ||
-           reg == BSD_HWS_PGA_GEN7 ||
-           reg == BLT_HWS_PGA_GEN7)
-               return false;
-
-       if (reg == GEN6_BSD_SLEEP_PSMI_CONTROL ||
-           reg == GEN6_BSD_RNCID)
-               return false;
-
-       if (reg == GEN6_BLITTER_ECOSKPD)
-               return false;
-
-       if (reg >= 0x4000c &&
-           reg <= 0x4002c)
-               return false;
-
-       if (reg >= 0x4f000 &&
-           reg <= 0x4f08f)
-               return false;
-
-       if (reg >= 0x4f100 &&
-           reg <= 0x4f11f)
-               return false;
-
-       if (reg >= VLV_MASTER_IER &&
-           reg <= GEN6_PMIER)
-               return false;
-
-       if (reg >= FENCE_REG_SANDYBRIDGE_0 &&
-           reg < (FENCE_REG_SANDYBRIDGE_0 + (16*8)))
-               return false;
-
-       if (reg >= VLV_IIR_RW &&
-           reg <= VLV_ISR)
-               return false;
-
-       if (reg == FORCEWAKE_VLV ||
-           reg == FORCEWAKE_ACK_VLV)
-               return false;
-
-       if (reg == GEN6_GDRST)
-               return false;
-
-       switch (reg) {
-       case _3D_CHICKEN3:
-       case IVB_CHICKEN3:
-       case GEN7_COMMON_SLICE_CHICKEN1:
-       case GEN7_L3CNTLREG1:
-       case GEN7_L3_CHICKEN_MODE_REGISTER:
-       case GEN7_ROW_CHICKEN2:
-       case GEN7_L3SQCREG4:
-       case GEN7_SQ_CHICKEN_MBCUNIT_CONFIG:
-       case GEN7_HALF_SLICE_CHICKEN1:
-       case GEN6_MBCTL:
-       case GEN6_UCGCTL2:
-               return false;
-       default:
-               break;
-       }
-
-       return true;
-}
-
 static void
 ilk_dummy_write(struct drm_i915_private *dev_priv)
 {
@@ -1011,8 +989,6 @@ u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
                if (dev_priv->forcewake_count == 0) \
                        dev_priv->gt.force_wake_put(dev_priv); \
                lockmgr(&dev_priv->gt_lock, LK_RELEASE); \
-       } else if (IS_VALLEYVIEW(dev_priv->dev) && IS_DISPLAYREG(reg)) { \
-               val = DRM_READ##y(dev_priv->mmio_map, reg + 0x180000);  \
        } else { \
                val = DRM_READ##y(dev_priv->mmio_map, reg);     \
        } \
@@ -1039,11 +1015,7 @@ void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
                DRM_ERROR("Unknown unclaimed register before writing to %x\n", reg); \
                I915_WRITE_NOTRACE(GEN7_ERR_INT, ERR_INT_MMIO_UNCLAIMED); \
        } \
-       if (IS_VALLEYVIEW(dev_priv->dev) && IS_DISPLAYREG(reg)) { \
-               DRM_WRITE##y(dev_priv->mmio_map, reg + 0x180000, val);  \
-       } else {                                                        \
-               DRM_WRITE##y(dev_priv->mmio_map, reg, val);             \
-       }                                                               \
+       DRM_WRITE##y(dev_priv->mmio_map, reg, val); \
        if (unlikely(__fifo_ret)) { \
                gen6_gt_check_fifodbg(dev_priv); \
        } \
index d6d2db2..8a8f427 100644 (file)
 #ifndef _I915_DRV_H_
 #define _I915_DRV_H_
 
+#include <uapi_drm/i915_drm.h>
+
 #include "i915_reg.h"
 #include "intel_bios.h"
 #include "intel_ringbuffer.h"
-#include <linux/completion.h>
 #include <linux/i2c.h>
 #include <drm/intel-gtt.h>
 #include <linux/kref.h>
-#include <linux/workqueue.h>
+#include <linux/pm_qos.h>
 
 #define CONFIG_ACPI 1
 
@@ -83,7 +84,12 @@ enum port {
 };
 #define port_name(p) ((p) + 'A')
 
-#define I915_GEM_GPU_DOMAINS   (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+#define I915_GEM_GPU_DOMAINS \
+       (I915_GEM_DOMAIN_RENDER | \
+        I915_GEM_DOMAIN_SAMPLER | \
+        I915_GEM_DOMAIN_COMMAND | \
+        I915_GEM_DOMAIN_INSTRUCTION | \
+        I915_GEM_DOMAIN_VERTEX)
 
 #define for_each_pipe(p) for ((p) = 0; (p) < dev_priv->num_pipe; (p)++)
 
@@ -101,6 +107,19 @@ struct intel_pch_pll {
 };
 #define I915_NUM_PLLS 2
 
+/* Used by dp and fdi links */
+struct intel_link_m_n {
+       uint32_t        tu;
+       uint32_t        gmch_m;
+       uint32_t        gmch_n;
+       uint32_t        link_m;
+       uint32_t        link_n;
+};
+
+void intel_link_compute_m_n(int bpp, int nlanes,
+                           int pixel_clock, int link_clock,
+                           struct intel_link_m_n *m_n);
+
 struct intel_ddi_plls {
        int spll_refcount;
        int wrpll1_refcount;
@@ -132,6 +151,7 @@ struct intel_ddi_plls {
 
 struct drm_i915_gem_phys_object {
        int id;
+       struct page **page_list;
        drm_dma_handle_t *handle;
        struct drm_i915_gem_object *cur_obj;
 };
@@ -278,6 +298,7 @@ struct drm_i915_display_funcs {
                          struct drm_i915_gem_object *obj);
        int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                            int x, int y);
+       void (*hpd_irq_setup)(struct drm_device *dev);
        /* clock updates for mode set */
        /* cursor updates */
        /* render clock increase/decrease */
@@ -317,6 +338,7 @@ struct drm_i915_gt_funcs {
        DEV_INFO_FLAG(has_llc)
 
 struct intel_device_info {
+       u32 display_mmio_offset;
        u8 gen;
        u8 is_mobile:1;
        u8 is_i85x:1;
@@ -344,6 +366,50 @@ struct intel_device_info {
        u8 has_llc:1;
 };
 
+enum i915_cache_level {
+       I915_CACHE_NONE = 0,
+       I915_CACHE_LLC,
+       I915_CACHE_LLC_MLC, /* gen6+, in docs at least! */
+};
+
+/* The Graphics Translation Table is the way in which GEN hardware translates a
+ * Graphics Virtual Address into a Physical Address. In addition to the normal
+ * collateral associated with any va->pa translations GEN hardware also has a
+ * portion of the GTT which can be mapped by the CPU and remain both coherent
+ * and correct (in cases like swizzling). That region is referred to as GMADR in
+ * the spec.
+ */
+struct i915_gtt {
+       unsigned long start;            /* Start offset of used GTT */
+       size_t total;                   /* Total size GTT can map */
+       size_t stolen_size;             /* Total size of stolen memory */
+
+       unsigned long mappable_end;     /* End offset that we can CPU map */
+       struct io_mapping *mappable;    /* Mapping to our CPU mappable region */
+       phys_addr_t mappable_base;      /* PA of our GMADR */
+
+       /** "Graphics Stolen Memory" holds the global PTEs */
+       void __iomem *gsm;
+
+       bool do_idle_maps;
+       dma_addr_t scratch_page_dma;
+       struct page *scratch_page;
+
+       /* global gtt ops */
+       int (*gtt_probe)(struct drm_device *dev, size_t *gtt_total,
+                         size_t *stolen, phys_addr_t *mappable_base,
+                         unsigned long *mappable_end);
+       void (*gtt_remove)(struct drm_device *dev);
+       void (*gtt_clear_range)(struct drm_device *dev,
+                               unsigned int first_entry,
+                               unsigned int num_entries);
+       void (*gtt_insert_entries)(struct drm_device *dev,
+                                  struct sg_table *st,
+                                  unsigned int pg_start,
+                                  enum i915_cache_level cache_level);
+};
+#define gtt_total_entries(gtt) ((gtt).total >> PAGE_SHIFT)
+
 #define I915_PPGTT_PD_ENTRIES 512
 #define I915_PPGTT_PT_ENTRIES 1024
 struct i915_hw_ppgtt {
@@ -351,8 +417,18 @@ struct i915_hw_ppgtt {
        unsigned num_pd_entries;
        vm_page_t *pt_pages;
        uint32_t pd_offset;
-       vm_paddr_t *pt_dma_addr;
-       vm_paddr_t scratch_page_dma_addr;
+       dma_addr_t *pt_dma_addr;
+       dma_addr_t scratch_page_dma_addr;
+
+       /* pte functions, mirroring the interface of the global gtt. */
+       void (*clear_range)(struct i915_hw_ppgtt *ppgtt,
+                           unsigned int first_entry,
+                           unsigned int num_entries);
+       void (*insert_entries)(struct i915_hw_ppgtt *ppgtt,
+                              struct sg_table *st,
+                              unsigned int pg_start,
+                              enum i915_cache_level cache_level);
+       void (*cleanup)(struct i915_hw_ppgtt *ppgtt);
 };
 
 
@@ -586,6 +662,9 @@ struct intel_gen6_power_mgmt {
        struct lock hw_lock;
 };
 
+/* defined intel_pm.c */
+extern struct lock mchdev_lock;
+
 struct intel_ilk_power_mgmt {
        u8 cur_delay;
        u8 min_delay;
@@ -626,8 +705,164 @@ struct intel_l3_parity {
        struct work_struct error_work;
 };
 
+struct i915_gem_mm {
+       /** Bridge to intel-gtt-ko */
+       struct intel_gtt *gtt;
+       /** Memory allocator for GTT stolen memory */
+       struct drm_mm stolen;
+       /** Memory allocator for GTT */
+       struct drm_mm gtt_space;
+       /** List of all objects in gtt_space. Used to restore gtt
+        * mappings on resume */
+       struct list_head bound_list;
+       /**
+        * List of objects which are not bound to the GTT (thus
+        * are idle and not used by the GPU) but still have
+        * (presumably uncached) pages still attached.
+        */
+       struct list_head unbound_list;
+
+       /** Usable portion of the GTT for GEM */
+       unsigned long stolen_base; /* limited to low memory (32-bit) */
+
+       int gtt_mtrr;
+
+       /** PPGTT used for aliasing the PPGTT with the GTT */
+       struct i915_hw_ppgtt *aliasing_ppgtt;
+
+       eventhandler_tag inactive_shrinker;
+       bool shrinker_no_lock_stealing;
+
+       /**
+        * List of objects currently involved in rendering.
+        *
+        * Includes buffers having the contents of their GPU caches
+        * flushed, not necessarily primitives.  last_rendering_seqno
+        * represents when the rendering involved will be completed.
+        *
+        * A reference is held on the buffer while on this list.
+        */
+       struct list_head active_list;
+
+       /**
+        * LRU list of objects which are not in the ringbuffer and
+        * are ready to unbind, but are still in the GTT.
+        *
+        * last_rendering_seqno is 0 while an object is in this list.
+        *
+        * A reference is not held on the buffer while on this list,
+        * as merely being GTT-bound shouldn't prevent its being
+        * freed, and we'll pull it off the list in the free path.
+        */
+       struct list_head inactive_list;
+
+       /** LRU list of objects with fence regs on them. */
+       struct list_head fence_list;
+
+       /**
+        * We leave the user IRQ off as much as possible,
+        * but this means that requests will finish and never
+        * be retired once the system goes idle. Set a timer to
+        * fire periodically while the ring is running. When it
+        * fires, go retire requests.
+        */
+       struct delayed_work retire_work;
+
+       /**
+        * Are we in a non-interruptible section of code like
+        * modesetting?
+        */
+       bool interruptible;
+
+       /**
+        * Flag if the X Server, and thus DRM, is not currently in
+        * control of the device.
+        *
+        * This is set between LeaveVT and EnterVT.  It needs to be
+        * replaced with a semaphore.  It also needs to be
+        * transitioned away from for kernel modesetting.
+        */
+       int suspended;
+
+       /** Bit 6 swizzling required for X tiling */
+       uint32_t bit_6_swizzle_x;
+       /** Bit 6 swizzling required for Y tiling */
+       uint32_t bit_6_swizzle_y;
+
+       /* storage for physical objects */
+       struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
+
+       /* accounting, useful for userland debugging */
+       size_t object_memory;
+       u32 object_count;
+};
+
+struct i915_gpu_error {
+       /* For hangcheck timer */
+#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
+#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
+       struct timer_list hangcheck_timer;
+       int hangcheck_count;
+       uint32_t last_acthd[I915_NUM_RINGS];
+       uint32_t prev_instdone[I915_NUM_INSTDONE_REG];
+
+       /* For reset and error_state handling. */
+       struct lock lock;
+       /* Protected by the above dev->gpu_error.lock. */
+       struct drm_i915_error_state *first_error;
+       struct work_struct work;
+
+       unsigned long last_reset;
+
+       /**
+        * State variable and reset counter controlling the reset flow
+        *
+        * Upper bits are for the reset counter.  This counter is used by the
+        * wait_seqno code to race-free noticed that a reset event happened and
+        * that it needs to restart the entire ioctl (since most likely the
+        * seqno it waited for won't ever signal anytime soon).
+        *
+        * This is important for lock-free wait paths, where no contended lock
+        * naturally enforces the correct ordering between the bail-out of the
+        * waiter and the gpu reset work code.
+        *
+        * Lowest bit controls the reset state machine: Set means a reset is in
+        * progress. This state will (presuming we don't have any bugs) decay
+        * into either unset (successful reset) or the special WEDGED value (hw
+        * terminally sour). All waiters on the reset_queue will be woken when
+        * that happens.
+        */
+       atomic_t reset_counter;
+
+       /**
+        * Special values/flags for reset_counter
+        *
+        * Note that the code relies on
+        *      I915_WEDGED & I915_RESET_IN_PROGRESS_FLAG
+        * being true.
+        */
+#define I915_RESET_IN_PROGRESS_FLAG    1
+#define I915_WEDGED                    0xffffffff
+
+       /**
+        * Waitqueue to signal when the reset has completed. Used by clients
+        * that wait for dev_priv->mm.wedged to settle.
+        */
+       wait_queue_head_t reset_queue;
+
+       /* For gpu hang simulation. */
+       unsigned int stop_rings;
+};
+
+enum modeset_restore {
+       MODESET_ON_LID_OPEN,
+       MODESET_DONE,
+       MODESET_SUSPENDED,
+};
+
 typedef struct drm_i915_private {
        struct drm_device *dev;
+       struct kmem_cache *slab;
 
        const struct intel_device_info *info;
 
@@ -652,6 +887,7 @@ typedef struct drm_i915_private {
 
        device_t *gmbus;
 
+
        /** gmbus_mutex protects against concurrent usage of the single hw gmbus
         * controller on different i2c buses. */
        struct lock gmbus_mutex;
@@ -662,9 +898,11 @@ typedef struct drm_i915_private {
         */
        uint32_t gpio_mmio_base;
 
+       wait_queue_head_t gmbus_wait_queue;
+
        struct pci_dev *bridge_dev;
        struct intel_ring_buffer ring[I915_NUM_RINGS];
-       uint32_t next_seqno;
+       uint32_t last_seqno, next_seqno;
 
        drm_dma_handle_t *status_page_dmah;
        struct resource *mch_res;
@@ -675,31 +913,24 @@ typedef struct drm_i915_private {
        /* protects the irq masks */
        struct lock irq_lock;
 
+       /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */
+       struct pm_qos_request pm_qos;
+
        /* DPIO indirect register protection */
-       struct spinlock dpio_lock;
+       struct lock dpio_lock;
 
        /** Cached value of IMR to avoid reads in updating the bitfield */
        u32 pipestat[2];
        u32 irq_mask;
        u32 gt_irq_mask;
-       u32 pch_irq_mask;
 
        u32 hotplug_supported_mask;
        struct work_struct hotplug_work;
+       bool enable_hotplug_processing;
 
        int num_pipe;
        int num_pch_pll;
 
-       /* For hangcheck timer */
-#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
-#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
-       struct timer_list hangcheck_timer;
-       int hangcheck_count;
-       uint32_t last_acthd[I915_NUM_RINGS];
-       uint32_t prev_instdone[I915_NUM_INSTDONE_REG];
-
-       unsigned int stop_rings;
-
        unsigned long cfb_size;
        unsigned int cfb_fb;
        enum plane cfb_plane;
@@ -710,7 +941,7 @@ typedef struct drm_i915_private {
 
        /* overlay */
        struct intel_overlay *overlay;
-       bool sprite_scaling_enabled;
+       unsigned int sprite_scaling_enabled;
 
        /* LVDS info */
        int backlight_level;  /* restore backlight to this value */
@@ -728,7 +959,6 @@ typedef struct drm_i915_private {
        unsigned int fdi_rx_polarity_inverted:1;
        int lvds_ssc_freq;
        unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */
-       unsigned int lvds_val; /* used for checking LVDS channel mode */
        struct {
                int rate;
                int lanes;
@@ -749,11 +979,6 @@ typedef struct drm_i915_private {
 
        unsigned int fsb_freq, mem_freq, is_ddr3;
 
-       struct lock error_lock;
-       /* Protected by dev->error_lock. */
-       struct drm_i915_error_state *first_error;
-       struct work_struct error_work;
-       struct completion error_completion;
        struct workqueue_struct *wq;
 
        /* Display functions */
@@ -765,117 +990,12 @@ typedef struct drm_i915_private {
 
        unsigned long quirks;
 
-       /* Register state */
-       bool modeset_on_lid;
+       enum modeset_restore modeset_restore;
+       struct lock modeset_restore_lock;
 
-       struct {
-               /** Bridge to intel-gtt-ko */
-               struct intel_gtt *gtt;
-               /** Memory allocator for GTT stolen memory */
-               struct drm_mm stolen;
-               /** Memory allocator for GTT */
-               struct drm_mm gtt_space;
-               /** List of all objects in gtt_space. Used to restore gtt
-                * mappings on resume */
-               struct list_head bound_list;
-               /**
-                * List of objects which are not bound to the GTT (thus
-                * are idle and not used by the GPU) but still have
-                * (presumably uncached) pages still attached.
-                */
-               struct list_head unbound_list;
-
-               /** Usable portion of the GTT for GEM */
-               unsigned long gtt_start;
-               unsigned long gtt_mappable_end;
-               unsigned long gtt_end;
-               unsigned long stolen_base; /* limited to low memory (32-bit) */
-
-               struct io_mapping *gtt_mapping;
-               phys_addr_t gtt_base_addr;
-               int gtt_mtrr;
-
-               /** PPGTT used for aliasing the PPGTT with the GTT */
-               struct i915_hw_ppgtt *aliasing_ppgtt;
-
-               bool shrinker_no_lock_stealing;
-
-               /**
-                * List of objects currently involved in rendering.
-                *
-                * Includes buffers having the contents of their GPU caches
-                * flushed, not necessarily primitives.  last_rendering_seqno
-                * represents when the rendering involved will be completed.
-                *
-                * A reference is held on the buffer while on this list.
-                */
-               struct list_head active_list;
-
-               /**
-                * LRU list of objects which are not in the ringbuffer and
-                * are ready to unbind, but are still in the GTT.
-                *
-                * last_rendering_seqno is 0 while an object is in this list.
-                *
-                * A reference is not held on the buffer while on this list,
-                * as merely being GTT-bound shouldn't prevent its being
-                * freed, and we'll pull it off the list in the free path.
-                */
-               struct list_head inactive_list;
-
-               /** LRU list of objects with fence regs on them. */
-               struct list_head fence_list;
-
-               /**
-                * We leave the user IRQ off as much as possible,
-                * but this means that requests will finish and never
-                * be retired once the system goes idle. Set a timer to
-                * fire periodically while the ring is running. When it
-                * fires, go retire requests.
-                */
-               struct delayed_work retire_work;
-
-               /**
-                * Are we in a non-interruptible section of code like
-                * modesetting?
-                */
-               bool interruptible;
-
-               /**
-                * Flag if the X Server, and thus DRM, is not currently in
-                * control of the device.
-                *
-                * This is set between LeaveVT and EnterVT.  It needs to be
-                * replaced with a semaphore.  It also needs to be
-                * transitioned away from for kernel modesetting.
-                */
-               int suspended;
-
-               /**
-                * Flag if the hardware appears to be wedged.
-                *
-                * This is set when attempts to idle the device timeout.
-                * It prevents command submission from occurring and makes
-                * every pending request fail
-                */
-               atomic_t wedged;
-
-               /** Bit 6 swizzling required for X tiling */
-               uint32_t bit_6_swizzle_x;
-               /** Bit 6 swizzling required for Y tiling */
-               uint32_t bit_6_swizzle_y;
-
-               /* storage for physical objects */
-               struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
-
-               /* accounting, useful for userland debugging */
-               size_t gtt_total;
-               size_t mappable_gtt_total;
-               size_t object_memory;
-               u32 object_count;
-
-               eventhandler_tag i915_lowmem;
-       } mm;
+       struct i915_gtt gtt;
+
+       struct i915_gem_mm mm;
 
        /* Kernel Modesetting */
 
@@ -917,7 +1037,7 @@ typedef struct drm_i915_private {
        struct drm_mm_node *compressed_fb;
        struct drm_mm_node *compressed_llb;
 
-       unsigned long last_gpu_reset;
+       struct i915_gpu_error gpu_error;
 
        /* list of fbdev register on this device */
        struct intel_fbdev *fbdev;
@@ -957,11 +1077,7 @@ enum hdmi_force_audio {
        HDMI_AUDIO_ON,                  /* force turn on HDMI audio */
 };
 
-enum i915_cache_level {
-       I915_CACHE_NONE = 0,
-       I915_CACHE_LLC,
-       I915_CACHE_LLC_MLC, /* gen6+, in docs at least! */
-};
+#define I915_GTT_RESERVED ((struct drm_mm_node *)0x1)
 
 struct drm_i915_gem_object_ops {
        /* Interface between the GEM object and its backing storage.
@@ -988,6 +1104,8 @@ struct drm_i915_gem_object {
 
        /** Current space allocated to this object in the GTT, if any. */
        struct drm_mm_node *gtt_space;
+       /** Stolen memory for this object, instead of being backed by shmem. */
+       struct drm_mm_node *stolen;
        struct list_head gtt_list;
 
        /** This object's place on the active/inactive lists */
@@ -1109,13 +1227,6 @@ struct drm_i915_gem_object {
 
        /** for phy allocated objects */
        struct drm_i915_gem_phys_object *phys_obj;
-
-       /**
-        * Number of crtcs where this object is currently the fb, but
-        * will be page flipped away on the next vblank.  When it
-        * reaches 0, dev_priv->pending_flip_queue will be woken up.
-        */
-       atomic_t pending_flip;
 };
 #define to_gem_object(obj) (&((struct drm_i915_gem_object *)(obj))->base)
 
@@ -1240,6 +1351,8 @@ struct drm_i915_file_private {
 
 #define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5)
 
+#define HAS_DDI(dev)           (IS_HASWELL(dev))
+
 #define INTEL_PCH_DEVICE_ID_MASK               0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE           0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE           0x1c00
@@ -1259,6 +1372,8 @@ struct drm_i915_file_private {
 
 #define GT_FREQUENCY_MULTIPLIER 50
 
+#include "i915_trace.h"
+
 /**
  * RC6 is a special power stage which allows the GPU to enter an very
  * low-voltage mode when idle, using down to 0V while at this stage.  This
@@ -1280,9 +1395,7 @@ struct drm_i915_file_private {
 #define INTEL_RC6p_ENABLE                      (1<<1)
 #define INTEL_RC6pp_ENABLE                     (1<<2)
 
-extern int intel_iommu_enabled;
 extern struct drm_ioctl_desc i915_ioctls[];
-extern struct drm_driver i915_driver_info;
 extern struct cdev_pager_ops i915_gem_pager_ops;
 extern int i915_max_ioctl;
 extern unsigned int i915_fbpercrtc __always_unused;
@@ -1295,9 +1408,13 @@ extern int i915_panel_use_ssc __read_mostly;
 extern int i915_vbt_sdvo_panel_type __read_mostly;
 extern int i915_enable_rc6 __read_mostly;
 extern int i915_enable_fbc __read_mostly;
-extern int i915_enable_hangcheck;
+extern bool i915_enable_hangcheck __read_mostly;
 extern int i915_enable_ppgtt __read_mostly;
 extern unsigned int i915_preliminary_hw_support __read_mostly;
+extern int i915_disable_power_well __read_mostly;
+
+extern int i915_master_create(struct drm_device *dev, struct drm_master *master);
+extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master);
 
                                /* i915_dma.c */
 void i915_update_dri1_breadcrumb(struct drm_device *dev);
@@ -1332,6 +1449,7 @@ void i915_hangcheck_elapsed(unsigned long data);
 void i915_handle_error(struct drm_device *dev, bool wedged);
 
 extern void intel_irq_init(struct drm_device *dev);
+extern void intel_hpd_init(struct drm_device *dev);
 extern void intel_gt_init(struct drm_device *dev);
 extern void intel_gt_reset(struct drm_device *dev);
 
@@ -1400,22 +1518,41 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv);
 void i915_gem_load(struct drm_device *dev);
-void i915_gem_unload(struct drm_device *dev);
+void *i915_gem_object_alloc(struct drm_device *dev);
+void i915_gem_object_free(struct drm_i915_gem_object *obj);
 int i915_gem_init_object(struct drm_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
                         const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
                                                  size_t size);
 void i915_gem_free_object(struct drm_gem_object *obj);
+
 int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
                                     uint32_t alignment,
                                     bool map_and_fenceable,
                                     bool nonblocking);
 void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
+int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 
+int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+static inline struct vm_page *i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
+{
+       return obj->pages[n];
+}
+static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+{
+       BUG_ON(obj->pages == NULL);
+       obj->pages_pin_count++;
+}
+static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+{
+       BUG_ON(obj->pages_pin_count == 0);
+       obj->pages_pin_count--;
+}
+
 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
 int i915_gem_object_sync(struct drm_i915_gem_object *obj,
                         struct intel_ring_buffer *to);
@@ -1438,8 +1575,8 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
        return (int32_t)(seq1 - seq2) >= 0;
 }
 
-extern int i915_gem_get_seqno(struct drm_device *dev, u32 *seqno);
-
+int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno);
+int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
 int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
 
@@ -1465,8 +1602,18 @@ i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
 
 void i915_gem_retire_requests(struct drm_device *dev);
 void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring);
-int __must_check i915_gem_check_wedge(struct drm_i915_private *dev_priv,
+int __must_check i915_gem_check_wedge(struct i915_gpu_error *error,
                                      bool interruptible);
+static inline bool i915_reset_in_progress(struct i915_gpu_error *error)
+{
+       return unlikely(atomic_read(&error->reset_counter)
+                       & I915_RESET_IN_PROGRESS_FLAG);
+}
+
+static inline bool i915_terminally_wedged(struct i915_gpu_error *error)
+{
+       return atomic_read(&error->reset_counter) == I915_WEDGED;
+}
 
 void i915_gem_reset(struct drm_device *dev);
 void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
@@ -1508,13 +1655,22 @@ void i915_gem_free_all_phys_object(struct drm_device *dev);
 void i915_gem_release(struct drm_device *dev, struct drm_file *file);
 
 uint32_t
-i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
-                                   uint32_t size,
-                                   int tiling_mode);
+i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode);
+uint32_t
+i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
+                           int tiling_mode, bool fenced);
 
 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                                    enum i915_cache_level cache_level);
 
+#if 0
+struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
+                               struct dma_buf *dma_buf);
+
+struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
+                               struct drm_gem_object *gem_obj, int flags);
+#endif
+
 /* i915_gem_context.c */
 void i915_gem_context_init(struct drm_device *dev);
 void i915_gem_context_fini(struct drm_device *dev);
@@ -1527,7 +1683,6 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
                                   struct drm_file *file);
 
 /* i915_gem_gtt.c */
-int __must_check i915_gem_init_aliasing_ppgtt(struct drm_device *dev);
 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev);
 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
                            struct drm_i915_gem_object *obj,
@@ -1541,12 +1696,10 @@ void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
                                enum i915_cache_level cache_level);
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
-void i915_gem_init_global_gtt(struct drm_device *dev,
-                             unsigned long start,
-                             unsigned long mappable_end,
-                             unsigned long end);
+void i915_gem_init_global_gtt(struct drm_device *dev);
+void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
+                              unsigned long mappable_end, unsigned long end);
 int i915_gem_gtt_init(struct drm_device *dev);
-void i915_gem_gtt_fini(struct drm_device *dev);
 static inline void i915_gem_chipset_flush(struct drm_device *dev)
 {
        if (INTEL_INFO(dev)->gen < 6)
@@ -1564,14 +1717,29 @@ int i915_gem_evict_everything(struct drm_device *dev);
 
 /* i915_gem_stolen.c */
 int i915_gem_init_stolen(struct drm_device *dev);
+int i915_gem_stolen_setup_compression(struct drm_device *dev, int size);
+void i915_gem_stolen_cleanup_compression(struct drm_device *dev);
 void i915_gem_cleanup_stolen(struct drm_device *dev);
+struct drm_i915_gem_object *
+i915_gem_object_create_stolen(struct drm_device *dev, u32 size);
+void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj);
 
 /* i915_gem_tiling.c */
+inline static bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
+{
+       drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
+
+       return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
+               obj->tiling_mode != I915_TILING_NONE;
+}
+
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
 void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
 void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
 
 /* i915_gem_debug.c */
+void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
+                         const char *where, uint32_t mark);
 #if WATCH_LISTS
 int i915_verify_lists(struct drm_device *dev);
 #else
@@ -1588,6 +1756,10 @@ void i915_debugfs_cleanup(struct drm_minor *minor);
 extern int i915_save_state(struct drm_device *dev);
 extern int i915_restore_state(struct drm_device *dev);
 
+/* i915_ums.c */
+void i915_save_display_reg(struct drm_device *dev);
+void i915_restore_display_reg(struct drm_device *dev);
+
 /* i915_sysfs.c */
 void i915_setup_sysfs(struct drm_device *dev_priv);
 void i915_teardown_sysfs(struct drm_device *dev_priv);
@@ -1602,8 +1774,8 @@ static inline bool intel_gmbus_is_port_valid(unsigned port)
 
 extern struct device *intel_gmbus_get_adapter(
                struct drm_i915_private *dev_priv, unsigned port);
-extern void intel_gmbus_set_speed(device_t idev, int speed);
-extern void intel_gmbus_force_bit(device_t idev, bool force_bit);
+extern void intel_gmbus_set_speed(struct device *adapter, int speed);
+extern void intel_gmbus_force_bit(struct device *adapter, bool force_bit);
 static inline bool intel_gmbus_is_forced_bit(struct device *adapter)
 {
        struct intel_iic_softc *sc;
@@ -1646,6 +1818,7 @@ extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
 extern void intel_modeset_setup_hw_state(struct drm_device *dev,
                                         bool force_restore);
+extern void i915_redisable_vga(struct drm_device *dev);
 extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
 extern void intel_init_pch_refclk(struct drm_device *dev);
@@ -1658,17 +1831,6 @@ extern bool i915_semaphore_is_enabled(struct drm_device *dev);
 int i915_reg_read_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file);
 
-extern void intel_overlay_print_error_state(struct sbuf *m,
-    struct intel_overlay_error_state *error);
-extern void intel_display_print_error_state(struct sbuf *m,
-    struct drm_device *dev, struct intel_display_error_state *error);
-
-static inline void
-trace_i915_reg_rw(boolean_t rw, int reg, uint64_t val, int sz)
-{
-       return;
-}
-
 const struct intel_device_info *i915_get_device_id(int device);
 
 /* overlay */
@@ -1730,5 +1892,19 @@ __i915_write(64, 64)
 #define POSTING_READ(reg)      (void)I915_READ_NOTRACE(reg)
 #define POSTING_READ16(reg)    (void)I915_READ16_NOTRACE(reg)
 
+/* "Broadcast RGB" property */
+#define INTEL_BROADCAST_RGB_AUTO 0
+#define INTEL_BROADCAST_RGB_FULL 1
+#define INTEL_BROADCAST_RGB_LIMITED 2
+
+static inline uint32_t i915_vgacntrl_reg(struct drm_device *dev)
+{
+       if (HAS_PCH_SPLIT(dev))
+               return CPU_VGACNTRL;
+       else if (IS_VALLEYVIEW(dev))
+               return VLV_VGACNTRL;
+       else
+               return VGACNTRL;
+}
 
 #endif
index 0904f94..e02738d 100644 (file)
@@ -82,10 +82,8 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
                                         struct drm_i915_fence_reg *fence,
                                         bool enable);
 
-static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
-    int tiling_mode);
-static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
-    uint32_t size, int tiling_mode);
+static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
+
 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
 
@@ -103,7 +101,6 @@ static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
 
 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
-static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
 static void i915_gem_reset_fences(struct drm_device *dev);
 static void i915_gem_lowmem(void *arg);
 
@@ -123,13 +120,13 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 }
 
 static int
-i915_gem_wait_for_error(struct drm_device *dev)
+i915_gem_wait_for_error(struct i915_gpu_error *error)
 {
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       struct completion *x = &dev_priv->error_completion;
        int ret;
 
-       if (!atomic_read(&dev_priv->mm.wedged))
+#define EXIT_COND (!i915_reset_in_progress(error) || \
+                  i915_terminally_wedged(error))
+       if (EXIT_COND)
                return 0;
 
        /*
@@ -137,36 +134,30 @@ i915_gem_wait_for_error(struct drm_device *dev)
         * userspace. If it takes that long something really bad is going on and
         * we should simply try to bail out and fail as gracefully as possible.
         */
-       ret = wait_for_completion_interruptible_timeout(x, 10*hz);
+       ret = wait_event_interruptible_timeout(error->reset_queue,
+                                              EXIT_COND,
+                                              10*HZ);
        if (ret == 0) {
                DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
                return -EIO;
        } else if (ret < 0) {
                return ret;
        }
+#undef EXIT_COND
 
-       if (atomic_read(&dev_priv->mm.wedged)) {
-               /* GPU is hung, bump the completion count to account for
-                * the token we just consumed so that we never hit zero and
-                * end up waiting upon a subsequent completion event that
-                * will never happen.
-                */
-               lockmgr(&x->wait.lock, LK_EXCLUSIVE);
-               x->done++;
-               lockmgr(&x->wait.lock, LK_RELEASE);
-       }
        return 0;
 }
 
 int i915_mutex_lock_interruptible(struct drm_device *dev)
 {
+       struct drm_i915_private *dev_priv = dev->dev_private;
        int ret;
 
-       ret = i915_gem_wait_for_error(dev);
+       ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
        if (ret)
                return ret;
 
-       ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
+       ret = lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_SLEEPFAIL);
        if (ret)
                return -EINTR;
 
@@ -197,10 +188,10 @@ i915_gem_init_ioctl(struct drm_device *dev, void *data,
        if (INTEL_INFO(dev)->gen >= 5)
                return -ENODEV;
 
-       lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
-       i915_gem_init_global_gtt(dev, args->gtt_start,
-                                args->gtt_end, args->gtt_end);
-       lockmgr(&dev->dev_lock, LK_RELEASE);
+       mutex_lock(&dev->struct_mutex);
+       i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
+                                 args->gtt_end);
+       mutex_unlock(&dev->struct_mutex);
 
        return 0;
 }
@@ -215,13 +206,13 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
        size_t pinned;
 
        pinned = 0;
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
                if (obj->pin_count)
                        pinned += obj->gtt_space->size;
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
-       args->aper_size = dev_priv->mm.gtt_total;
+       args->aper_size = dev_priv->gtt.total;
        args->aper_available_size = args->aper_size - pinned;
 
        return 0;
@@ -295,14 +286,6 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
                               args->size, &args->handle);
 }
 
-static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
-{
-       drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
-
-       return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
-               obj->tiling_mode != I915_TILING_NONE;
-}
-
 static inline void vm_page_reference(vm_page_t m)
 {
        vm_page_flag_set(m, PG_REFERENCED);
@@ -413,9 +396,101 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 out:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
+       return ret;
+}
+
+#if 0
+/* This is the fast write path which cannot handle
+ * page faults in the source data
+ */
+
+static inline int
+fast_user_write(struct io_mapping *mapping,
+               loff_t page_base, int page_offset,
+               char __user *user_data,
+               int length)
+{
+       void __iomem *vaddr_atomic;
+       void *vaddr;
+       unsigned long unwritten;
+
+       vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
+       /* We can use the cpu mem copy function because this is X86. */
+       vaddr = (void __force*)vaddr_atomic + page_offset;
+       unwritten = __copy_from_user_inatomic_nocache(vaddr,
+                                                     user_data, length);
+       io_mapping_unmap_atomic(vaddr_atomic);
+       return unwritten;
+}
+
+/**
+ * This is the fast pwrite path, where we copy the data directly from the
+ * user into the GTT, uncached.
+ */
+static int
+i915_gem_gtt_pwrite_fast(struct drm_device *dev,
+                        struct drm_i915_gem_object *obj,
+                        struct drm_i915_gem_pwrite *args,
+                        struct drm_file *file)
+{
+       drm_i915_private_t *dev_priv = dev->dev_private;
+       ssize_t remain;
+       loff_t offset, page_base;
+       char __user *user_data;
+       int page_offset, page_length, ret;
+
+       ret = i915_gem_object_pin(obj, 0, true, true);
+       if (ret)
+               goto out;
+
+       ret = i915_gem_object_set_to_gtt_domain(obj, true);
+       if (ret)
+               goto out_unpin;
+
+       ret = i915_gem_object_put_fence(obj);
+       if (ret)
+               goto out_unpin;
+
+       user_data = (char __user *) (uintptr_t) args->data_ptr;
+       remain = args->size;
+
+       offset = obj->gtt_offset + args->offset;
+
+       while (remain > 0) {
+               /* Operation in this page
+                *
+                * page_base = page offset within aperture
+                * page_offset = offset within page
+                * page_length = bytes to copy for this page
+                */
+               page_base = offset & PAGE_MASK;
+               page_offset = offset_in_page(offset);
+               page_length = remain;
+               if ((page_offset + remain) > PAGE_SIZE)
+                       page_length = PAGE_SIZE - page_offset;
+
+               /* If we get a fault while copying data, then (presumably) our
+                * source page isn't available.  Return the error and we'll
+                * retry in the slow path.
+                */
+               if (fast_user_write(dev_priv->gtt.mappable, page_base,
+                                   page_offset, user_data, page_length)) {
+                       ret = -EFAULT;
+                       goto out_unpin;
+               }
+
+               remain -= page_length;
+               user_data += page_length;
+               offset += page_length;
+       }
+
+out_unpin:
+       i915_gem_object_unpin(obj);
+out:
        return ret;
 }
+#endif
 
 static int
 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
@@ -580,7 +655,7 @@ out_unpin:
 out:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 unlocked:
        vm_page_unhold_pages(ma, npages);
 free_ma:
@@ -589,25 +664,17 @@ free_ma:
 }
 
 int
-i915_gem_check_wedge(struct drm_i915_private *dev_priv,
+i915_gem_check_wedge(struct i915_gpu_error *error,
                     bool interruptible)
 {
-       if (atomic_read(&dev_priv->mm.wedged)) {
-               struct completion *x = &dev_priv->error_completion;
-               bool recovery_complete;
-
-               /* Give the error handler a chance to run. */
-               lockmgr(&x->wait.lock, LK_EXCLUSIVE);
-               recovery_complete = x->done > 0;
-               lockmgr(&x->wait.lock, LK_RELEASE);
-
+       if (i915_reset_in_progress(error)) {
                /* Non-interruptible callers can't handle -EAGAIN, hence return
                 * -EIO unconditionally for these. */
                if (!interruptible)
                        return -EIO;
 
-               /* Recovery complete, but still wedged means reset failure. */
-               if (recovery_complete)
+               /* Recovery complete, but the reset failed ... */
+               if (i915_terminally_wedged(error))
                        return -EIO;
 
                return -EAGAIN;
@@ -638,13 +705,22 @@ i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
  * __wait_seqno - wait until execution of seqno has finished
  * @ring: the ring expected to report seqno
  * @seqno: duh!
+ * @reset_counter: reset sequence associated with the given seqno
  * @interruptible: do an interruptible wait (normally yes)
  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
  *
+ * Note: It is of utmost importance that the passed in seqno and reset_counter
+ * values have been read by the caller in an smp safe manner. Where read-side
+ * locks are involved, it is sufficient to read the reset_counter before
+ * unlocking the lock that protects the seqno. For lockless tricks, the
+ * reset_counter _must_ be read before, and an appropriate smp_rmb must be
+ * inserted.
+ *
  * Returns 0 if the seqno was found within the alloted time. Else returns the
  * errno with remaining time filled in timeout argument.
  */
 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
+                       unsigned reset_counter,
                        bool interruptible, struct timespec *timeout)
 {
        drm_i915_private_t *dev_priv = ring->dev->dev_private;
@@ -672,7 +748,8 @@ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
 
 #define EXIT_COND \
        (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
-       atomic_read(&dev_priv->mm.wedged))
+        i915_reset_in_progress(&dev_priv->gpu_error) || \
+        reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
        do {
                if (interruptible)
                        end = wait_event_interruptible_timeout(ring->irq_queue,
@@ -682,7 +759,14 @@ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
                        end = wait_event_timeout(ring->irq_queue, EXIT_COND,
                                                 timeout_jiffies);
 
-               ret = i915_gem_check_wedge(dev_priv, interruptible);
+               /* We need to check whether any gpu reset happened in between
+                * the caller grabbing the seqno and now ... */
+               if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
+                       end = -EAGAIN;
+
+               /* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
+                * gone. */
+               ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
                if (ret)
                        end = ret;
        } while (end == 0 && wait_forever);
@@ -727,7 +811,7 @@ i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
        DRM_LOCK_ASSERT(dev);
        BUG_ON(seqno == 0);
 
-       ret = i915_gem_check_wedge(dev_priv, interruptible);
+       ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
        if (ret)
                return ret;
 
@@ -735,7 +819,9 @@ i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
        if (ret)
                return ret;
 
-       return __wait_seqno(ring, seqno, interruptible, NULL);
+       return __wait_seqno(ring, seqno,
+                           atomic_read(&dev_priv->gpu_error.reset_counter),
+                           interruptible, NULL);
 }
 
 /**
@@ -782,6 +868,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
        struct drm_device *dev = obj->base.dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_ring_buffer *ring = obj->ring;
+       unsigned reset_counter;
        u32 seqno;
        int ret;
 
@@ -792,7 +879,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
        if (seqno == 0)
                return 0;
 
-       ret = i915_gem_check_wedge(dev_priv, true);
+       ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
        if (ret)
                return ret;
 
@@ -800,9 +887,10 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
        if (ret)
                return ret;
 
-       DRM_UNLOCK(dev);
-       ret = __wait_seqno(ring, seqno, true, NULL);
-       DRM_LOCK(dev);
+       reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
+       mutex_unlock(&dev->struct_mutex);
+       ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
+       mutex_lock(&dev->struct_mutex);
 
        i915_gem_retire_requests_ring(ring);
 
@@ -879,7 +967,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 unref:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -909,7 +997,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -984,6 +1072,93 @@ out:
  * suffer if the GTT working set is large or there are few fence registers
  * left.
  */
+#if 0
+int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+       struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
+       struct drm_device *dev = obj->base.dev;
+       drm_i915_private_t *dev_priv = dev->dev_private;
+       pgoff_t page_offset;
+       unsigned long pfn;
+       int ret = 0;
+       bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
+
+       /* We don't use vmf->pgoff since that has the fake offset */
+       page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
+               PAGE_SHIFT;
+
+       ret = i915_mutex_lock_interruptible(dev);
+       if (ret)
+               goto out;
+
+       trace_i915_gem_object_fault(obj, page_offset, true, write);
+
+       /* Access to snoopable pages through the GTT is incoherent. */
+       if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
+               ret = -EINVAL;
+               goto unlock;
+       }
+
+       /* Now bind it into the GTT if needed */
+       ret = i915_gem_object_pin(obj, 0, true, false);
+       if (ret)
+               goto unlock;
+
+       ret = i915_gem_object_set_to_gtt_domain(obj, write);
+       if (ret)
+               goto unpin;
+
+       ret = i915_gem_object_get_fence(obj);
+       if (ret)
+               goto unpin;
+
+       obj->fault_mappable = true;
+
+       pfn = ((dev_priv->gtt.mappable_base + obj->gtt_offset) >> PAGE_SHIFT) +
+               page_offset;
+
+       /* Finally, remap it using the new GTT offset */
+       ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
+unpin:
+       i915_gem_object_unpin(obj);
+unlock:
+       mutex_unlock(&dev->struct_mutex);
+out:
+       switch (ret) {
+       case -EIO:
+               /* If this -EIO is due to a gpu hang, give the reset code a
+                * chance to clean up the mess. Otherwise return the proper
+                * SIGBUS. */
+               if (i915_terminally_wedged(&dev_priv->gpu_error))
+                       return VM_FAULT_SIGBUS;
+       case -EAGAIN:
+               /* Give the error handler a chance to run and move the
+                * objects off the GPU active list. Next time we service the
+                * fault, we should be able to transition the page into the
+                * GTT without touching the GPU (and so avoid further
+                * EIO/EGAIN). If the GPU is wedged, then there is no issue
+                * with coherency, just lost writes.
+                */
+               set_need_resched();
+       case 0:
+       case -ERESTARTSYS:
+       case -EINTR:
+       case -EBUSY:
+               /*
+                * EBUSY is ok: this just means that another thread
+                * already did the job.
+                */
+               return VM_FAULT_NOPAGE;
+       case -ENOMEM:
+               return VM_FAULT_OOM;
+       case -ENOSPC:
+               return VM_FAULT_SIGBUS;
+       default:
+               WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
+               return VM_FAULT_SIGBUS;
+       }
+}
+#endif
 
 /**
  * i915_gem_release_mmap - remove physical page mappings
@@ -1027,7 +1202,7 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj)
        obj->fault_mappable = false;
 }
 
-static uint32_t
+uint32_t
 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 {
        uint32_t gtt_size;
@@ -1055,17 +1230,16 @@ i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
  * Return the required GTT alignment for an object, taking into account
  * potential fence register mapping.
  */
-static uint32_t
-i915_gem_get_gtt_alignment(struct drm_device *dev,
-                          uint32_t size,
-                          int tiling_mode)
+uint32_t
+i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
+                          int tiling_mode, bool fenced)
 {
 
        /*
         * Minimum alignment is 4k (GTT page size), but might be greater
         * if a fence register is needed for the object.
         */
-       if (INTEL_INFO(dev)->gen >= 4 ||
+       if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
            tiling_mode == I915_TILING_NONE)
                return 4096;
 
@@ -1076,35 +1250,6 @@ i915_gem_get_gtt_alignment(struct drm_device *dev,
        return i915_gem_get_gtt_size(dev, size, tiling_mode);
 }
 
-/**
- * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
- *                                      unfenced object
- * @dev: the device
- * @size: size of the object
- * @tiling_mode: tiling mode of the object
- *
- * Return the required GTT alignment for an object, only taking into account
- * unfenced tiled surface requirements.
- */
-uint32_t
-i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
-                                   uint32_t size,
-                                   int tiling_mode)
-{
-       /*
-        * Minimum alignment is 4k (GTT page size) for sane hw.
-        */
-       if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
-           tiling_mode == I915_TILING_NONE)
-               return 4096;
-
-       /* Previous hardware however needs to be aligned to a power-of-two
-        * tile height. The simplest method for determining this is to reuse
-        * the power-of-tile object size.
-        */
-       return i915_gem_get_gtt_size(dev, size, tiling_mode);
-}
-
 int
 i915_gem_mmap_gtt(struct drm_file *file,
                  struct drm_device *dev,
@@ -1125,7 +1270,7 @@ i915_gem_mmap_gtt(struct drm_file *file,
                goto unlock;
        }
 
-       if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
+       if (obj->base.size > dev_priv->gtt.mappable_end) {
                ret = -E2BIG;
                goto out;
        }
@@ -1145,7 +1290,7 @@ i915_gem_mmap_gtt(struct drm_file *file,
 out:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -1225,9 +1370,37 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
        obj->pages = NULL;
 }
 
+int
+i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
+{
+       const struct drm_i915_gem_object_ops *ops = obj->ops;
+
+       if (obj->pages == NULL)
+               return 0;
+
+       BUG_ON(obj->gtt_space);
+
+       if (obj->pages_pin_count)
+               return -EBUSY;
+
+       /* ->put_pages might need to allocate memory for the bit17 swizzle
+        * array, hence protect them from being reaped by removing them from gtt
+        * lists early. */
+       list_del(&obj->gtt_list);
+
+       ops->put_pages(obj);
+       obj->pages = NULL;
+
+       if (i915_gem_object_is_purgeable(obj))
+               i915_gem_object_truncate(obj);
+
+       return 0;
+}
+
 static int
 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 {
+       struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
        struct drm_device *dev;
        vm_object_t vm_obj;
        int page_count, i, j;
@@ -1244,8 +1417,10 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 
        for (i = 0; i < page_count; i++) {
                page = shmem_read_mapping_page(vm_obj, i);
-               if (IS_ERR(page))
+               if (IS_ERR(page)) {
+                       i915_gem_purge(dev_priv, page_count);
                        goto err_pages;
+               }
 
                obj->pages[i] = page;
        }
@@ -1269,6 +1444,38 @@ err_pages:
        return (-EIO);
 }
 
+/* Ensure that the associated pages are gathered from the backing storage
+ * and pinned into our object. i915_gem_object_get_pages() may be called
+ * multiple times before they are released by a single call to
+ * i915_gem_object_put_pages() - once the pages are no longer referenced
+ * either as a result of memory pressure (reaping pages under the shrinker)
+ * or as the object is itself released.
+ */
+int
+i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+{
+       struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+       const struct drm_i915_gem_object_ops *ops = obj->ops;
+       int ret;
+
+       if (obj->pages)
+               return 0;
+
+       if (obj->madv != I915_MADV_WILLNEED) {
+               DRM_ERROR("Attempting to obtain a purgeable object\n");
+               return -EINVAL;
+       }
+
+       BUG_ON(obj->pages_pin_count);
+
+       ret = ops->get_pages(obj);
+       if (ret)
+               return ret;
+
+       list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
+       return 0;
+}
+
 void
 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
                               struct intel_ring_buffer *ring)
@@ -1334,30 +1541,24 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 }
 
 static int
-i915_gem_handle_seqno_wrap(struct drm_device *dev)
+i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_ring_buffer *ring;
        int ret, i, j;
 
-       /* The hardware uses various monotonic 32-bit counters, if we
-        * detect that they will wraparound we need to idle the GPU
-        * and reset those counters.
-        */
-       ret = 0;
+       /* Carefully retire all requests without writing to the rings */
        for_each_ring(ring, dev_priv, i) {
-               for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
-                       ret |= ring->sync_seqno[j] != 0;
+               ret = intel_ring_idle(ring);
+               if (ret)
+                       return ret;
        }
-       if (ret == 0)
-               return ret;
-
-       ret = i915_gpu_idle(dev);
-       if (ret)
-               return ret;
-
        i915_gem_retire_requests(dev);
+
+       /* Finally reset hw state */
        for_each_ring(ring, dev_priv, i) {
+               intel_ring_init_seqno(ring, seqno);
+
                for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
                        ring->sync_seqno[j] = 0;
        }
@@ -1365,6 +1566,32 @@ i915_gem_handle_seqno_wrap(struct drm_device *dev)
        return 0;
 }
 
+int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       int ret;
+
+       if (seqno == 0)
+               return -EINVAL;
+
+       /* HWS page needs to be set less than what we
+        * will inject to ring
+        */
+       ret = i915_gem_init_seqno(dev, seqno - 1);
+       if (ret)
+               return ret;
+
+       /* Carefully set the last_seqno value so that wrap
+        * detection still works
+        */
+       dev_priv->next_seqno = seqno;
+       dev_priv->last_seqno = seqno - 1;
+       if (dev_priv->last_seqno == 0)
+               dev_priv->last_seqno--;
+
+       return 0;
+}
+
 int
 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
 {
@@ -1372,14 +1599,14 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
 
        /* reserve 0 for non-seqno */
        if (dev_priv->next_seqno == 0) {
-               int ret = i915_gem_handle_seqno_wrap(dev);
+               int ret = i915_gem_init_seqno(dev, 0);
                if (ret)
                        return ret;
 
                dev_priv->next_seqno = 1;
        }
 
-       *seqno = dev_priv->next_seqno++;
+       *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
        return 0;
 }
 
@@ -1445,7 +1672,7 @@ i915_add_request(struct intel_ring_buffer *ring,
 
        if (!dev_priv->mm.suspended) {
                if (i915_enable_hangcheck) {
-                       mod_timer(&dev_priv->hangcheck_timer,
+                       mod_timer(&dev_priv->gpu_error.hangcheck_timer,
                                  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
                }
                if (was_empty) {
@@ -1620,6 +1847,49 @@ i915_gem_retire_requests(struct drm_device *dev)
                i915_gem_retire_requests_ring(ring);
 }
 
+static long
+__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
+                 bool purgeable_only)
+{
+       struct drm_i915_gem_object *obj, *next;
+       long count = 0;
+
+       list_for_each_entry_safe(obj, next,
+                                &dev_priv->mm.unbound_list,
+                                gtt_list) {
+#if 0
+               if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
+                   i915_gem_object_put_pages(obj) == 0) {
+                       count += obj->base.size >> PAGE_SHIFT;
+                       if (count >= target)
+                               return count;
+               }
+#endif
+       }
+
+       list_for_each_entry_safe(obj, next,
+                                &dev_priv->mm.inactive_list,
+                                mm_list) {
+#if 0
+               if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
+                   i915_gem_object_unbind(obj) == 0 &&
+                   i915_gem_object_put_pages(obj) == 0) {
+                       count += obj->base.size >> PAGE_SHIFT;
+                       if (count >= target)
+                               return count;
+               }
+#endif
+       }
+
+       return count;
+}
+
+static long
+i915_gem_purge(struct drm_i915_private *dev_priv, long target)
+{
+       return __i915_gem_shrink(dev_priv, target, true);
+}
+
 static void
 i915_gem_retire_work_handler(struct work_struct *work)
 {
@@ -1634,7 +1904,7 @@ i915_gem_retire_work_handler(struct work_struct *work)
        dev = dev_priv->dev;
 
        /* Come back later if the device is busy... */
-       if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
+       if (lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_NOWAIT)) {
                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
                                   round_jiffies_up_relative(hz));
                return;
@@ -1659,7 +1929,7 @@ i915_gem_retire_work_handler(struct work_struct *work)
        if (idle)
                intel_mark_idle(dev);
 
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 }
 /**
  * Ensures that an object will eventually get non-busy by flushing any required
@@ -1707,10 +1977,12 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
 int
 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
+       drm_i915_private_t *dev_priv = dev->dev_private;
        struct drm_i915_gem_wait *args = data;
        struct drm_i915_gem_object *obj;
        struct intel_ring_buffer *ring = NULL;
        struct timespec timeout_stack, *timeout = NULL;
+       unsigned reset_counter;
        u32 seqno = 0;
        int ret = 0;
 
@@ -1725,7 +1997,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
        obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
        if (&obj->base == NULL) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                return -ENOENT;
        }
 
@@ -1751,9 +2023,10 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
        }
 
        drm_gem_object_unreference(&obj->base);
-       DRM_UNLOCK(dev);
+       reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
+       mutex_unlock(&dev->struct_mutex);
 
-       ret = __wait_seqno(ring, seqno, true, timeout);
+       ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
        if (timeout) {
                WARN_ON(!timespec_valid(timeout));
                args->timeout_ns = timespec_to_ns(timeout);
@@ -1762,7 +2035,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
 out:
        drm_gem_object_unreference(&obj->base);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -1817,15 +2090,15 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
 {
        u32 old_write_domain, old_read_domains;
 
-       /* Act a barrier for all accesses through the GTT */
-       cpu_mfence();
-
        /* Force a pagefault for domain tracking on next user access */
        i915_gem_release_mmap(obj);
 
        if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
                return;
 
+       /* Wait for any direct GTT access to complete */
+       cpu_mfence();
+
        old_read_domains = obj->base.read_domains;
        old_write_domain = obj->base.write_domain;
 
@@ -1841,7 +2114,7 @@ int
 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 {
        drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
-       int ret = 0;
+       int ret;
 
        if (obj->gtt_space == NULL)
                return 0;
@@ -1927,52 +2200,38 @@ int i915_gpu_idle(struct drm_device *dev)
        return 0;
 }
 
-static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
-                                       struct drm_i915_gem_object *obj)
-{
-       drm_i915_private_t *dev_priv = dev->dev_private;
-       uint64_t val;
-
-       if (obj) {
-               u32 size = obj->gtt_space->size;
-
-               val = (uint64_t)((obj->gtt_offset + size - 4096) &
-                                0xfffff000) << 32;
-               val |= obj->gtt_offset & 0xfffff000;
-               val |= (uint64_t)((obj->stride / 128) - 1) <<
-                       SANDYBRIDGE_FENCE_PITCH_SHIFT;
-
-               if (obj->tiling_mode == I915_TILING_Y)
-                       val |= 1 << I965_FENCE_TILING_Y_SHIFT;
-               val |= I965_FENCE_REG_VALID;
-       } else
-               val = 0;
-
-       I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
-       POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
-}
-
 static void i965_write_fence_reg(struct drm_device *dev, int reg,
                                 struct drm_i915_gem_object *obj)
 {
        drm_i915_private_t *dev_priv = dev->dev_private;
+       int fence_reg;
+       int fence_pitch_shift;
        uint64_t val;
 
+       if (INTEL_INFO(dev)->gen >= 6) {
+               fence_reg = FENCE_REG_SANDYBRIDGE_0;
+               fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
+       } else {
+               fence_reg = FENCE_REG_965_0;
+               fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
+       }
+
        if (obj) {
                u32 size = obj->gtt_space->size;
 
                val = (uint64_t)((obj->gtt_offset + size - 4096) &
                                 0xfffff000) << 32;
                val |= obj->gtt_offset & 0xfffff000;
-               val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
+               val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
                if (obj->tiling_mode == I915_TILING_Y)
                        val |= 1 << I965_FENCE_TILING_Y_SHIFT;
                val |= I965_FENCE_REG_VALID;
        } else
                val = 0;
 
-       I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
-       POSTING_READ(FENCE_REG_965_0 + reg * 8);
+       fence_reg += reg * 8;
+       I915_WRITE64(fence_reg, val);
+       POSTING_READ(fence_reg);
 }
 
 static void i915_write_fence_reg(struct drm_device *dev, int reg,
@@ -2051,18 +2310,37 @@ static void i830_write_fence_reg(struct drm_device *dev, int reg,
        POSTING_READ(FENCE_REG_830_0 + reg * 4);
 }
 
+inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
+{
+       return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
+}
+
 static void i915_gem_write_fence(struct drm_device *dev, int reg,
                                 struct drm_i915_gem_object *obj)
 {
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       /* Ensure that all CPU reads are completed before installing a fence
+        * and all writes before removing the fence.
+        */
+       if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
+               cpu_mfence();
+
        switch (INTEL_INFO(dev)->gen) {
        case 7:
-       case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
+       case 6:
        case 5:
        case 4: i965_write_fence_reg(dev, reg, obj); break;
        case 3: i915_write_fence_reg(dev, reg, obj); break;
        case 2: i830_write_fence_reg(dev, reg, obj); break;
-       default: break;
+       default: BUG();
        }
+
+       /* And similarly be paranoid that no direct access to this region
+        * is reordered to before the fence is installed.
+        */
+       if (i915_gem_object_needs_mb(obj))
+               cpu_mfence();
 }
 
 static inline int fence_number(struct drm_i915_private *dev_priv,
@@ -2105,7 +2383,7 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
 }
 
 static int
-i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
+i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
 {
        if (obj->last_fenced_seqno) {
                int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
@@ -2115,12 +2393,6 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
                obj->last_fenced_seqno = 0;
        }
 
-       /* Ensure that all CPU reads are completed before installing a fence
-        * and all writes before removing the fence.
-        */
-       if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
-               cpu_mfence();
-
        obj->fenced_gpu_access = false;
        return 0;
 }
@@ -2131,7 +2403,7 @@ i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
        int ret;
 
-       ret = i915_gem_object_flush_fence(obj);
+       ret = i915_gem_object_wait_fence(obj);
        if (ret)
                return ret;
 
@@ -2205,7 +2477,7 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
         * will need to serialise the write to the associated fence register?
         */
        if (obj->fence_dirty) {
-               ret = i915_gem_object_flush_fence(obj);
+               ret = i915_gem_object_wait_fence(obj);
                if (ret)
                        return ret;
        }
@@ -2226,7 +2498,7 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
                if (reg->obj) {
                        struct drm_i915_gem_object *old = reg->obj;
 
-                       ret = i915_gem_object_flush_fence(old);
+                       ret = i915_gem_object_wait_fence(old);
                        if (ret)
                                return ret;
 
@@ -2249,7 +2521,7 @@ static bool i915_gem_valid_gtt_space(struct drm_device *dev,
 
        /* On non-LLC machines we have to be careful when putting differing
         * types of snoopable memory together to avoid the prefetcher
-        * crossing memory domains and dieing.
+        * crossing memory domains and dying.
         */
        if (HAS_LLC(dev))
                return true;
@@ -2327,21 +2599,21 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
        bool mappable, fenceable;
        int ret;
 
-       if (obj->madv != I915_MADV_WILLNEED) {
-               DRM_ERROR("Attempting to bind a purgeable object\n");
-               return -EINVAL;
-       }
+       fence_size = i915_gem_get_gtt_size(dev,
+                                          obj->base.size,
+                                          obj->tiling_mode);
+       fence_alignment = i915_gem_get_gtt_alignment(dev,
+                                                    obj->base.size,
+                                                    obj->tiling_mode, true);
+       unfenced_alignment =
+               i915_gem_get_gtt_alignment(dev,
+                                                   obj->base.size,
+                                                   obj->tiling_mode, false);
 
-       fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
-           obj->tiling_mode);
-       fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
-           obj->tiling_mode);
-       unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
-           obj->base.size, obj->tiling_mode);
        if (alignment == 0)
                alignment = map_and_fenceable ? fence_alignment :
-                   unfenced_alignment;
-       if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
+                                               unfenced_alignment;
+       if (map_and_fenceable && alignment & (fence_alignment - 1)) {
                DRM_ERROR("Invalid object alignment requested %u\n", alignment);
                return -EINVAL;
        }
@@ -2351,10 +2623,9 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
        /* If the object is bigger than the entire aperture, reject it early
         * before evicting everything in a vain attempt to find space.
         */
-       if (obj->base.size > (map_and_fenceable ?
-           dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
-               DRM_ERROR(
-"Attempting to bind an object larger than the aperture\n");
+       if (obj->base.size >
+           (map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) {
+               DRM_ERROR("Attempting to bind an object larger than the aperture\n");
                return -E2BIG;
        }
 
@@ -2363,19 +2634,18 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                free_space =
                        drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
                                                          size, alignment, obj->cache_level,
-                                                         0, dev_priv->mm.gtt_mappable_end,
+                                                         0, dev_priv->gtt.mappable_end,
                                                          false);
        else
                free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
                                                      size, alignment, obj->cache_level,
                                                      false);
-
        if (free_space != NULL) {
                if (map_and_fenceable)
                        obj->gtt_space =
                                drm_mm_get_block_range_generic(free_space,
                                                               size, alignment, obj->cache_level,
-                                                              0, dev_priv->mm.gtt_mappable_end,
+                                                              0, dev_priv->gtt.mappable_end,
                                                               false);
                else
                        obj->gtt_space =
@@ -2425,7 +2695,8 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                (obj->gtt_space->start & (fence_alignment - 1)) == 0;
 
        mappable =
-               obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
+               obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end;
+
        obj->map_and_fenceable = mappable && fenceable;
 
        i915_gem_verify_gtt(dev);
@@ -2443,6 +2714,13 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj)
        if (obj->pages == NULL)
                return;
 
+       /*
+        * Stolen memory is always coherent with the GPU as it is explicitly
+        * marked as wc by the system, or the system is cache-coherent.
+        */
+       if (obj->stolen)
+               return;
+
        /* If the GPU is snooping the contents of the CPU cache,
         * we do not need to manually clear the CPU cache lines.  However,
         * the caches are only snooped when the render cache is
@@ -2521,6 +2799,13 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 
        i915_gem_object_flush_cpu_write_domain(obj);
 
+       /* Serialise direct access to this object with the barriers for
+        * coherent writes from the GPU, by effectively invalidating the
+        * GTT domain upon first access.
+        */
+       if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
+               cpu_mfence();
+
        old_write_domain = obj->base.write_domain;
        old_read_domains = obj->base.read_domains;
 
@@ -2635,7 +2920,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -2672,7 +2957,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -2812,11 +3097,17 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
        unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
        struct drm_i915_gem_request *request;
        struct intel_ring_buffer *ring = NULL;
+       unsigned reset_counter;
        u32 seqno = 0;
        int ret;
 
-       if (atomic_read(&dev_priv->mm.wedged))
-               return -EIO;
+       ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
+       if (ret)
+               return ret;
+
+       ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
+       if (ret)
+               return ret;
 
        spin_lock(&file_priv->mm.lock);
        list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
@@ -2826,13 +3117,13 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
                ring = request->ring;
                seqno = request->seqno;
        }
+       reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
        spin_unlock(&file_priv->mm.lock);
 
        if (seqno == 0)
                return 0;
 
-       ret = __wait_seqno(ring, seqno, true, NULL);
-
+       ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
        if (ret == 0)
                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 
@@ -2946,7 +3237,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 out:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -2983,7 +3274,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
 out:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -3019,7 +3310,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -3073,7 +3364,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 out:
        drm_gem_object_unreference(&obj->base);
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        return ret;
 }
 
@@ -3198,16 +3489,16 @@ i915_gem_idle(struct drm_device *dev)
        drm_i915_private_t *dev_priv = dev->dev_private;
        int ret;
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
 
        if (dev_priv->mm.suspended) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                return 0;
        }
 
        ret = i915_gpu_idle(dev);
        if (ret) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                return ret;
        }
        i915_gem_retire_requests(dev);
@@ -3223,12 +3514,12 @@ i915_gem_idle(struct drm_device *dev)
         * And not confound mm.suspended!
         */
        dev_priv->mm.suspended = 1;
-       del_timer_sync(&dev_priv->hangcheck_timer);
+       del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
 
        i915_kernel_lost_context(dev);
        i915_gem_cleanup_ringbuffer(dev);
 
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        /* Cancel the retire work handler, which should be idle now. */
        cancel_delayed_work_sync(&dev_priv->mm.retire_work);
@@ -3285,8 +3576,10 @@ void i915_gem_init_swizzling(struct drm_device *dev)
        I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
        if (IS_GEN6(dev))
                I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
-       else
+       else if (IS_GEN7(dev))
                I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
+       else
+               BUG();
 }
 
 static bool
@@ -3308,19 +3601,11 @@ intel_enable_blt(struct drm_device *dev)
        return true;
 }
 
-int
-i915_gem_init_hw(struct drm_device *dev)
+static int i915_gem_init_rings(struct drm_device *dev)
 {
-       drm_i915_private_t *dev_priv = dev->dev_private;
+       struct drm_i915_private *dev_priv = dev->dev_private;
        int ret;
 
-       if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
-               I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
-
-       i915_gem_l3_remap(dev);
-
-       i915_gem_init_swizzling(dev);
-
        ret = intel_init_render_ring_buffer(dev);
        if (ret)
                return ret;
@@ -3337,76 +3622,63 @@ i915_gem_init_hw(struct drm_device *dev)
                        goto cleanup_bsd_ring;
        }
 
-       dev_priv->next_seqno = 1;
-
-       /*
-        * XXX: There was some w/a described somewhere suggesting loading
-        * contexts before PPGTT.
-        */
-       i915_gem_context_init(dev);
-       i915_gem_init_ppgtt(dev);
+       ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
+       if (ret)
+               goto cleanup_blt_ring;
 
        return 0;
 
+cleanup_blt_ring:
+       intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
 cleanup_bsd_ring:
        intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
 cleanup_render_ring:
        intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
+
        return ret;
 }
 
-static bool
-intel_enable_ppgtt(struct drm_device *dev)
+int
+i915_gem_init_hw(struct drm_device *dev)
 {
-       if (i915_enable_ppgtt >= 0)
-               return i915_enable_ppgtt;
+       drm_i915_private_t *dev_priv = dev->dev_private;
+       int ret;
 
-       /* Disable ppgtt on SNB if VT-d is on. */
-       if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled)
-               return false;
+#if 0
+       if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
+               return -EIO;
+#endif
 
-       return true;
+       if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
+               I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
+
+       i915_gem_l3_remap(dev);
+
+       i915_gem_init_swizzling(dev);
+
+       ret = i915_gem_init_rings(dev);
+       if (ret)
+               return ret;
+
+       /*
+        * XXX: There was some w/a described somewhere suggesting loading
+        * contexts before PPGTT.
+        */
+       i915_gem_context_init(dev);
+       i915_gem_init_ppgtt(dev);
+
+       return 0;
 }
 
 int i915_gem_init(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
-       unsigned long gtt_size, mappable_size;
        int ret;
 
-       gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
-       mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
-
-       DRM_LOCK(dev);
-       if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
-               /* PPGTT pdes are stolen from global gtt ptes, so shrink the
-                * aperture accordingly when using aliasing ppgtt. */
-               gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
-
-               i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
-
-               ret = i915_gem_init_aliasing_ppgtt(dev);
-               if (ret) {
-                       DRM_UNLOCK(dev);
-                       return ret;
-               }
-       } else {
-               /* Let GEM Manage all of the aperture.
-                *
-                * However, leave one page at the end still bound to the scratch
-                * page.  There are a number of places where the hardware
-                * apparently prefetches past the end of the object, and we've
-                * seen multiple hangs with the GPU head pointer stuck in a
-                * batchbuffer bound at the last page of the aperture.  One page
-                * should be enough to keep any prefetching inside of the
-                * aperture.
-                */
-               i915_gem_init_global_gtt(dev, 0, mappable_size,
-                                        gtt_size);
-       }
-
+       mutex_lock(&dev->struct_mutex);
+       i915_gem_init_global_gtt(dev);
        ret = i915_gem_init_hw(dev);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        if (ret) {
                i915_gem_cleanup_aliasing_ppgtt(dev);
                return ret;
@@ -3439,22 +3711,22 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
        if (drm_core_check_feature(dev, DRIVER_MODESET))
                return 0;
 
-       if (atomic_read(&dev_priv->mm.wedged)) {
+       if (i915_reset_in_progress(&dev_priv->gpu_error)) {
                DRM_ERROR("Reenabling wedged hardware, good luck\n");
-               atomic_set(&dev_priv->mm.wedged, 0);
+               atomic_set(&dev_priv->gpu_error.reset_counter, 0);
        }
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        dev_priv->mm.suspended = 0;
 
        ret = i915_gem_init_hw(dev);
        if (ret != 0) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                return ret;
        }
 
        KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        ret = drm_irq_install(dev);
        if (ret)
@@ -3463,10 +3735,10 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
        return 0;
 
 cleanup_ringbuffer:
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        i915_gem_cleanup_ringbuffer(dev);
        dev_priv->mm.suspended = 1;
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        return ret;
 }
@@ -3519,7 +3791,7 @@ i915_gem_load(struct drm_device *dev)
                INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
        INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
                          i915_gem_retire_work_handler);
-       init_completion(&dev_priv->error_completion);
+       init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
 
        /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
        if (IS_GEN3(dev)) {
@@ -3551,7 +3823,7 @@ i915_gem_load(struct drm_device *dev)
        dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
        register_shrinker(&dev_priv->mm.inactive_shrinker);
 #else
-       dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
+       dev_priv->mm.inactive_shrinker = EVENTHANDLER_REGISTER(vm_lowmem,
            i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
 #endif
 }
@@ -3748,9 +4020,9 @@ i915_gem_phys_pwrite(struct drm_device *dev,
                 * of the obj, so we can safely drop the lock and continue
                 * to access vaddr.
                 */
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                unwritten = copy_from_user(vaddr, user_data, args->size);
-               DRM_LOCK(dev);
+               mutex_lock(&dev->struct_mutex);
                if (unwritten)
                        return -EFAULT;
        }
@@ -3844,7 +4116,7 @@ unlocked_vmobj:
                        goto out;
                }
        } else
-               DRM_LOCK(dev);
+               mutex_lock(&dev->struct_mutex);
 
        /*
         * Since the object lock was dropped, other thread might have
@@ -3855,7 +4127,7 @@ unlocked_vmobj:
        m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
        if (m != NULL) {
                if ((m->flags & PG_BUSY) != 0) {
-                       DRM_UNLOCK(dev);
+                       mutex_unlock(&dev->struct_mutex);
 #if 0 /* XXX */
                        vm_page_sleep(m, "915pee");
 #endif
@@ -3865,6 +4137,12 @@ unlocked_vmobj:
        } else
                VM_OBJECT_UNLOCK(vm_obj);
 
+       /* Access to snoopable pages through the GTT is incoherent. */
+       if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
+               ret = -EINVAL;
+               goto unlock;
+       }
+
        /* Now bind it into the GTT if needed */
        if (!obj->map_and_fenceable) {
                ret = i915_gem_object_unbind(obj);
@@ -3913,7 +4191,7 @@ unlocked_vmobj:
        KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
 
        if ((m->flags & PG_BUSY) != 0) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
 #if 0 /* XXX */
                vm_page_sleep(m, "915pbs");
 #endif
@@ -3925,7 +4203,7 @@ have_page:
        *mres = m;
        vm_page_busy_try(m, false);
 
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        if (oldm != NULL) {
                vm_page_free(oldm);
        }
@@ -3933,7 +4211,7 @@ have_page:
        return (VM_PAGER_OK);
 
 unlock:
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 out:
        KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
        if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
@@ -3953,11 +4231,11 @@ i915_gem_pager_dtor(void *handle)
        obj = handle;
        dev = obj->dev;
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        drm_gem_free_mmap_offset(obj);
        i915_gem_release_mmap(to_intel_bo(obj));
        drm_gem_object_unreference(obj);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 }
 
 struct cdev_pager_ops i915_gem_pager_ops = {
@@ -4012,7 +4290,7 @@ i915_gem_lowmem(void *arg)
        dev = arg;
        dev_priv = dev->dev_private;
 
-       if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
+       if (lockmgr(&dev->struct_mutex, LK_EXCLUSIVE|LK_NOWAIT))
                return;
 
 rescan:
@@ -4049,14 +4327,5 @@ rescan:
                if (i915_gpu_idle(dev) == 0)
                        goto rescan;
        }
-       DRM_UNLOCK(dev);
-}
-
-void
-i915_gem_unload(struct drm_device *dev)
-{
-       struct drm_i915_private *dev_priv;
-
-       dev_priv = dev->dev_private;
-       EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
+       mutex_unlock(&dev->struct_mutex);
 }
index d127441..237af04 100644 (file)
@@ -127,13 +127,8 @@ static int get_context_size(struct drm_device *dev)
 
 static void do_destroy(struct i915_hw_context *ctx)
 {
-       struct drm_device *dev = ctx->obj->base.dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
-
        if (ctx->file_priv)
                idr_remove(&ctx->file_priv->context_idr, ctx->id);
-       else
-               BUG_ON(ctx != dev_priv->ring[RCS].default_context);
 
        drm_gem_object_unreference(&ctx->obj->base);
        kfree(ctx, M_DRM);
@@ -250,7 +245,6 @@ err_destroy:
 void i915_gem_context_init(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
-       uint32_t ctx_size;
 
        if (!HAS_HW_CONTEXTS(dev)) {
                dev_priv->hw_contexts_disabled = true;
@@ -262,11 +256,9 @@ void i915_gem_context_init(struct drm_device *dev)
            dev_priv->ring[RCS].default_context)
                return;
 
-       ctx_size = get_context_size(dev);
-       dev_priv->hw_context_size = get_context_size(dev);
-       dev_priv->hw_context_size = round_up(dev_priv->hw_context_size, 4096);
+       dev_priv->hw_context_size = round_up(get_context_size(dev), 4096);
 
-       if (ctx_size <= 0 || ctx_size > (1<<20)) {
+       if (dev_priv->hw_context_size > (1<<20)) {
                dev_priv->hw_contexts_disabled = true;
                return;
        }
@@ -311,10 +303,10 @@ void i915_gem_context_close(struct drm_device *dev, struct drm_file *file)
 {
        struct drm_i915_file_private *file_priv = file->driver_priv;
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL);
        idr_destroy(&file_priv->context_idr);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 }
 
 static struct i915_hw_context *
@@ -501,7 +493,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
                return ret;
 
        ctx = create_hw_context(dev, file_priv);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);
 
@@ -528,13 +520,13 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
 
        ctx = i915_gem_context_get(file_priv, args->ctx_id);
        if (!ctx) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                return -ENOENT;
        }
 
        do_destroy(ctx);
 
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        DRM_DEBUG_DRIVER("HW context %d destroyed\n", args->ctx_id);
        return 0;
index 0ba51e6..c86d5d9 100644 (file)
@@ -27,8 +27,9 @@
  */
 
 #include <drm/drmP.h>
-#include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include <drm/i915_drm.h>
+#include "i915_trace.h"
 
 static bool
 mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
@@ -50,6 +51,8 @@ i915_gem_evict_something(struct drm_device *dev, int min_size,
        struct drm_i915_gem_object *obj;
        int ret = 0;
 
+       trace_i915_gem_evict(dev, min_size, alignment, mappable);
+
        /*
         * The goal is to evict objects and amalgamate space in LRU order.
         * The oldest idle objects reside on the inactive list, which is in
@@ -77,7 +80,7 @@ i915_gem_evict_something(struct drm_device *dev, int min_size,
        if (mappable)
                drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space,
                                            min_size, alignment, cache_level,
-                                           0, dev_priv->mm.gtt_mappable_end);
+                                           0, dev_priv->gtt.mappable_end);
        else
                drm_mm_init_scan(&dev_priv->mm.gtt_space,
                                 min_size, alignment, cache_level);
@@ -160,6 +163,8 @@ i915_gem_evict_everything(struct drm_device *dev)
        if (lists_empty)
                return -ENOSPC;
 
+       trace_i915_gem_evict_everything(dev);
+
        /* The gpu_idle will flush everything in the write domain to the
         * active list. Then we must move everything off the active list
         * with retire requests.
index d52208a..357e7ed 100644 (file)
 
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
-#include <linux/highmem.h>
 #include "i915_drv.h"
+#include "i915_trace.h"
 #include "intel_drv.h"
+#include <linux/highmem.h>
 
 struct eb_objects {
+       struct list_head objects;
        int and;
-       struct hlist_head buckets[0];
+       union {
+               struct drm_i915_gem_object *lut[0];
+               struct hlist_head buckets[0];
+       };
 };
 
 static struct eb_objects *
-eb_create(int size)
+eb_create(struct drm_i915_gem_execbuffer2 *args)
 {
-       struct eb_objects *eb;
-       int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
-       while (count > size)
-               count >>= 1;
-       eb = kmalloc(count*sizeof(struct hlist_head) +
-                    sizeof(struct eb_objects),
-                    M_DRM, M_WAITOK | M_ZERO);
-       if (eb == NULL)
-               return eb;
-
-       eb->and = count - 1;
+       struct eb_objects *eb = NULL;
+
+       if (args->flags & I915_EXEC_HANDLE_LUT) {
+               int size = args->buffer_count;
+               size *= sizeof(struct drm_i915_gem_object *);
+               size += sizeof(struct eb_objects);
+               eb = kmalloc(size, M_DRM, M_WAITOK);
+       }
+
+       if (eb == NULL) {
+               int size = args->buffer_count;
+               int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
+               BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
+               while (count > 2*size)
+                       count >>= 1;
+               eb = kmalloc(count*sizeof(struct hlist_head) +
+                            sizeof(struct eb_objects),
+                            M_DRM, M_WAITOK | M_ZERO);
+               if (eb == NULL)
+                       return eb;
+
+               eb->and = count - 1;
+       } else
+               eb->and = -args->buffer_count;
+
+       INIT_LIST_HEAD(&eb->objects);
        return eb;
 }
 
 static void
 eb_reset(struct eb_objects *eb)
 {
-       memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
+       if (eb->and >= 0)
+               memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
 }
 
-static void
-eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
+static int
+eb_lookup_objects(struct eb_objects *eb,
+                 struct drm_i915_gem_exec_object2 *exec,
+                 const struct drm_i915_gem_execbuffer2 *args,
+                 struct drm_file *file)
 {
-       hlist_add_head(&obj->exec_node,
-                      &eb->buckets[obj->exec_handle & eb->and]);
+       int i;
+
+       lockmgr(&file->table_lock, LK_EXCLUSIVE);
+       for (i = 0; i < args->buffer_count; i++) {
+               struct drm_i915_gem_object *obj;
+
+               obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
+               if (obj == NULL) {
+                       lockmgr(&file->table_lock, LK_RELEASE);
+                       DRM_DEBUG("Invalid object handle %d at index %d\n",
+                                  exec[i].handle, i);
+                       return -ENOENT;
+               }
+
+               if (!list_empty(&obj->exec_list)) {
+                       lockmgr(&file->table_lock, LK_RELEASE);
+                       DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
+                                  obj, exec[i].handle, i);
+                       return -EINVAL;
+               }
+
+               drm_gem_object_reference(&obj->base);
+               list_add_tail(&obj->exec_list, &eb->objects);
+
+               obj->exec_entry = &exec[i];
+               if (eb->and < 0) {
+                       eb->lut[i] = obj;
+               } else {
+                       uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
+                       obj->exec_handle = handle;
+                       hlist_add_head(&obj->exec_node,
+                                      &eb->buckets[handle & eb->and]);
+               }
+       }
+       lockmgr(&file->table_lock, LK_RELEASE);
+
+       return 0;
 }
 
 static struct drm_i915_gem_object *
 eb_get_object(struct eb_objects *eb, unsigned long handle)
 {
-       struct hlist_head *head;
-       struct hlist_node *node;
-       struct drm_i915_gem_object *obj;
+       if (eb->and < 0) {
+               if (handle >= -eb->and)
+                       return NULL;
+               return eb->lut[handle];
+       } else {
+               struct hlist_head *head;
+               struct hlist_node *node;
 
-       head = &eb->buckets[handle & eb->and];
-       hlist_for_each(node, head) {
-               obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
-               if (obj->exec_handle == handle)
-                       return obj;
-       }
+               head = &eb->buckets[handle & eb->and];
+               hlist_for_each(node, head) {
+                       struct drm_i915_gem_object *obj;
 
-       return NULL;
+                       obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
+                       if (obj->exec_handle == handle)
+                               return obj;
+               }
+               return NULL;
+       }
 }
 
 static void
 eb_destroy(struct eb_objects *eb)
 {
+       while (!list_empty(&eb->objects)) {
+               struct drm_i915_gem_object *obj;
+
+               obj = list_first_entry(&eb->objects,
+                                      struct drm_i915_gem_object,
+                                      exec_list);
+               list_del_init(&obj->exec_list);
+               drm_gem_object_unreference(&obj->base);
+       }
        drm_free(eb, M_DRM);
 }
 
@@ -148,17 +222,6 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                          reloc->write_domain);
                return ret;
        }
-       if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
-                    reloc->write_domain != target_obj->pending_write_domain)) {
-               DRM_DEBUG("Write domain conflict: "
-                         "obj %p target %d offset %d "
-                         "new %08x old %08x\n",
-                         obj, reloc->target_handle,
-                         (int) reloc->offset,
-                         reloc->write_domain,
-                         target_obj->pending_write_domain);
-               return ret;
-       }
 
        target_obj->pending_read_domains |= reloc->read_domains;
        target_obj->pending_write_domain |= reloc->write_domain;
@@ -199,7 +262,8 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                if (ret)
                        return ret;
 
-               vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
+               vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+                                                            reloc->offset >> PAGE_SHIFT));
                *(uint32_t *)(vaddr + page_offset) = reloc->delta;
                kunmap_atomic(vaddr);
        } else {
@@ -220,7 +284,6 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                    ~PAGE_MASK), PAGE_SIZE, PAT_WRITE_COMBINING);
                reloc_entry = (uint32_t *)(reloc_page + (reloc->offset &
                    PAGE_MASK));
-
                iowrite32(reloc->delta, reloc_entry);
                pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE);
        }
@@ -296,8 +359,7 @@ i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
 
 static int
 i915_gem_execbuffer_relocate(struct drm_device *dev,
-                            struct eb_objects *eb,
-                            struct list_head *objects)
+                            struct eb_objects *eb)
 {
        struct drm_i915_gem_object *obj;
        int ret = 0;
@@ -312,7 +374,7 @@ i915_gem_execbuffer_relocate(struct drm_device *dev,
 #if 0
        pagefault_disable();
 #endif
-       list_for_each_entry(obj, objects, exec_list) {
+       list_for_each_entry(obj, &eb->objects, exec_list) {
                ret = i915_gem_execbuffer_relocate_object(obj, eb);
                if (ret)
                        break;
@@ -336,7 +398,8 @@ need_reloc_mappable(struct drm_i915_gem_object *obj)
 
 static int
 i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
-                                  struct intel_ring_buffer *ring)
+                                  struct intel_ring_buffer *ring,
+                                  bool *need_reloc)
 {
        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
        struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
@@ -377,7 +440,20 @@ i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
                obj->has_aliasing_ppgtt_mapping = 1;
        }
 
-       entry->offset = obj->gtt_offset;
+       if (entry->offset != obj->gtt_offset) {
+               entry->offset = obj->gtt_offset;
+               *need_reloc = true;
+       }
+
+       if (entry->flags & EXEC_OBJECT_WRITE) {
+               obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
+               obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
+       }
+
+       if (entry->flags & EXEC_OBJECT_NEEDS_GTT &&
+           !obj->has_global_gtt_mapping)
+               i915_gem_gtt_bind_object(obj, obj->cache_level);
+
        return 0;
 }
 
@@ -403,7 +479,8 @@ i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
 static int
 i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
                            struct drm_file *file,
-                           struct list_head *objects)
+                           struct list_head *objects,
+                           bool *need_relocs)
 {
        struct drm_i915_gem_object *obj;
        struct list_head ordered_objects;
@@ -431,7 +508,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
                else
                        list_move_tail(&obj->exec_list, &ordered_objects);
 
-               obj->base.pending_read_domains = 0;
+               obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
                obj->base.pending_write_domain = 0;
                obj->pending_fenced_gpu_access = false;
        }
@@ -471,7 +548,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
                            (need_mappable && !obj->map_and_fenceable))
                                ret = i915_gem_object_unbind(obj);
                        else
-                               ret = i915_gem_execbuffer_reserve_object(obj, ring);
+                               ret = i915_gem_execbuffer_reserve_object(obj, ring, need_relocs);
                        if (ret)
                                goto err;
                }
@@ -481,7 +558,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
                        if (obj->gtt_space)
                                continue;
 
-                       ret = i915_gem_execbuffer_reserve_object(obj, ring);
+                       ret = i915_gem_execbuffer_reserve_object(obj, ring, need_relocs);
                        if (ret)
                                goto err;
                }
@@ -501,28 +578,29 @@ err:              /* Decrement pin count for bound objects */
 
 static int
 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
+                                 struct drm_i915_gem_execbuffer2 *args,
                                  struct drm_file *file,
                                  struct intel_ring_buffer *ring,
-                                 struct list_head *objects,
                                  struct eb_objects *eb,
-                                 struct drm_i915_gem_exec_object2 *exec,
-                                 int count)
+                                 struct drm_i915_gem_exec_object2 *exec)
 {
        struct drm_i915_gem_relocation_entry *reloc;
        struct drm_i915_gem_object *obj;
+       bool need_relocs;
        int *reloc_offset;
        int i, total, ret;
+       int count = args->buffer_count;
 
        /* We may process another execbuffer during the unlock... */
-       while (!list_empty(objects)) {
-               obj = list_first_entry(objects,
+       while (!list_empty(&eb->objects)) {
+               obj = list_first_entry(&eb->objects,
                                       struct drm_i915_gem_object,
                                       exec_list);
                list_del_init(&obj->exec_list);
                drm_gem_object_unreference(&obj->base);
        }
 
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        total = 0;
        for (i = 0; i < count; i++)
@@ -533,7 +611,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
        if (reloc == NULL || reloc_offset == NULL) {
                drm_free_large(reloc);
                drm_free_large(reloc_offset);
-               DRM_LOCK(dev);
+               mutex_lock(&dev->struct_mutex);
                return -ENOMEM;
        }
 
@@ -548,7 +626,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
                if (copy_from_user(reloc+total, user_relocs,
                                   exec[i].relocation_count * sizeof(*reloc))) {
                        ret = -EFAULT;
-                       DRM_LOCK(dev);
+                       mutex_lock(&dev->struct_mutex);
                        goto err;
                }
 
@@ -566,7 +644,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
                                         &invalid_offset,
                                         sizeof(invalid_offset))) {
                                ret = -EFAULT;
-                               DRM_LOCK(dev);
+                               mutex_lock(&dev->struct_mutex);
                                goto err;
                        }
                }
@@ -577,33 +655,22 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 
        ret = i915_mutex_lock_interruptible(dev);
        if (ret) {
-               DRM_LOCK(dev);
+               mutex_lock(&dev->struct_mutex);
                goto err;
        }
 
        /* reacquire the objects */
        eb_reset(eb);
-       for (i = 0; i < count; i++) {
-               obj = to_intel_bo(drm_gem_object_lookup(dev, file,
-                                                       exec[i].handle));
-               if (&obj->base == NULL) {
-                       DRM_DEBUG("Invalid object handle %d at index %d\n",
-                                  exec[i].handle, i);
-                       ret = -ENOENT;
-                       goto err;
-               }
-
-               list_add_tail(&obj->exec_list, objects);
-               obj->exec_handle = exec[i].handle;
-               obj->exec_entry = &exec[i];
-               eb_add_object(eb, obj);
-       }
+       ret = eb_lookup_objects(eb, exec, args, file);
+       if (ret)
+               goto err;
 
-       ret = i915_gem_execbuffer_reserve(ring, file, objects);
+       need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
+       ret = i915_gem_execbuffer_reserve(ring, file, &eb->objects, &need_relocs);
        if (ret)
                goto err;
 
-       list_for_each_entry(obj, objects, exec_list) {
+       list_for_each_entry(obj, &eb->objects, exec_list) {
                int offset = obj->exec_entry - exec;
                ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
                                                               reloc + reloc_offset[offset]);
@@ -623,45 +690,12 @@ err:
        return ret;
 }
 
-static int
-i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
-{
-       u32 plane, flip_mask;
-       int ret;
-
-       /* Check for any pending flips. As we only maintain a flip queue depth
-        * of 1, we can simply insert a WAIT for the next display flip prior
-        * to executing the batch and avoid stalling the CPU.
-        */
-
-       for (plane = 0; flips >> plane; plane++) {
-               if (((flips >> plane) & 1) == 0)
-                       continue;
-
-               if (plane)
-                       flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
-               else
-                       flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-
-               ret = intel_ring_begin(ring, 2);
-               if (ret)
-                       return ret;
-
-               intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
-               intel_ring_emit(ring, MI_NOOP);
-               intel_ring_advance(ring);
-       }
-
-       return 0;
-}
-
 static int
 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
                                struct list_head *objects)
 {
        struct drm_i915_gem_object *obj;
        uint32_t flush_domains = 0;
-       uint32_t flips = 0;
        int ret;
 
        list_for_each_entry(obj, objects, exec_list) {
@@ -672,18 +706,9 @@ i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
                if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
                        i915_gem_clflush_object(obj);
 
-               if (obj->base.pending_write_domain)
-                       flips |= atomic_read(&obj->pending_flip);
-
                flush_domains |= obj->base.write_domain;
        }
 
-       if (flips) {
-               ret = i915_gem_execbuffer_wait_for_flips(ring, flips);
-               if (ret)
-                       return ret;
-       }
-
        if (flush_domains & I915_GEM_DOMAIN_CPU)
                i915_gem_chipset_flush(ring->dev);
 
@@ -699,6 +724,9 @@ i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 static bool
 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 {
+       if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
+               return false;
+
        return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
 }
 
@@ -716,6 +744,9 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
 #endif
                int length; /* limited by fault_in_pages_readable() */
 
+               if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS)
+                       return -EINVAL;
+
                /* First check for malicious input causing overflow in
                 * the worst case where we need to allocate the entire
                 * relocation tree as a single array.
@@ -727,9 +758,6 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
                length = exec[i].relocation_count *
                        sizeof(struct drm_i915_gem_relocation_entry);
 #if 0
-               if (!access_ok(VERIFY_READ, ptr, length))
-                       return -EFAULT;
-
                /* we may also need to update the presumed offsets */
                if (!access_ok(VERIFY_WRITE, ptr, length))
                        return -EFAULT;
@@ -749,8 +777,11 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
        struct drm_i915_gem_object *obj;
 
        list_for_each_entry(obj, objects, exec_list) {
-               obj->base.read_domains = obj->base.pending_read_domains;
+
                obj->base.write_domain = obj->base.pending_write_domain;
+               if (obj->base.write_domain == 0)
+                       obj->base.pending_read_domains |= obj->base.read_domains;
+               obj->base.read_domains = obj->base.pending_read_domains;
                obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
 
                i915_gem_object_move_to_active(obj, ring);
@@ -760,6 +791,8 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
                        if (obj->pin_count) /* check for potential scanout */
                                intel_mark_fb_busy(obj);
                }
+
+               trace_i915_gem_object_change_domain(obj, old_read, old_write);
        }
 }
 
@@ -807,21 +840,18 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                       struct drm_i915_gem_exec_object2 *exec)
 {
        drm_i915_private_t *dev_priv = dev->dev_private;
-       struct list_head objects;
        struct eb_objects *eb;
        struct drm_i915_gem_object *batch_obj;
        struct drm_clip_rect *cliprects = NULL;
        struct intel_ring_buffer *ring;
        u32 ctx_id = i915_execbuffer2_get_context_id(*args);
        u32 exec_start, exec_len;
-       u32 mask;
-       u32 flags;
+       u32 mask, flags;
        int ret, mode, i;
+       bool need_relocs;
 
-       if (!i915_gem_check_execbuffer(args)) {
-               DRM_DEBUG("execbuf with invalid offset/length\n");
+       if (!i915_gem_check_execbuffer(args))
                return -EINVAL;
-       }
 
        ret = validate_exec_list(exec, args->buffer_count);
        if (ret)
@@ -912,6 +942,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                                  args->num_cliprects);
                        return -EINVAL;
                }
+
                cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
                                    M_DRM, M_WAITOK | M_ZERO);
                if (cliprects == NULL) {
@@ -933,64 +964,41 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                goto pre_mutex_err;
 
        if (dev_priv->mm.suspended) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                ret = -EBUSY;
                goto pre_mutex_err;
        }
 
-       eb = eb_create(args->buffer_count);
+       eb = eb_create(args);
        if (eb == NULL) {
-               DRM_UNLOCK(dev);
+               mutex_unlock(&dev->struct_mutex);
                ret = -ENOMEM;
                goto pre_mutex_err;
        }
 
        /* Look up object handles */
-       INIT_LIST_HEAD(&objects);
-       for (i = 0; i < args->buffer_count; i++) {
-               struct drm_i915_gem_object *obj;
-
-               obj = to_intel_bo(drm_gem_object_lookup(dev, file,
-                                                       exec[i].handle));
-               if (&obj->base == NULL) {
-                       DRM_DEBUG("Invalid object handle %d at index %d\n",
-                                  exec[i].handle, i);
-                       /* prevent error path from reading uninitialized data */
-                       ret = -ENOENT;
-                       goto err;
-               }
-
-               if (!list_empty(&obj->exec_list)) {
-                       DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
-                                  obj, exec[i].handle, i);
-                       ret = -EINVAL;
-                       goto err;
-               }
-
-               list_add_tail(&obj->exec_list, &objects);
-               obj->exec_handle = exec[i].handle;
-               obj->exec_entry = &exec[i];
-               eb_add_object(eb, obj);
-       }
+       ret = eb_lookup_objects(eb, exec, args, file);
+       if (ret)
+               goto err;
 
        /* take note of the batch buffer before we might reorder the lists */
-       batch_obj = list_entry(objects.prev,
+       batch_obj = list_entry(eb->objects.prev,
                               struct drm_i915_gem_object,
                               exec_list);
 
        /* Move the objects en-masse into the GTT, evicting if necessary. */
-       ret = i915_gem_execbuffer_reserve(ring, file, &objects);
+       need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
+       ret = i915_gem_execbuffer_reserve(ring, file, &eb->objects, &need_relocs);
        if (ret)
                goto err;
 
        /* The objects are in their final locations, apply the relocations. */
-       ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
+       if (need_relocs)
+               ret = i915_gem_execbuffer_relocate(dev, eb);
        if (ret) {
                if (ret == -EFAULT) {
-                       ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
-                                                               &objects, eb,
-                                                               exec,
-                                                               args->buffer_count);
+                       ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
+                                                               eb, exec);
                        DRM_LOCK_ASSERT(dev);
                }
                if (ret)
@@ -1012,7 +1020,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
        if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
                i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
 
-       ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
+       ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->objects);
        if (ret)
                goto err;
 
@@ -1064,22 +1072,15 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                        goto err;
        }
 
-       i915_gem_execbuffer_move_to_active(&objects, ring);
+       trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
+
+       i915_gem_execbuffer_move_to_active(&eb->objects, ring);
        i915_gem_execbuffer_retire_commands(dev, file, ring);
 
 err:
        eb_destroy(eb);
-       while (!list_empty(&objects)) {
-               struct drm_i915_gem_object *obj;
-
-               obj = list_first_entry(&objects,
-                                      struct drm_i915_gem_object,
-                                      exec_list);
-               list_del_init(&obj->exec_list);
-               drm_gem_object_unreference(&obj->base);
-       }
 
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
 pre_mutex_err:
        drm_free(cliprects, M_DRM);
@@ -1115,7 +1116,6 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
                drm_free_large(exec2_list);
                return -ENOMEM;
        }
-
        ret = copy_from_user(exec_list,
                             (void __user *)(uintptr_t)args->buffers_ptr,
                             sizeof(*exec_list) * args->buffer_count);
index 2b447c1..21105f9 100644 (file)
@@ -45,9 +45,9 @@ typedef uint32_t gtt_pte_t;
 #define GEN6_PTE_CACHE_LLC_MLC         (3 << 1)
 #define GEN6_PTE_ADDR_ENCODE(addr)     GEN6_GTT_ADDR_ENCODE(addr)
 
-static inline gtt_pte_t pte_encode(struct drm_device *dev,
-                                  dma_addr_t addr,
-                                  enum i915_cache_level level)
+static inline gtt_pte_t gen6_pte_encode(struct drm_device *dev,
+                                       dma_addr_t addr,
+                                       enum i915_cache_level level)
 {
        gtt_pte_t pte = GEN6_PTE_VALID;
        pte |= GEN6_PTE_ADDR_ENCODE(addr);
@@ -78,7 +78,7 @@ static inline gtt_pte_t pte_encode(struct drm_device *dev,
 }
 
 /* PPGTT support for Sandybdrige/Gen6 and later */
-static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
+static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
                                   unsigned first_entry,
                                   unsigned num_entries)
 {
@@ -88,8 +88,9 @@ static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
        unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
        unsigned last_pte, i;
 
-       scratch_pte = pte_encode(ppgtt->dev, ppgtt->scratch_page_dma_addr,
-                                I915_CACHE_LLC);
+       scratch_pte = gen6_pte_encode(ppgtt->dev,
+                                     ppgtt->scratch_page_dma_addr,
+                                     I915_CACHE_LLC);
 
        while (num_entries) {
                last_pte = first_pte + num_entries;
@@ -109,10 +110,74 @@ static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
        }
 }
 
-int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
+static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
+                                     struct sg_table *pages,
+                                     unsigned first_entry,
+                                     enum i915_cache_level cache_level)
 {
+       gtt_pte_t *pt_vaddr;
+       unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
+       unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
+       unsigned i, j, m, segment_len;
+       dma_addr_t page_addr;
+       struct scatterlist *sg;
+
+       /* init sg walking */
+       sg = pages->sgl;
+       i = 0;
+       segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
+       m = 0;
+
+       while (i < pages->nents) {
+               pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
+
+               for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
+                       page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
+                       pt_vaddr[j] = gen6_pte_encode(ppgtt->dev, page_addr,
+                                                     cache_level);
+
+                       /* grab the next page */
+                       if (++m == segment_len) {
+                               if (++i == pages->nents)
+                                       break;
+
+                               sg = sg_next(sg);
+                               segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
+                               m = 0;
+                       }
+               }
+
+               kunmap_atomic(pt_vaddr);
+
+               first_pte = 0;
+               act_pd++;
+       }
+}
+
+static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
+{
+#if 0
+       int i;
+
+       if (ppgtt->pt_dma_addr) {
+               for (i = 0; i < ppgtt->num_pd_entries; i++)
+                       pci_unmap_page(ppgtt->dev->pdev,
+                                      ppgtt->pt_dma_addr[i],
+                                      4096, PCI_DMA_BIDIRECTIONAL);
+       }
+
+       kfree(ppgtt->pt_dma_addr);
+       for (i = 0; i < ppgtt->num_pd_entries; i++)
+               __free_page(ppgtt->pt_pages[i]);
+       kfree(ppgtt->pt_pages);
+       kfree(ppgtt);
+#endif
+}
+
+static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
+{
+       struct drm_device *dev = ppgtt->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
-       struct i915_hw_ppgtt *ppgtt;
        unsigned first_pd_entry_in_global_pt;
        int i;
        int ret = -ENOMEM;
@@ -121,18 +186,16 @@ int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
         * entries. For aliasing ppgtt support we just steal them at the end for
         * now.
         */
-       first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES;
+       first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
 
-       ppgtt = kmalloc(sizeof(*ppgtt), M_DRM, M_WAITOK | M_ZERO);
-       if (!ppgtt)
-               return ret;
-
-       ppgtt->dev = dev;
        ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
+       ppgtt->clear_range = gen6_ppgtt_clear_range;
+       ppgtt->insert_entries = gen6_ppgtt_insert_entries;
+       ppgtt->cleanup = gen6_ppgtt_cleanup;
        ppgtt->pt_pages = kmalloc(sizeof(vm_page_t) * ppgtt->num_pd_entries,
            M_DRM, M_WAITOK | M_ZERO);
        if (!ppgtt->pt_pages)
-               goto err_ppgtt;
+               return -ENOMEM;
 
        for (i = 0; i < ppgtt->num_pd_entries; i++) {
                ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0,
@@ -141,28 +204,55 @@ int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
                        goto err_pt_alloc;
        }
 
-       ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
+       ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma;
 
-       i915_ppgtt_clear_range(ppgtt, 0,
-                              ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
+       ppgtt->clear_range(ppgtt, 0,
+                          ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 
        ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t);
 
-       dev_priv->mm.aliasing_ppgtt = ppgtt;
-
        return 0;
 
-
 err_pt_alloc:
        dev_priv->mm.aliasing_ppgtt = ppgtt;
        i915_gem_cleanup_aliasing_ppgtt(dev);
-       return (-ENOMEM);
-err_ppgtt:
-       kfree(ppgtt, M_DRM);
+
+       return ret;
+}
+
+static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct i915_hw_ppgtt *ppgtt;
+       int ret;
+
+       ppgtt = kmalloc(sizeof(*ppgtt), M_DRM, M_WAITOK | M_ZERO);
+       if (!ppgtt)
+               return -ENOMEM;
+
+       ppgtt->dev = dev;
+
+       ret = gen6_ppgtt_init(ppgtt);
+       if (ret)
+               kfree(ppgtt, M_DRM);
+       else
+               dev_priv->mm.aliasing_ppgtt = ppgtt;
 
        return ret;
 }
 
+void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+
+       if (!ppgtt)
+               return;
+
+       ppgtt->cleanup(ppgtt);
+}
+
+#if 0
 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
@@ -185,6 +275,7 @@ void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
        drm_free(ppgtt->pt_pages, M_DRM);
        drm_free(ppgtt, M_DRM);
 }
+#endif
 
 static void
 i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, unsigned first_entry,
@@ -205,7 +296,7 @@ i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, unsigned first_entry,
 
                for (i = first_pte; i < last_pte; i++) {
                        page_addr = VM_PAGE_TO_PHYS(*pages);
-                       pt_vaddr[i] = pte_encode(ppgtt->dev, page_addr,
+                       pt_vaddr[i] = gen6_pte_encode(ppgtt->dev, page_addr,
                                                 cache_level);
 
                        pages++;
@@ -230,9 +321,9 @@ void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
                              struct drm_i915_gem_object *obj)
 {
-       i915_ppgtt_clear_range(ppgtt,
-                              obj->gtt_space->start >> PAGE_SHIFT,
-                              obj->base.size >> PAGE_SHIFT);
+       ppgtt->clear_range(ppgtt,
+                          obj->gtt_space->start >> PAGE_SHIFT,
+                          obj->base.size >> PAGE_SHIFT);
 }
 
 void i915_gem_init_ppgtt(struct drm_device *dev)
@@ -291,11 +382,27 @@ void i915_gem_init_ppgtt(struct drm_device *dev)
        }
 }
 
+extern int intel_iommu_gfx_mapped;
+/* Certain Gen5 chipsets require require idling the GPU before
+ * unmapping anything from the GTT when VT-d is enabled.
+ */
+static inline bool needs_idle_maps(struct drm_device *dev)
+{
+#ifdef CONFIG_INTEL_IOMMU
+       /* Query intel_iommu to see if we need the workaround. Presumably that
+        * was loaded first.
+        */
+       if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
+               return true;
+#endif
+       return false;
+}
+
 static bool do_idling(struct drm_i915_private *dev_priv)
 {
        bool ret = dev_priv->mm.interruptible;
 
-       if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
+       if (unlikely(dev_priv->gtt.do_idle_maps)) {
                dev_priv->mm.interruptible = false;
                if (i915_gpu_idle(dev_priv->dev)) {
                        DRM_ERROR("Couldn't idle GPU\n");
@@ -309,45 +416,18 @@ static bool do_idling(struct drm_i915_private *dev_priv)
 
 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 {
-       if (unlikely(dev_priv->mm.gtt->do_idle_maps))
+       if (unlikely(dev_priv->gtt.do_idle_maps))
                dev_priv->mm.interruptible = interruptible;
 }
 
-
-static void i915_ggtt_clear_range(struct drm_device *dev,
-                                unsigned first_entry,
-                                unsigned num_entries)
-{
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       gtt_pte_t scratch_pte;
-       gtt_pte_t __iomem *gtt_base = dev_priv->mm.gtt->gtt + first_entry;
-       const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
-       int i;
-
-       if (INTEL_INFO(dev)->gen < 6) {
-               intel_gtt_clear_range(first_entry, num_entries);
-               return;
-       }
-
-       if (WARN(num_entries > max_entries,
-                "First entry = %d; Num entries = %d (max=%d)\n",
-                first_entry, num_entries, max_entries))
-               num_entries = max_entries;
-
-       scratch_pte = pte_encode(dev, dev_priv->mm.gtt->scratch_page_dma, I915_CACHE_LLC);
-       for (i = 0; i < num_entries; i++)
-               iowrite32(scratch_pte, &gtt_base[i]);
-       readl(gtt_base);
-}
-
 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct drm_i915_gem_object *obj;
 
        /* First fill our portion of the GTT with scratch pages */
-       i915_ggtt_clear_range(dev, dev_priv->mm.gtt_start / PAGE_SIZE,
-                             (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
+       dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
+                                     dev_priv->gtt.total / PAGE_SIZE);
 
        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
                i915_gem_clflush_object(obj);
@@ -370,6 +450,7 @@ int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 
        return 0;
 }
+#endif
 
 /*
  * Binds an object into the global gtt with the specified cache level. The object
@@ -377,16 +458,16 @@ int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  * within the global GTT as well as accessible by the GPU through the GMADR
  * mapped BAR (dev_priv->mm.gtt->gtt).
  */
-static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj,
-                                 enum i915_cache_level level)
+static void gen6_ggtt_insert_entries(struct drm_device *dev,
+                                    struct sg_table *st,
+                                    unsigned int first_entry,
+                                    enum i915_cache_level level)
 {
-       struct drm_device *dev = obj->base.dev;
+#if 0
        struct drm_i915_private *dev_priv = dev->dev_private;
-       struct sg_table *st = obj->pages;
        struct scatterlist *sg = st->sgl;
-       const int first_entry = obj->gtt_space->start >> PAGE_SHIFT;
-       const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
-       gtt_pte_t __iomem *gtt_entries = dev_priv->mm.gtt->gtt + first_entry;
+       gtt_pte_t __iomem *gtt_entries =
+               (gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
        int unused, i = 0;
        unsigned int len, m = 0;
        dma_addr_t addr;
@@ -395,14 +476,12 @@ static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj,
                len = sg_dma_len(sg) >> PAGE_SHIFT;
                for (m = 0; m < len; m++) {
                        addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
-                       iowrite32(pte_encode(dev, addr, level), &gtt_entries[i]);
+                       iowrite32(gen6_pte_encode(dev, addr, level),
+                                 &gtt_entries[i]);
                        i++;
                }
        }
 
-       BUG_ON(i > max_entries);
-       BUG_ON(i != obj->base.size / PAGE_SIZE);
-
        /* XXX: This serves as a posting read to make sure that the PTE has
         * actually been updated. There is some concern that even though
         * registers and PTEs are within the same BAR that they are potentially
@@ -410,7 +489,8 @@ static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj,
         * hardware should work, we must keep this posting read for paranoia.
         */
        if (i != 0)
-               WARN_ON(readl(&gtt_entries[i-1]) != pte_encode(dev, addr, level));
+               WARN_ON(readl(&gtt_entries[i-1])
+                       != gen6_pte_encode(dev, addr, level));
 
        /* This next bit makes the above posting read even more important. We
         * want to flush the TLBs only after we're certain all the PTE updates
@@ -418,8 +498,50 @@ static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj,
         */
        I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
        POSTING_READ(GFX_FLSH_CNTL_GEN6);
+#endif
 }
+
+static void gen6_ggtt_clear_range(struct drm_device *dev,
+                                 unsigned int first_entry,
+                                 unsigned int num_entries)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       gtt_pte_t scratch_pte;
+       gtt_pte_t __iomem *gtt_base = (gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
+       const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry;
+       int i;
+
+       if (WARN(num_entries > max_entries,
+                "First entry = %d; Num entries = %d (max=%d)\n",
+                first_entry, num_entries, max_entries))
+               num_entries = max_entries;
+
+       scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma,
+                                     I915_CACHE_LLC);
+       for (i = 0; i < num_entries; i++)
+               iowrite32(scratch_pte, &gtt_base[i]);
+       readl(gtt_base);
+}
+
+static void i915_ggtt_insert_entries(struct drm_device *dev,
+                                    struct sg_table *st,
+                                    unsigned int pg_start,
+                                    enum i915_cache_level cache_level)
+{
+#if 0
+       unsigned int flags = (cache_level == I915_CACHE_NONE) ?
+               AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
+
+       intel_gtt_insert_sg_entries(st, pg_start, flags);
 #endif
+}
+
+static void i915_ggtt_clear_range(struct drm_device *dev,
+                                 unsigned int first_entry,
+                                 unsigned int num_entries)
+{
+       intel_gtt_clear_range(first_entry, num_entries);
+}
 
 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
                              enum i915_cache_level cache_level)
@@ -436,14 +558,11 @@ void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 {
        struct drm_device *dev = obj->base.dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
-       bool interruptible;
-
-       interruptible = do_idling(dev_priv);
 
-       intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
-           obj->base.size >> PAGE_SHIFT);
+       dev_priv->gtt.gtt_clear_range(obj->base.dev,
+                                     obj->gtt_space->start >> PAGE_SHIFT,
+                                     obj->base.size >> PAGE_SHIFT);
 
-       undo_idling(dev_priv, interruptible);
        obj->has_global_gtt_mapping = 0;
 }
 
@@ -481,17 +600,26 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node,
                        *end -= 4096;
        }
 }
-
-void i915_gem_init_global_gtt(struct drm_device *dev,
-                             unsigned long start,
-                             unsigned long mappable_end,
-                             unsigned long end)
+void i915_gem_setup_global_gtt(struct drm_device *dev,
+                              unsigned long start,
+                              unsigned long mappable_end,
+                              unsigned long end)
 {
+       /* Let GEM Manage all of the aperture.
+        *
+        * However, leave one page at the end still bound to the scratch page.
+        * There are a number of places where the hardware apparently prefetches
+        * past the end of the object, and we've seen multiple hangs with the
+        * GPU head pointer stuck in a batchbuffer bound at the last page of the
+        * aperture.  One page should be enough to keep any prefetching inside
+        * of the aperture.
+        */
        drm_i915_private_t *dev_priv = dev->dev_private;
-
        unsigned long mappable;
        int error;
 
+       BUG_ON(mappable_end > end);
+
        mappable = min(end, mappable_end) - start;
 
        /* Substract the guard page ... */
@@ -499,11 +627,9 @@ void i915_gem_init_global_gtt(struct drm_device *dev,
        if (!HAS_LLC(dev))
                dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
 
-       dev_priv->mm.gtt_start = start;
-       dev_priv->mm.gtt_mappable_end = mappable_end;
-       dev_priv->mm.gtt_end = end;
-       dev_priv->mm.gtt_total = end - start;
-       dev_priv->mm.mappable_gtt_total = mappable;
+       dev_priv->gtt.start = start;
+       dev_priv->gtt.mappable_end = mappable_end;
+       dev_priv->gtt.total = end - start;
 
        /* ... but ensure that we clear the entire range. */
        intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
@@ -514,6 +640,57 @@ void i915_gem_init_global_gtt(struct drm_device *dev,
            dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
 }
 
+static bool
+intel_enable_ppgtt(struct drm_device *dev)
+{
+       if (i915_enable_ppgtt >= 0)
+               return i915_enable_ppgtt;
+
+#ifdef CONFIG_INTEL_IOMMU
+       /* Disable ppgtt on SNB if VT-d is on. */
+       if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
+               return false;
+#endif
+
+       return true;
+}
+
+void i915_gem_init_global_gtt(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       unsigned long gtt_size, mappable_size;
+       int ret;
+
+       gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
+       mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
+
+       if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
+               /* PPGTT pdes are stolen from global gtt ptes, so shrink the
+                * aperture accordingly when using aliasing ppgtt. */
+               gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
+
+               i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
+
+               ret = i915_gem_init_aliasing_ppgtt(dev);
+               if (ret) {
+                       DRM_UNLOCK(dev);
+                       return;
+               }
+       } else {
+               /* Let GEM Manage all of the aperture.
+                *
+                * However, leave one page at the end still bound to the scratch
+                * page.  There are a number of places where the hardware
+                * apparently prefetches past the end of the object, and we've
+                * seen multiple hangs with the GPU head pointer stuck in a
+                * batchbuffer bound at the last page of the aperture.  One page
+                * should be enough to keep any prefetching inside of the
+                * aperture.
+                */
+               i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
+       }
+}
+
 int i915_gem_gtt_init(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
@@ -527,6 +704,12 @@ int i915_gem_gtt_init(struct drm_device *dev)
                        DRM_ERROR("Failed to initialize GTT\n");
                        return -ENODEV;
                }
+
+               dev_priv->gtt.do_idle_maps = needs_idle_maps(dev);
+
+               dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
+               dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
+
                return 0;
        }
 
@@ -534,14 +717,13 @@ int i915_gem_gtt_init(struct drm_device *dev)
        if (!dev_priv->mm.gtt)
                return -ENOMEM;
 
-#ifdef CONFIG_INTEL_IOMMU
-       dev_priv->mm.gtt->needs_dmar = 1;
-#endif
-
        /* GMADR is the PCI aperture used by SW to access tiled GFX surfaces in a linear fashion. */
        DRM_INFO("Memory usable by graphics device = %dM\n", dev_priv->mm.gtt->gtt_total_entries >> 8);
        DRM_DEBUG_DRIVER("GMADR size = %dM\n", dev_priv->mm.gtt->gtt_mappable_entries >> 8);
        DRM_DEBUG_DRIVER("GTT stolen size = %dM\n", dev_priv->mm.gtt->stolen_size >> 20);
 
+       dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
+       dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
+
        return 0;
 }
diff --git a/sys/dev/drm/i915/i915_gem_stolen.c b/sys/dev/drm/i915/i915_gem_stolen.c
new file mode 100644 (file)
index 0000000..ca378d6
--- /dev/null
@@ -0,0 +1,197 @@
+/*
+ * Copyright © 2008-2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *    Chris Wilson <chris@chris-wilson.co.uk>
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+/*
+ * The BIOS typically reserves some of the system's memory for the exclusive
+ * use of the integrated graphics. This memory is no longer available for
+ * use by the OS and so the user finds that his system has less memory
+ * available than he put in. We refer to this memory as stolen.
+ *
+ * The BIOS will allocate its framebuffer from the stolen memory. Our
+ * goal is try to reuse that object for our own fbcon which must always
+ * be available for panics. Anything else we can reuse the stolen memory
+ * for is a boon.
+ */
+
+static unsigned long i915_stolen_to_physical(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct pci_dev *pdev = dev_priv->bridge_dev;
+       u32 base;
+
+       /* On the machines I have tested the Graphics Base of Stolen Memory
+        * is unreliable, so on those compute the base by subtracting the
+        * stolen memory from the Top of Low Usable DRAM which is where the
+        * BIOS places the graphics stolen memory.
+        *
+        * On gen2, the layout is slightly different with the Graphics Segment
+        * immediately following Top of Memory (or Top of Usable DRAM). Note
+        * it appears that TOUD is only reported by 865g, so we just use the
+        * top of memory as determined by the e820 probe.
+        *
+        * XXX gen2 requires an unavailable symbol and 945gm fails with
+        * its value of TOLUD.
+        */
+       base = 0;
+       if (INTEL_INFO(dev)->gen >= 6) {
+               /* Read Base Data of Stolen Memory Register (BDSM) directly.
+                * Note that there is also a MCHBAR miror at 0x1080c0 or
+                * we could use device 2:0x5c instead.
+               */
+               pci_read_config_dword(pdev, 0xB0, &base);
+               base &= ~4095; /* lower bits used for locking register */
+       } else if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+               /* Read Graphics Base of Stolen Memory directly */
+               pci_read_config_dword(pdev, 0xA4, &base);
+#if 0
+       } else if (IS_GEN3(dev)) {
+               u8 val;
+               /* Stolen is immediately below Top of Low Usable DRAM */
+               pci_read_config_byte(pdev, 0x9c, &val);
+               base = val >> 3 << 27;
+               base -= dev_priv->mm.gtt->stolen_size;
+       } else {
+               /* Stolen is immediately above Top of Memory */
+               base = max_low_pfn_mapped << PAGE_SHIFT;
+#endif
+       }
+
+       return base;
+}
+
+static int i915_setup_compression(struct drm_device *dev, int size)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct drm_mm_node *compressed_fb, *compressed_llb = NULL;
+
+       /* Try to over-allocate to reduce reallocations and fragmentation */
+       compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen,
+                                          size <<= 1, 4096, 0);
+       if (!compressed_fb)
+               compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen,
+                                                  size >>= 1, 4096, 0);
+       if (compressed_fb)
+               compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
+       if (!compressed_fb)
+               goto err;
+
+       if (HAS_PCH_SPLIT(dev))
+               I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
+       else if (IS_GM45(dev)) {
+               I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
+       } else {
+               compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
+                                                   4096, 4096, 0);
+               if (compressed_llb)
+                       compressed_llb = drm_mm_get_block(compressed_llb,
+                                                         4096, 4096);
+               if (!compressed_llb)
+                       goto err_fb;
+
+               dev_priv->compressed_llb = compressed_llb;
+
+               I915_WRITE(FBC_CFB_BASE,
+                          dev_priv->mm.stolen_base + compressed_fb->start);
+               I915_WRITE(FBC_LL_BASE,
+                          dev_priv->mm.stolen_base + compressed_llb->start);
+       }
+
+       dev_priv->compressed_fb = compressed_fb;
+       dev_priv->cfb_size = size;
+
+       DRM_DEBUG_KMS("reserved %d bytes of contiguous stolen space for FBC\n",
+                     size);
+
+       return 0;
+
+err_fb:
+       drm_mm_put_block(compressed_fb);
+err:
+       return -ENOSPC;
+}
+
+int i915_gem_stolen_setup_compression(struct drm_device *dev, int size)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       if (dev_priv->mm.stolen_base == 0)
+               return -ENODEV;
+
+       if (size < dev_priv->cfb_size)
+               return 0;
+
+       /* Release any current block */
+       i915_gem_stolen_cleanup_compression(dev);
+
+       return i915_setup_compression(dev, size);
+}
+
+void i915_gem_stolen_cleanup_compression(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       if (dev_priv->cfb_size == 0)
+               return;
+
+       if (dev_priv->compressed_fb)
+               drm_mm_put_block(dev_priv->compressed_fb);
+
+       if (dev_priv->compressed_llb)
+               drm_mm_put_block(dev_priv->compressed_llb);
+
+       dev_priv->cfb_size = 0;
+}
+
+void i915_gem_cleanup_stolen(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       i915_gem_stolen_cleanup_compression(dev);
+       drm_mm_takedown(&dev_priv->mm.stolen);
+}
+
+int i915_gem_init_stolen(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       dev_priv->mm.stolen_base = i915_stolen_to_physical(dev);
+       if (dev_priv->mm.stolen_base == 0)
+               return 0;
+
+       DRM_DEBUG_KMS("found %zd bytes of stolen memory at %08lx\n",
+                     dev_priv->gtt.stolen_size, dev_priv->mm.stolen_base);
+
+       /* Basic memrange allocator for stolen space */
+       drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->gtt.stolen_size);
+
+       return 0;
+}
index c7a2509..ce918e2 100644 (file)
@@ -272,18 +272,7 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
                        return false;
        }
 
-       /*
-        * Previous chips need to be aligned to the size of the smallest
-        * fence register that can contain the object.
-        */
-       if (INTEL_INFO(obj->base.dev)->gen == 3)
-               size = 1024*1024;
-       else
-               size = 512*1024;
-
-       while (size < obj->base.size)
-               size <<= 1;
-
+       size = i915_gem_get_gtt_size(obj->base.dev, obj->base.size, tiling_mode);
        if (obj->gtt_space->size != size)
                return false;
 
@@ -350,7 +339,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
                }
        }
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
        if (args->tiling_mode != obj->tiling_mode ||
            args->stride != obj->stride) {
                /* We need to rebind the object if its current allocation
@@ -368,15 +357,15 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 
                obj->map_and_fenceable =
                        obj->gtt_space == NULL ||
-                       (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end &&
+                       (obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end &&
                         i915_gem_object_fence_ok(obj, args->tiling_mode));
 
                /* Rebind if we need a change of alignment */
                if (!obj->map_and_fenceable) {
                        u32 unfenced_alignment =
-                               i915_gem_get_unfenced_gtt_alignment(dev,
-                                                                   obj->base.size,
-                                                                   args->tiling_mode);
+                               i915_gem_get_gtt_alignment(dev, obj->base.size,
+                                                           args->tiling_mode,
+                                                           false);
                        if (obj->gtt_offset & (unfenced_alignment - 1))
                                ret = i915_gem_object_unbind(obj);
                }
@@ -396,8 +385,20 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
        /* we have to maintain this existing ABI... */
        args->stride = obj->stride;
        args->tiling_mode = obj->tiling_mode;
+
+       /* Try to preallocate memory required to save swizzling on put-pages */
+       if (i915_gem_object_needs_bit17_swizzle(obj)) {
+               if (obj->bit_17 == NULL) {
+                       obj->bit_17 = kmalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT) *
+                                             sizeof(long), M_DRM, M_WAITOK);
+               }
+       } else {
+               drm_free(obj->bit_17, M_DRM);
+               obj->bit_17 = NULL;
+       }
+
        drm_gem_object_unreference(&obj->base);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        return ret;
 }
@@ -417,7 +418,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
        if (&obj->base == NULL)
                return -ENOENT;
 
-       DRM_LOCK(dev);
+       mutex_lock(&dev->struct_mutex);
 
        args->tiling_mode = obj->tiling_mode;
        switch (obj->tiling_mode) {
@@ -441,7 +442,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
                args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
        drm_gem_object_unreference(&obj->base);
-       DRM_UNLOCK(dev);
+       mutex_unlock(&dev->struct_mutex);
 
        return 0;
 }
index c6f649a..42f7618 100644 (file)
@@ -281,22 +281,23 @@ static void i915_hotplug_work_func(struct work_struct *work)
        struct drm_mode_config *mode_config = &dev->mode_config;
        struct intel_encoder *encoder;
 
-       lockmgr(&mode_config->mutex, LK_EXCLUSIVE);
+       /* HPD irq before everything is fully set up. */
+       if (!dev_priv->enable_hotplug_processing)
+               return;
+
+       mutex_lock(&mode_config->mutex);
        DRM_DEBUG_KMS("running encoder hotplug functions\n");
 
        list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
                if (encoder->hot_plug)
                        encoder->hot_plug(encoder);
 
-       lockmgr(&mode_config->mutex, LK_RELEASE);
+       mutex_unlock(&mode_config->mutex);
 
        /* Just fire off a uevent and let userspace tell us what to do */
        drm_helper_hpd_irq_event(dev);
 }
 
-/* defined intel_pm.c */
-extern struct lock mchdev_lock;
-
 static void ironlake_handle_rps_change(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = dev->dev_private;
@@ -346,8 +347,8 @@ static void notify_ring(struct drm_device *dev,
 
        wake_up_all(&ring->irq_queue);
        if (i915_enable_hangcheck) {
-               dev_priv->hangcheck_count = 0;
-               mod_timer(&dev_priv->hangcheck_timer,
+               dev_priv->gpu_error.hangcheck_count = 0;
+               mod_timer(&dev_priv->gpu_error.hangcheck_timer,
                          round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
        }
 }
@@ -369,7 +370,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
        if ((pm_iir & GEN6_PM_DEFERRED_EVENTS) == 0)
                return;
 
-       lockmgr(&dev_priv->rps.hw_lock, LK_EXCLUSIVE);
+       mutex_lock(&dev_priv->rps.hw_lock);
 
        if (pm_iir & GEN6_PM_RP_UP_THRESHOLD)
                new_delay = dev_priv->rps.cur_delay + 1;
@@ -384,7 +385,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
                gen6_set_rps(dev_priv->dev, new_delay);
        }
 
-       lockmgr(&dev_priv->rps.hw_lock, LK_RELEASE);
+       mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
 
@@ -402,13 +403,14 @@ static void ivybridge_parity_work(struct work_struct *work)
        drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
                                                    l3_parity.error_work);
        u32 error_status, row, bank, subbank;
+       char *parity_event[5];
        uint32_t misccpctl;
 
        /* We must turn off DOP level clock gating to access the L3 registers.
         * In order to prevent a get/put style interface, acquire struct mutex
         * any time we access those registers.
         */
-       DRM_LOCK(dev_priv->dev);
+       mutex_lock(&dev_priv->dev->struct_mutex);
 
        misccpctl = I915_READ(GEN7_MISCCPCTL);
        I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
@@ -430,7 +432,10 @@ static void ivybridge_parity_work(struct work_struct *work)
        I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
        lockmgr(&dev_priv->irq_lock, LK_RELEASE);
 
-       DRM_UNLOCK(dev_priv->dev);
+       mutex_unlock(&dev_priv->dev->struct_mutex);
+
+       parity_event[0] = "L3_PARITY_ERROR=1";
+       parity_event[4] = NULL;
 
        DRM_DEBUG("Parity error: Row = %d, Bank = %d, Sub bank = %d.\n",
                  row, bank, subbank);
@@ -498,6 +503,20 @@ static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
        queue_work(dev_priv->wq, &dev_priv->rps.work);
 }
 
+static void gmbus_irq_handler(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
+
+       wake_up_all(&dev_priv->gmbus_wait_queue);
+}
+
+static void dp_aux_irq_handler(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
+
+       wake_up_all(&dev_priv->gmbus_wait_queue);
+}
+
 static irqreturn_t valleyview_irq_handler(void *arg)
 {
        struct drm_device *dev = (struct drm_device *) arg;
@@ -505,7 +524,6 @@ static irqreturn_t valleyview_irq_handler(void *arg)
        u32 iir, gt_iir, pm_iir;
        int pipe;
        u32 pipe_stats[I915_MAX_PIPES];
-       bool blc_event;
 
        atomic_inc(&dev_priv->irq_received);
 
@@ -560,8 +578,8 @@ static irqreturn_t valleyview_irq_handler(void *arg)
                        I915_READ(PORT_HOTPLUG_STAT);
                }
 
-               if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
-                       blc_event = true;
+               if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
+                       gmbus_irq_handler(dev);
 
                if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
                        gen6_queue_rps_work(dev_priv, pm_iir);
@@ -588,8 +606,11 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
                                 (pch_iir & SDE_AUDIO_POWER_MASK) >>
                                 SDE_AUDIO_POWER_SHIFT);
 
+       if (pch_iir & SDE_AUX_MASK)
+               dp_aux_irq_handler(dev);
+
        if (pch_iir & SDE_GMBUS)
-               DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
+               gmbus_irq_handler(dev);
 
        if (pch_iir & SDE_AUDIO_HDCP_MASK)
                DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
@@ -632,10 +653,10 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
                                 SDE_AUDIO_POWER_SHIFT_CPT);
 
        if (pch_iir & SDE_AUX_MASK_CPT)
-               DRM_DEBUG_DRIVER("AUX channel interrupt\n");
+               dp_aux_irq_handler(dev);
 
        if (pch_iir & SDE_GMBUS_CPT)
-               DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");
+               gmbus_irq_handler(dev);
 
        if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
                DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
@@ -654,7 +675,7 @@ static irqreturn_t ivybridge_irq_handler(void *arg)
 {
        struct drm_device *dev = (struct drm_device *) arg;
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-       u32 de_iir, gt_iir, de_ier, pm_iir;
+       u32 de_iir, gt_iir, de_ier, pm_iir, sde_ier;
        int i;
 
        atomic_inc(&dev_priv->irq_received);
@@ -663,6 +684,15 @@ static irqreturn_t ivybridge_irq_handler(void *arg)
        de_ier = I915_READ(DEIER);
        I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
 
+       /* Disable south interrupts. We'll only write to SDEIIR once, so further
+        * interrupts will will be stored on its back queue, and then we'll be
+        * able to process them after we restore SDEIER (as soon as we restore
+        * it, we'll get an interrupt if SDEIIR still has something to process
+        * due to its back queue). */
+       sde_ier = I915_READ(SDEIER);
+       I915_WRITE(SDEIER, 0);
+       POSTING_READ(SDEIER);
+
        gt_iir = I915_READ(GTIIR);
        if (gt_iir) {
                snb_gt_irq_handler(dev, dev_priv, gt_iir);
@@ -671,6 +701,9 @@ static irqreturn_t ivybridge_irq_handler(void *arg)
 
        de_iir = I915_READ(DEIIR);
        if (de_iir) {
+               if (de_iir & DE_AUX_CHANNEL_A_IVB)
+                       dp_aux_irq_handler(dev);
+
                if (de_iir & DE_GSE_IVB)
                        intel_opregion_gse_intr(dev);
 
@@ -705,6 +738,8 @@ static irqreturn_t ivybridge_irq_handler(void *arg)
 
        I915_WRITE(DEIER, de_ier);
        POSTING_READ(DEIER);
+       I915_WRITE(SDEIER, sde_ier);
+       POSTING_READ(SDEIER);
 }
 
 static void ilk_gt_irq_handler(struct drm_device *dev,
@@ -721,7 +756,7 @@ static irqreturn_t ironlake_irq_handler(void *arg)
 {
        struct drm_device *dev = (struct drm_device *) arg;
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-       u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
+       u32 de_iir, gt_iir, de_ier, pm_iir, sde_ier;
 
        atomic_inc(&dev_priv->irq_received);
 
@@ -730,13 +765,20 @@ static irqreturn_t ironlake_irq_handler(void *arg)
        I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
        POSTING_READ(DEIER);
 
+       /* Disable south interrupts. We'll only write to SDEIIR once, so further
+        * interrupts will will be stored on its back queue, and then we'll be
+        * able to process them after we restore SDEIER (as soon as we restore
+        * it, we'll get an interrupt if SDEIIR still has something to process
+        * due to its back queue). */
+       sde_ier = I915_READ(SDEIER);
+       I915_WRITE(SDEIER, 0);
+       POSTING_READ(SDEIER);
+
        de_iir = I915_READ(DEIIR);
        gt_iir = I915_READ(GTIIR);
-       pch_iir = I915_READ(SDEIIR);
        pm_iir = I915_READ(GEN6_PMIIR);
 
-       if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
-           (!IS_GEN6(dev) || pm_iir == 0))
+       if (de_iir == 0 && gt_iir == 0 && (!IS_GEN6(dev) || pm_iir == 0))
                goto done;
 
        if (IS_GEN5(dev))
@@ -744,6 +786,9 @@ static irqreturn_t ironlake_irq_handler(void *arg)
        else
                snb_gt_irq_handler(dev, dev_priv, gt_iir);
 
+       if (de_iir & DE_AUX_CHANNEL_A)
+               dp_aux_irq_handler(dev);
+
        if (de_iir & DE_GSE)
                intel_opregion_gse_intr(dev);
 
@@ -765,10 +810,15 @@ static irqreturn_t ironlake_irq_handler(void *arg)
 
        /* check event from PCH */
        if (de_iir & DE_PCH_EVENT) {
+               u32 pch_iir = I915_READ(SDEIIR);
+
                if (HAS_PCH_CPT(dev))
                        cpt_irq_handler(dev, pch_iir);
                else
                        ibx_irq_handler(dev, pch_iir);
+
+               /* should clear PCH hotplug event before clear CPU irq */
+               I915_WRITE(SDEIIR, pch_iir);
        }
 
        if (IS_GEN5(dev) &&  de_iir & DE_PCU_EVENT)
@@ -777,8 +827,6 @@ static irqreturn_t ironlake_irq_handler(void *arg)
        if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS)
                gen6_queue_rps_work(dev_priv, pm_iir);
 
-       /* should clear PCH hotplug event before clear CPU irq */
-       I915_WRITE(SDEIIR, pch_iir);
        I915_WRITE(GTIIR, gt_iir);
        I915_WRITE(DEIIR, de_iir);
        I915_WRITE(GEN6_PMIIR, pm_iir);
@@ -786,6 +834,8 @@ static irqreturn_t ironlake_irq_handler(void *arg)
 done:
        I915_WRITE(DEIER, de_ier);
        POSTING_READ(DEIER);
+       I915_WRITE(SDEIER, sde_ier);
+       POSTING_READ(SDEIER);
 }
 
 /**
@@ -797,20 +847,66 @@ done:
  */
 static void i915_error_work_func(struct work_struct *work)
 {
-       drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
-                                                   error_work);
+       struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
+                                                   work);
+       drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
+                                                   gpu_error);
        struct drm_device *dev = dev_priv->dev;
+       struct intel_ring_buffer *ring;
+#if 0
+       char *error_event[] = { "ERROR=1", NULL };
+       char *reset_event[] = { "RESET=1", NULL };
+       char *reset_done_event[] = { "ERROR=0", NULL };
+#endif
+       int i, ret;
 
        /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event); */
 
-       if (atomic_read(&dev_priv->mm.wedged)) {
+       /*
+        * Note that there's only one work item which does gpu resets, so we
+        * need not worry about concurrent gpu resets potentially incrementing
+        * error->reset_counter twice. We only need to take care of another
+        * racing irq/hangcheck declaring the gpu dead for a second time. A
+        * quick check for that is good enough: schedule_work ensures the
+        * correct ordering between hang detection and this work item, and since
+        * the reset in-progress bit is only ever set by code outside of this
+        * work we don't need to worry about any other races.
+        */
+       if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
                DRM_DEBUG_DRIVER("resetting chip\n");
-               /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event); */
-               if (!i915_reset(dev)) {
-                       atomic_set(&dev_priv->mm.wedged, 0);
-                       /* kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event); */
+#if 0
+               kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
+                                  reset_event);
+#endif
+
+               ret = i915_reset(dev);
+
+               if (ret == 0) {
+                       /*
+                        * After all the gem state is reset, increment the reset
+                        * counter and wake up everyone waiting for the reset to
+                        * complete.
+                        *
+                        * Since unlock operations are a one-sided barrier only,
+                        * we need to insert a barrier here to order any seqno
+                        * updates before
+                        * the counter increment.
+                        */
+                       cpu_sfence();
+                       atomic_inc(&dev_priv->gpu_error.reset_counter);
+
+#if 0
+                       kobject_uevent_env(&dev->primary->kdev.kobj,
+                                          KOBJ_CHANGE, reset_done_event);
+#endif
+               } else {
+                       atomic_set(&error->reset_counter, I915_WEDGED);
                }
-               complete_all(&dev_priv->error_completion);
+
+               for_each_ring(ring, dev_priv, i)
+                       wake_up_all(&ring->irq_queue);
+
+               wake_up_all(&dev_priv->gpu_error.reset_queue);
        }
 }
 
@@ -873,7 +969,7 @@ i915_error_object_create(struct drm_i915_private *dev_priv,
                        goto unwind;
 
                local_irq_save(flags);
-               if (reloc_offset < dev_priv->mm.gtt_mappable_end &&
+               if (reloc_offset < dev_priv->gtt.mappable_end &&
                    src->has_global_gtt_mapping) {
                        void __iomem *s;
 
@@ -882,10 +978,18 @@ i915_error_object_create(struct drm_i915_private *dev_priv,
                         * captures what the GPU read.
                         */
 
-                       s = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
+                       s = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
                                                     reloc_offset);
                        memcpy_fromio(d, s, PAGE_SIZE);
                        io_mapping_unmap_atomic(s);
+               } else if (src->stolen) {
+                       unsigned long offset;
+
+                       offset = dev_priv->mm.stolen_base;
+                       offset += src->stolen->start;
+                       offset += i << PAGE_SHIFT;
+
+                       memcpy_fromio(d, (void *)offset, PAGE_SIZE);
                } else {
                        struct page *page;
                        void *s;
@@ -1033,6 +1137,8 @@ static void i915_gem_record_fences(struct drm_device *dev,
                        error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
                break;
 
+       default:
+               BUG();
        }
 }
 
@@ -1181,9 +1287,9 @@ static void i915_capture_error_state(struct drm_device *dev)
        unsigned long flags;
        int i, pipe;
 
-       spin_lock_irqsave(&dev_priv->error_lock, flags);
-       error = dev_priv->first_error;
-       spin_unlock_irqrestore(&dev_priv->error_lock, flags);
+       spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
+       error = dev_priv->gpu_error.first_error;
+       spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
        if (error)
                return;
 
@@ -1194,7 +1300,8 @@ static void i915_capture_error_state(struct drm_device *dev)
                return;
        }
 
-       DRM_INFO("capturing error event; look for more information in /debug/dri/%d/i915_error_state\n",
+       DRM_INFO("capturing error event; look for more information in"
+                "/sys/kernel/debug/dri/%d/i915_error_state\n",
                 dev->primary->index);
 
        kref_init(&error->ref);
@@ -1277,12 +1384,12 @@ static void i915_capture_error_state(struct drm_device *dev)
        error->overlay = intel_overlay_capture_error_state(dev);
        error->display = intel_display_capture_error_state(dev);
 
-       spin_lock_irqsave(&dev_priv->error_lock, flags);
-       if (dev_priv->first_error == NULL) {
-               dev_priv->first_error = error;
+       spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
+       if (dev_priv->gpu_error.first_error == NULL) {
+               dev_priv->gpu_error.first_error = error;
                error = NULL;
        }
-       spin_unlock_irqrestore(&dev_priv->error_lock, flags);
+       spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
 
        if (error)
                i915_error_state_free(&error->ref);
@@ -1293,10 +1400,10 @@ void i915_destroy_error_state(struct drm_device *dev)
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct drm_i915_error_state *error;
 
-       lockmgr(&dev_priv->error_lock, LK_EXCLUSIVE);
-       error = dev_priv->first_error;
-       dev_priv->first_error = NULL;
-       lockmgr(&dev_priv->error_lock, LK_RELEASE);
+       lockmgr(&dev_priv->gpu_error.lock, LK_EXCLUSIVE);
+       error = dev_priv->gpu_error.first_error;
+       dev_priv->gpu_error.first_error = NULL;
+       lockmgr(&dev_priv->gpu_error.lock, LK_RELEASE);
 
        if (error)
                i915_error_state_free(dev, error);
@@ -1417,17 +1524,18 @@ void i915_handle_error(struct drm_device *dev, bool wedged)
        i915_report_and_clear_eir(dev);
 
        if (wedged) {
-               INIT_COMPLETION(dev_priv->error_completion);
-               atomic_set(&dev_priv->mm.wedged, 1);
+               atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
+                               &dev_priv->gpu_error.reset_counter);
 
                /*
-                * Wakeup waiting processes so they don't hang
+                * Wakeup waiting processes so that the reset work item
+                * doesn't deadlock trying to grab various locks.
                 */
                for_each_ring(ring, dev_priv, i)
                        wake_up_all(&ring->irq_queue);
        }
 
-       queue_work(dev_priv->wq, &dev_priv->error_work);
+       queue_work(dev_priv->wq, &dev_priv->gpu_error.work);
 }
 
 static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
@@ -1653,7 +1761,7 @@ static bool i915_hangcheck_hung(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = dev->dev_private;
 
-       if (dev_priv->hangcheck_count++ > 1) {
+       if (dev_priv->gpu_error.hangcheck_count++ > 1) {
                bool hung = true;
 
                DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
@@ -1712,25 +1820,29 @@ void i915_hangcheck_elapsed(unsigned long data)
                        goto repeat;
                }
 
-               dev_priv->hangcheck_count = 0;
+               dev_priv->gpu_error.hangcheck_count = 0;
                return;
        }
 
        i915_get_extra_instdone(dev, instdone);
-       if (memcmp(dev_priv->last_acthd, acthd, sizeof(acthd)) == 0 &&
-           memcmp(dev_priv->prev_instdone, instdone, sizeof(instdone)) == 0) {
+       if (memcmp(dev_priv->gpu_error.last_acthd, acthd,
+                  sizeof(acthd)) == 0 &&
+           memcmp(dev_priv->gpu_error.prev_instdone, instdone,
+                  sizeof(instdone)) == 0) {
                if (i915_hangcheck_hung(dev))
                        return;
        } else {
-               dev_priv->hangcheck_count = 0;
+               dev_priv->gpu_error.hangcheck_count = 0;
 
-               memcpy(dev_priv->last_acthd, acthd, sizeof(acthd));
-               memcpy(dev_priv->prev_instdone, instdone, sizeof(instdone));
+               memcpy(dev_priv->gpu_error.last_acthd, acthd,
+                      sizeof(acthd));
+               memcpy(dev_priv->gpu_error.prev_instdone, instdone,
+                      sizeof(instdone));
        }
 
 repeat:
        /* Reset timer case chip hangs without another request being added */
-       mod_timer(&dev_priv->hangcheck_timer,
+       mod_timer(&dev_priv->gpu_error.hangcheck_timer,
                  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
 }
 
@@ -1800,7 +1912,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev)
  * This register is the same on all known PCH chips.
  */
 
-static void ironlake_enable_pch_hotplug(struct drm_device *dev)
+static void ibx_enable_hotplug(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
        u32     hotplug;
@@ -1813,14 +1925,36 @@ static void ironlake_enable_pch_hotplug(struct drm_device *dev)
        I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
 }
 
+static void ibx_irq_postinstall(struct drm_device *dev)
+{
+       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+       u32 mask;
+
+       if (HAS_PCH_IBX(dev))
+               mask = SDE_HOTPLUG_MASK |
+                      SDE_GMBUS |
+                      SDE_AUX_MASK;
+       else
+               mask = SDE_HOTPLUG_MASK_CPT |
+                      SDE_GMBUS_CPT |
+                      SDE_AUX_MASK_CPT;
+
+       I915_WRITE(SDEIIR, I915_READ(SDEIIR));
+       I915_WRITE(SDEIMR, ~mask);
+       I915_WRITE(SDEIER, mask);
+       POSTING_READ(SDEIER);
+
+       ibx_enable_hotplug(dev);
+}
+
 static int ironlake_irq_postinstall(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
        /* enable kind of interrupts always enabled */
        u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
-                          DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
+                          DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
+                          DE_AUX_CHANNEL_A;
        u32 render_irqs;
-       u32 hotplug_mask;
 
        dev_priv->irq_mask = ~display_mask;
 
@@ -1848,27 +1982,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
        I915_WRITE(GTIER, render_irqs);
        POSTING_READ(GTIER);
 
-       if (HAS_PCH_CPT(dev)) {
-               hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
-                               SDE_PORTB_HOTPLUG_CPT |
-                               SDE_PORTC_HOTPLUG_CPT |
-                               SDE_PORTD_HOTPLUG_CPT);
-       } else {
-               hotplug_mask = (SDE_CRT_HOTPLUG |
-                               SDE_PORTB_HOTPLUG |
-                               SDE_PORTC_HOTPLUG |
-                               SDE_PORTD_HOTPLUG |
-                               SDE_AUX_MASK);
-       }
-
-       dev_priv->pch_irq_mask = ~hotplug_mask;
-
-       I915_WRITE(SDEIIR, I915_READ(SDEIIR));
-       I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
-       I915_WRITE(SDEIER, hotplug_mask);
-       POSTING_READ(SDEIER);
-
-       ironlake_enable_pch_hotplug(dev);
+       ibx_irq_postinstall(dev);
 
        if (IS_IRONLAKE_M(dev)) {
                /* Clear & enable PCU event interrupts */
@@ -1888,9 +2002,9 @@ static int ivybridge_irq_postinstall(struct drm_device *dev)
                DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB |
                DE_PLANEC_FLIP_DONE_IVB |
                DE_PLANEB_FLIP_DONE_IVB |
-               DE_PLANEA_FLIP_DONE_IVB;
+               DE_PLANEA_FLIP_DONE_IVB |
+               DE_AUX_CHANNEL_A_IVB;
        u32 render_irqs;
-       u32 hotplug_mask;
 
        dev_priv->irq_mask = ~display_mask;
 
@@ -1914,18 +2028,7 @@ static int ivybridge_irq_postinstall(struct drm_device *dev)
        I915_WRITE(GTIER, render_irqs);
        POSTING_READ(GTIER);
 
-       hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
-                       SDE_PORTB_HOTPLUG_CPT |
-                       SDE_PORTC_HOTPLUG_CPT |
-                       SDE_PORTD_HOTPLUG_CPT);
-       dev_priv->pch_irq_mask = ~hotplug_mask;
-
-       I915_WRITE(SDEIIR, I915_READ(SDEIIR));
-       I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
-       I915_WRITE(SDEIER, hotplug_mask);
-       POSTING_READ(SDEIER);
-
-       ironlake_enable_pch_hotplug(dev);
+       ibx_irq_postinstall(dev);
 
        return 0;
 }
@@ -1934,7 +2037,6 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
        u32 enable_mask;
-       u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
        u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV;
        u32 render_irqs;
        u16 msid;
@@ -1963,6 +2065,9 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
        msid |= (1<<14);
        pci_write_config_word(dev_priv->dev->pdev, 0x98, msid);
 
+       I915_WRITE(PORT_HOTPLUG_EN, 0);
+       POSTING_READ(PORT_HOTPLUG_EN);
+
        I915_WRITE(VLV_IMR, dev_priv->irq_mask);
        I915_WRITE(VLV_IER, enable_mask);
        I915_WRITE(VLV_IIR, 0xffffffff);
@@ -1971,6 +2076,7 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
        POSTING_READ(VLV_IER);
 
        i915_enable_pipestat(dev_priv, 0, pipestat_enable);
+       i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
        i915_enable_pipestat(dev_priv, 1, pipestat_enable);
 
        I915_WRITE(VLV_IIR, 0xffffffff);
@@ -1991,13 +2097,22 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
 #endif
 
        I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
+
+       return 0;
+}
+
+static void valleyview_hpd_irq_setup(struct drm_device *dev)
+{
+       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+       u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
+
        /* Note HDMI and DP share bits */
-       if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
-               hotplug_en |= HDMIB_HOTPLUG_INT_EN;
-       if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
-               hotplug_en |= HDMIC_HOTPLUG_INT_EN;
-       if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
-               hotplug_en |= HDMID_HOTPLUG_INT_EN;
+       if (dev_priv->hotplug_supported_mask & PORTB_HOTPLUG_INT_STATUS)
+               hotplug_en |= PORTB_HOTPLUG_INT_EN;
+       if (dev_priv->hotplug_supported_mask & PORTC_HOTPLUG_INT_STATUS)
+               hotplug_en |= PORTC_HOTPLUG_INT_EN;
+       if (dev_priv->hotplug_supported_mask & PORTD_HOTPLUG_INT_STATUS)
+               hotplug_en |= PORTD_HOTPLUG_INT_EN;
        if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_I915)
                hotplug_en |= SDVOC_HOTPLUG_INT_EN;
        if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS_I915)
@@ -2008,8 +2123,6 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
        }
 
        I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
-
-       return 0;
 }
 
 static void valleyview_irq_uninstall(struct drm_device *dev)
@@ -2238,6 +2351,9 @@ static int i915_irq_postinstall(struct drm_device *dev)
                I915_USER_INTERRUPT;
 
        if (I915_HAS_HOTPLUG(dev)) {
+               I915_WRITE(PORT_HOTPLUG_EN, 0);
+               POSTING_READ(PORT_HOTPLUG_EN);
+
                /* Enable in IER... */
                enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
                /* and unmask in IMR */
@@ -2248,15 +2364,25 @@ static int i915_irq_postinstall(struct drm_device *dev)
        I915_WRITE(IER, enable_mask);
        POSTING_READ(IER);
 
+       intel_opregion_enable_asle(dev);
+
+       return 0;
+}
+
+static void i915_hpd_irq_setup(struct drm_device *dev)
+{
+       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+       u32 hotplug_en;
+
        if (I915_HAS_HOTPLUG(dev)) {
-               u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
-
-               if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
-                       hotplug_en |= HDMIB_HOTPLUG_INT_EN;
-               if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
-                       hotplug_en |= HDMIC_HOTPLUG_INT_EN;
-               if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
-                       hotplug_en |= HDMID_HOTPLUG_INT_EN;
+               hotplug_en = I915_READ(PORT_HOTPLUG_EN);
+
+               if (dev_priv->hotplug_supported_mask & PORTB_HOTPLUG_INT_STATUS)
+                       hotplug_en |= PORTB_HOTPLUG_INT_EN;
+               if (dev_priv->hotplug_supported_mask & PORTC_HOTPLUG_INT_STATUS)
+                       hotplug_en |= PORTC_HOTPLUG_INT_EN;
+               if (dev_priv->hotplug_supported_mask & PORTD_HOTPLUG_INT_STATUS)
+                       hotplug_en |= PORTD_HOTPLUG_INT_EN;
                if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_I915)
                        hotplug_en |= SDVOC_HOTPLUG_INT_EN;
                if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS_I915)
@@ -2270,10 +2396,6 @@ static int i915_irq_postinstall(struct drm_device *dev)
 
                I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
        }
-
-       intel_opregion_enable_asle(dev);
-
-       return 0;
 }
 
 static irqreturn_t i915_irq_handler(void *arg)
@@ -2429,7 +2551,6 @@ static void i965_irq_preinstall(struct drm_device * dev)
 static int i965_irq_postinstall(struct drm_device *dev)
 {
        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-       u32 hotplug_en;
        u32 enable_mask;
        u32 error_mask;
 
@@ -2450,6 +2571,7 @@ static int i965_irq_postinstall(struct drm_device *dev)
 
        dev_priv->pipestat[0] = 0;
        dev_priv->pipestat[1] = 0;
+       i915_enable_pipestat(dev_priv, 0, PIPE_GMBUS_EVENT_ENABLE);
 
        /*
         * Enable some error detection, note the instruction error mask
@@ -2470,14 +2592,27 @@ static int i965_irq_postinstall(struct drm_device *dev)
        I915_WRITE(IER, enable_mask);
        POSTING_READ(IER);
 
+       I915_WRITE(PORT_HOTPLUG_EN, 0);
+       POSTING_READ(PORT_HOTPLUG_EN);
+
+       intel_opregion_enable_asle(dev);
+
+       return 0;
+}
+
+static void i965_hpd_irq_setup(struct drm_device *dev)
+{
+       drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+       u32 hotplug_en;
+
        /* Note HDMI and DP share hotplug bits */
        hotplug_en = 0;
-       if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
-               hotplug_en |= HDMIB_HOTPLUG_INT_EN;
-       if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
-               hotplug_en |= HDMIC_HOTPLUG_INT_EN;
-       if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
-               hotplug_en |= HDMID_HOTPLUG_INT_EN;
+       if (dev_priv->hotplug_supported_mask & PORTB_HOTPLUG_INT_STATUS)
+               hotplug_en |= PORTB_HOTPLUG_INT_EN;
+       if (dev_priv->hotplug_supported_mask & PORTC_HOTPLUG_INT_STATUS)
+               hotplug_en |= PORTC_HOTPLUG_INT_EN;
+       if (dev_priv->hotplug_supported_mask & PORTD_HOTPLUG_INT_STATUS)
+               hotplug_en |= PORTD_HOTPLUG_INT_EN;
        if (IS_G4X(dev)) {
                if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS_G4X)
                        hotplug_en |= SDVOC_HOTPLUG_INT_EN;
@@ -2504,10 +2639,6 @@ static int i965_irq_postinstall(struct drm_device *dev)
        /* Ignore TV since it's buggy */
 
        I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
-
-       intel_opregion_enable_asle(dev);
-
-       return 0;
 }
 
 static irqreturn_t i965_irq_handler(void *arg)
@@ -2599,6 +2730,9 @@ static irqreturn_t i965_irq_handler(void *arg)
                if (blc_event || (iir & I915_ASLE_INTERRUPT))
                        intel_opregion_asle_intr(dev);
 
+               if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
+                       gmbus_irq_handler(dev);
+
                /* With MSI, interrupts are only generated when iir
                 * transitions from zero to nonzero.  If another bit got
                 * set while we were handling the existing iir bits, then
@@ -2648,10 +2782,16 @@ void intel_irq_init(struct drm_device *dev)
        struct drm_i915_private *dev_priv = dev->dev_private;
 
        INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
-       INIT_WORK(&dev_priv->error_work, i915_error_work_func);
+       INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
        INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
        INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
 
+       setup_timer(&dev_priv->gpu_error.hangcheck_timer,
+                   i915_hangcheck_elapsed,
+                   (unsigned long) dev);
+
+       pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
+
        dev->driver->get_vblank_counter = i915_get_vblank_counter;
        dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
        if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
@@ -2672,7 +2812,8 @@ void intel_irq_init(struct drm_device *dev)
                dev->driver->irq_uninstall = valleyview_irq_uninstall;
                dev->driver->enable_vblank = valleyview_enable_vblank;
                dev->driver->disable_vblank = valleyview_disable_vblank;
-       } else if (IS_IVYBRIDGE(dev)) {
+               dev_priv->display.hpd_irq_setup = valleyview_hpd_irq_setup;
+       } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
                /* Share pre & uninstall handlers with ILK/SNB */
                dev->driver->irq_handler = ivybridge_irq_handler;
                dev->driver->irq_preinstall = ironlake_irq_preinstall;
@@ -2680,14 +2821,6 @@ void intel_irq_init(struct drm_device *dev)
                dev->driver->irq_uninstall = ironlake_irq_uninstall;
                dev->driver->enable_vblank = ivybridge_enable_vblank;
                dev->driver->disable_vblank = ivybridge_disable_vblank;
-       } else if (IS_HASWELL(dev)) {
-               /* Share interrupts handling with IVB */
-               dev->driver->irq_handler = ivybridge_irq_handler;
-               dev->driver->irq_preinstall = ironlake_irq_preinstall;
-               dev->driver->irq_postinstall = ivybridge_irq_postinstall;
-               dev->driver->irq_uninstall = ironlake_irq_uninstall;
-               dev->driver->enable_vblank = ivybridge_enable_vblank;
-               dev->driver->disable_vblank = ivybridge_disable_vblank;
        } else if (HAS_PCH_SPLIT(dev)) {
                dev->driver->irq_handler = ironlake_irq_handler;
                dev->driver->irq_preinstall = ironlake_irq_preinstall;
@@ -2706,13 +2839,23 @@ void intel_irq_init(struct drm_device *dev)
                        dev->driver->irq_postinstall = i915_irq_postinstall;
                        dev->driver->irq_uninstall = i915_irq_uninstall;
                        dev->driver->irq_handler = i915_irq_handler;
+                       dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
                } else {
                        dev->driver->irq_preinstall = i965_irq_preinstall;
                        dev->driver->irq_postinstall = i965_irq_postinstall;
                        dev->driver->irq_uninstall = i965_irq_uninstall;
                        dev->driver->irq_handler = i965_irq_handler;
+                       dev_priv->display.hpd_irq_setup = i965_hpd_irq_setup;
                }
                dev->driver->enable_vblank = i915_enable_vblank;
                dev->driver->disable_vblank = i915_disable_vblank;
        }
 }
+
+void intel_hpd_init(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
+       if (dev_priv->display.hpd_irq_setup)
+               dev_priv->display.hpd_irq_setup(dev);
+}
index ce70f0a..c91124f 100644 (file)
 #define   VGA_MSR_MEM_EN (1<<1)
 #define   VGA_MSR_CGA_MODE (1<<0)
 
-#define VGA_SR_INDEX 0x3c4
-#define VGA_SR_DATA 0x3c5
+/*
+ * SR01 is the only VGA register touched on non-UMS setups.
+ * VLV doesn't do UMS, so the sequencer index/data registers
+ * are the only VGA registers which need to include
+ * display_mmio_offset.
+ */
+#define VGA_SR_INDEX (dev_priv->info->display_mmio_offset + 0x3c4)
+#define SR01                   1
+#define VGA_SR_DATA (dev_priv->info->display_mmio_offset + 0x3c5)
 
 #define VGA_AR_INDEX 0x3c0
 #define   VGA_AR_VID_EN (1<<5)
 #define   DISPLAY_PLANE_A           (0<<20)
 #define   DISPLAY_PLANE_B           (1<<20)
 #define GFX_OP_PIPE_CONTROL(len)       ((0x3<<29)|(0x3<<27)|(0x2<<24)|(len-2))
+#define   PIPE_CONTROL_GLOBAL_GTT_IVB                  (1<<24) /* gen7+ */
 #define   PIPE_CONTROL_CS_STALL                                (1<<20)
 #define   PIPE_CONTROL_TLB_INVALIDATE                  (1<<18)
 #define   PIPE_CONTROL_QW_WRITE                                (1<<14)
  *  0x801c/3c: core clock bits
  *  0x8048/68: low pass filter coefficients
  *  0x8100: fast clock controls
+ *
+ * DPIO is VLV only.
  */
-#define DPIO_PKT                       0x2100
+#define DPIO_PKT                       (VLV_DISPLAY_BASE + 0x2100)
 #define  DPIO_RID                      (0<<24)
 #define  DPIO_OP_WRITE                 (1<<16)
 #define  DPIO_OP_READ                  (0<<16)
 #define  DPIO_PORTID                   (0x12<<8)
 #define  DPIO_BYTE                     (0xf<<4)
 #define  DPIO_BUSY                     (1<<0) /* status only */
-#define DPIO_DATA                      0x2104
-#define DPIO_REG                       0x2108
-#define DPIO_CTL                       0x2110
+#define DPIO_DATA                      (VLV_DISPLAY_BASE + 0x2104)
+#define DPIO_REG                       (VLV_DISPLAY_BASE + 0x2108)
+#define DPIO_CTL                       (VLV_DISPLAY_BASE + 0x2110)
 #define  DPIO_MODSEL1                  (1<<3) /* if ref clk b == 27 */
 #define  DPIO_MODSEL0                  (1<<2) /* if ref clk a == 27 */
 #define  DPIO_SFR_BYPASS               (1<<1)
 #define IIR            0x020a4
 #define IMR            0x020a8
 #define ISR            0x020ac
-#define VLV_GUNIT_CLOCK_GATE   0x182060
+#define VLV_GUNIT_CLOCK_GATE   (VLV_DISPLAY_BASE + 0x2060)
 #define   GCFG_DIS             (1<<8)
-#define VLV_IIR_RW     0x182084
-#define VLV_IER                0x1820a0
-#define VLV_IIR                0x1820a4
-#define VLV_IMR                0x1820a8
-#define VLV_ISR                0x1820ac
+#define VLV_IIR_RW     (VLV_DISPLAY_BASE + 0x2084)
+#define VLV_IER                (VLV_DISPLAY_BASE + 0x20a0)
+#define VLV_IIR                (VLV_DISPLAY_BASE + 0x20a4)
+#define VLV_IMR                (VLV_DISPLAY_BASE + 0x20a8)
+#define VLV_ISR                (VLV_DISPLAY_BASE + 0x20ac)
 #define   I915_PIPE_CONTROL_NOTIFY_INTERRUPT           (1<<18)
 #define   I915_DISPLAY_PORT_INTERRUPT                  (1<<17)
 #define   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT   (1<<15)
 #define   GEN7_FF_TS_SCHED_HS0         (0x3<<16)
 #define   GEN7_FF_TS_SCHED_LOAD_BALANCE        (0x1<<16)
 #define   GEN7_FF_TS_SCHED_HW          (0x0<<16) /* Default */
+#define   GEN7_FF_VS_REF_CNT_FFME      (1 << 15)
 #define   GEN7_FF_VS_SCHED_HS1         (0x5<<12)
 #define   GEN7_FF_VS_SCHED_HS0         (0x3<<12)
 #define   GEN7_FF_VS_SCHED_LOAD_BALANCE        (0x1<<12) /* Default */
 #define   VGA1_PD_P1_DIV_2     (1 << 13)
 #define   VGA1_PD_P1_SHIFT     8
 #define   VGA1_PD_P1_MASK      (0x1f << 8)
-#define _DPLL_A        0x06014
-#define _DPLL_B        0x06018
+#define _DPLL_A        (dev_priv->info->display_mmio_offset + 0x6014)
+#define _DPLL_B        (dev_priv->info->display_mmio_offset + 0x6018)
 #define DPLL(pipe) _PIPE(pipe, _DPLL_A, _DPLL_B)
 #define   DPLL_VCO_ENABLE              (1 << 31)
 #define   DPLL_DVO_HIGH_SPEED          (1 << 30)
 #define   DPLL_LOCK_VLV                        (1<<15)
 #define   DPLL_INTEGRATED_CLOCK_VLV    (1<<13)
 
-#define SRX_INDEX              0x3c4
-#define SRX_DATA               0x3c5
-#define SR01                   1
-#define SR01_SCREEN_OFF                (1<<5)
-
-#define PPCR                   0x61204
-#define PPCR_ON                        (1<<0)
-
-#define DVOB                   0x61140
-#define DVOB_ON                        (1<<31)
-#define DVOC                   0x61160
-#define DVOC_ON                        (1<<31)
-#define LVDS                   0x61180
-#define LVDS_ON                        (1<<31)
-
-/* Scratch pad debug 0 reg:
- */
 #define   DPLL_FPA01_P1_POST_DIV_MASK_I830     0x001f0000
 /*
  * The i830 generation, in LVDS mode, defines P1 as the bit number set within
 #define   SDVO_MULTIPLIER_MASK                 0x000000ff
 #define   SDVO_MULTIPLIER_SHIFT_HIRES          4
 #define   SDVO_MULTIPLIER_SHIFT_VGA            0
-#define _DPLL_A_MD 0x0601c /* 965+ only */
+#define _DPLL_A_MD (dev_priv->info->display_mmio_offset + 0x601c) /* 965+ only */
 /*
  * UDI pixel divider, controlling how many pixels are stuffed into a packet.
  *
  */
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_MASK      0x0000003f
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT     0
-#define _DPLL_B_MD 0x06020 /* 965+ only */
+#define _DPLL_B_MD (dev_priv->info->display_mmio_offset + 0x6020) /* 965+ only */
 #define DPLL_MD(pipe) _PIPE(pipe, _DPLL_A_MD, _DPLL_B_MD)
 
 #define _FPA0  0x06040
 #define RAMCLK_GATE_D          0x6210          /* CRL only */
 #define DEUC                   0x6214          /* CRL only */
 
-#define FW_BLC_SELF_VLV                0x6500
+#define FW_BLC_SELF_VLV                (VLV_DISPLAY_BASE + 0x6500)
 #define  FW_CSPWRDWNEN         (1<<15)
 
 /*
  * Palette regs
  */
 
-#define _PALETTE_A             0x0a000
-#define _PALETTE_B             0x0a800
+#define _PALETTE_A             (dev_priv->info->display_mmio_offset + 0xa000)
+#define _PALETTE_B             (dev_priv->info->display_mmio_offset + 0xa800)
 #define PALETTE(pipe) _PIPE(pipe, _PALETTE_A, _PALETTE_B)
 
 /* MCH MMIO space */
 #define   MAD_DIMM_A_SIZE_SHIFT                0
 #define   MAD_DIMM_A_SIZE_MASK         (0xff << MAD_DIMM_A_SIZE_SHIFT)
 
+/** snb MCH registers for priority tuning */
+#define MCH_SSKPD                      (MCHBAR_MIRROR_BASE_SNB + 0x5d10)
+#define   MCH_SSKPD_WM0_MASK           0x3f
+#define   MCH_SSKPD_WM0_VAL            0xc
 
 /* Clocking configuration register */
 #define CLKCFG                 0x10c00
  */
 
 /* Pipe A timing regs */
-#define _HTOTAL_A      0x60000
-#define _HBLANK_A      0x60004
-#define _HSYNC_A               0x60008
-#define _VTOTAL_A      0x6000c
-#define _VBLANK_A      0x60010
-#define _VSYNC_A               0x60014
-#define _PIPEASRC      0x6001c
-#define _BCLRPAT_A     0x60020
-#define _VSYNCSHIFT_A  0x60028
+#define _HTOTAL_A      (dev_priv->info->display_mmio_offset + 0x60000)
+#define _HBLANK_A      (dev_priv->info->display_mmio_offset + 0x60004)
+#define _HSYNC_A       (dev_priv->info->display_mmio_offset + 0x60008)
+#define _VTOTAL_A      (dev_priv->info->display_mmio_offset + 0x6000c)
+#define _VBLANK_A      (dev_priv->info->display_mmio_offset + 0x60010)
+#define _VSYNC_A       (dev_priv->info->display_mmio_offset + 0x60014)
+#define _PIPEASRC      (dev_priv->info->display_mmio_offset + 0x6001c)
+#define _BCLRPAT_A     (dev_priv->info->display_mmio_offset + 0x60020)
+#define _VSYNCSHIFT_A  (dev_priv->info->display_mmio_offset + 0x60028)
 
 /* Pipe B timing regs */
-#define _HTOTAL_B      0x61000
-#define _HBLANK_B      0x61004
-#define _HSYNC_B               0x61008
-#define _VTOTAL_B      0x6100c
-#define _VBLANK_B      0x61010
-#define _VSYNC_B               0x61014
-#define _PIPEBSRC      0x6101c
-#define _BCLRPAT_B     0x61020
-#define _VSYNCSHIFT_B  0x61028
+#define _HTOTAL_B      (dev_priv->info->display_mmio_offset + 0x61000)
+#define _HBLANK_B      (dev_priv->info->display_mmio_offset + 0x61004)
+#define _HSYNC_B       (dev_priv->info->display_mmio_offset + 0x61008)
+#define _VTOTAL_B      (dev_priv->info->display_mmio_offset + 0x6100c)
+#define _VBLANK_B      (dev_priv->info->display_mmio_offset + 0x61010)
+#define _VSYNC_B       (dev_priv->info->display_mmio_offset + 0x61014)
+#define _PIPEBSRC      (dev_priv->info->display_mmio_offset + 0x6101c)
+#define _BCLRPAT_B     (dev_priv->info->display_mmio_offset + 0x61020)
+#define _VSYNCSHIFT_B  (dev_priv->info->display_mmio_offset + 0x61028)
 
 
 #define HTOTAL(trans) _TRANSCODER(trans, _HTOTAL_A, _HTOTAL_B)
 #define   ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1<<16)
 #define   ADPA_USE_VGA_HVPOLARITY (1<<15)
 #define   ADPA_SETS_HVPOLARITY 0
-#define   ADPA_VSYNC_CNTL_DISABLE (1<<11)
+#define   ADPA_VSYNC_CNTL_DISABLE (1<<10)
 #define   ADPA_VSYNC_CNTL_ENABLE 0
-#define   ADPA_HSYNC_CNTL_DISABLE (1<<10)
+#define   ADPA_HSYNC_CNTL_DISABLE (1<<11)
 #define   ADPA_HSYNC_CNTL_ENABLE 0
 #define   ADPA_VSYNC_ACTIVE_HIGH (1<<4)
 #define   ADPA_VSYNC_ACTIVE_LOW        0
 
 
 /* Hotplug control (945+ only) */
-#define PORT_HOTPLUG_EN                0x61110
-#define   HDMIB_HOTPLUG_INT_EN                 (1 << 29)
-#define   DPB_HOTPLUG_INT_EN                   (1 << 29)
-#define   HDMIC_HOTPLUG_INT_EN                 (1 << 28)
-#define   DPC_HOTPLUG_INT_EN                   (1 << 28)
-#define   HDMID_HOTPLUG_INT_EN                 (1 << 27)
-#define   DPD_HOTPLUG_INT_EN                   (1 << 27)
+#define PORT_HOTPLUG_EN                (dev_priv->info->display_mmio_offset + 0x61110)
+#define   PORTB_HOTPLUG_INT_EN                 (1 << 29)
+#define   PORTC_HOTPLUG_INT_EN                 (1 << 28)
+#define   PORTD_HOTPLUG_INT_EN                 (1 << 27)
 #define   SDVOB_HOTPLUG_INT_EN                 (1 << 26)
 #define   SDVOC_HOTPLUG_INT_EN                 (1 << 25)
 #define   TV_HOTPLUG_INT_EN                    (1 << 18)
 #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV       (0 << 2)
 #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV       (1 << 2)
 
-#define PORT_HOTPLUG_STAT      0x61114
+#define PORT_HOTPLUG_STAT      (dev_priv->info->display_mmio_offset + 0x61114)
 /* HDMI/DP bits are gen4+ */
-#define   DPB_HOTPLUG_LIVE_STATUS               (1 << 29)
-#define   DPC_HOTPLUG_LIVE_STATUS               (1 << 28)
-#define   DPD_HOTPLUG_LIVE_STATUS               (1 << 27)
-#define   DPD_HOTPLUG_INT_STATUS               (3 << 21)
-#define   DPC_HOTPLUG_INT_STATUS               (3 << 19)
-#define   DPB_HOTPLUG_INT_STATUS               (3 << 17)
-/* HDMI bits are shared with the DP bits */
-#define   HDMIB_HOTPLUG_LIVE_STATUS             (1 << 29)
-#define   HDMIC_HOTPLUG_LIVE_STATUS             (1 << 28)
-#define   HDMID_HOTPLUG_LIVE_STATUS             (1 << 27)
-#define   HDMID_HOTPLUG_INT_STATUS             (3 << 21)
-#define   HDMIC_HOTPLUG_INT_STATUS             (3 << 19)
-#define   HDMIB_HOTPLUG_INT_STATUS             (3 << 17)
+#define   PORTB_HOTPLUG_LIVE_STATUS               (1 << 29)
+#define   PORTC_HOTPLUG_LIVE_STATUS               (1 << 28)
+#define   PORTD_HOTPLUG_LIVE_STATUS               (1 << 27)
+#define   PORTD_HOTPLUG_INT_STATUS             (3 << 21)
+#define   PORTC_HOTPLUG_INT_STATUS             (3 << 19)
+#define   PORTB_HOTPLUG_INT_STATUS             (3 << 17)
 /* CRT/TV common between gen3+ */
 #define   CRT_HOTPLUG_INT_STATUS               (1 << 11)
 #define   TV_HOTPLUG_INT_STATUS                        (1 << 10)
 #define PP_DIVISOR     0x61210
 
 /* Panel fitting */
-#define PFIT_CONTROL   0x61230
+#define PFIT_CONTROL   (dev_priv->info->display_mmio_offset + 0x61230)
 #define   PFIT_ENABLE          (1 << 31)
 #define   PFIT_PIPE_MASK       (3 << 29)
 #define   PFIT_PIPE_SHIFT      29
 #define   PFIT_SCALING_PROGRAMMED (1 << 26)
 #define   PFIT_SCALING_PILLAR  (2 << 26)
 #define   PFIT_SCALING_LETTER  (3 << 26)
-#define PFIT_PGM_RATIOS        0x61234
-#define   PFIT_VERT_SCALE_MASK                 0xfff00000
-#define   PFIT_HORIZ_SCALE_MASK                        0x0000fff0
+#define PFIT_PGM_RATIOS        (dev_priv->info->display_mmio_offset + 0x61234)
 /* Pre-965 */
 #define                PFIT_VERT_SCALE_SHIFT           20
 #define                PFIT_VERT_SCALE_MASK            0xfff00000
 #define                PFIT_HORIZ_SCALE_SHIFT_965      0
 #define                PFIT_HORIZ_SCALE_MASK_965       0x00001fff
 
-#define PFIT_AUTO_RATIOS 0x61238
+#define PFIT_AUTO_RATIOS (dev_priv->info->display_mmio_offset + 0x61238)
 
 /* Backlight control */
 #define BLC_PWM_CTL2           0x61250 /* 965+ only */
 /* Display & cursor control */
 
 /* Pipe A */
-#define _PIPEADSL              0x70000
+#define _PIPEADSL              (dev_priv->info->display_mmio_offset + 0x70000)
 #define   DSL_LINEMASK_GEN2    0x00000fff
 #define   DSL_LINEMASK_GEN3    0x00001fff
-#define _PIPEACONF             0x70008
+#define _PIPEACONF             (dev_priv->info->display_mmio_offset + 0x70008)
 #define   PIPECONF_ENABLE      (1<<31)
 #define   PIPECONF_DISABLE     0
 #define   PIPECONF_DOUBLE_WIDE (1<<30)
 #define   PIPECONF_INTERLACED_DBL_ILK          (4 << 21) /* ilk/snb only */
 #define   PIPECONF_PFIT_PF_INTERLACED_DBL_ILK  (5 << 21) /* ilk/snb only */
 #define   PIPECONF_CXSR_DOWNCLOCK      (1<<16)
-#define   PIPECONF_BPP_MASK    (0x000000e0)
-#define   PIPECONF_BPP_8       (0<<5)
-#define   PIPECONF_BPP_10      (1<<5)
-#define   PIPECONF_BPP_6       (2<<5)
-#define   PIPECONF_BPP_12      (3<<5)
+#define   PIPECONF_COLOR_RANGE_SELECT  (1 << 13)
+#define   PIPECONF_BPC_MASK    (0x7 << 5)
+#define   PIPECONF_8BPC                (0<<5)
+#define   PIPECONF_10BPC       (1<<5)
+#define   PIPECONF_6BPC                (2<<5)
+#define   PIPECONF_12BPC       (3<<5)
 #define   PIPECONF_DITHER_EN   (1<<4)
 #define   PIPECONF_DITHER_TYPE_MASK (0x0000000c)
 #define   PIPECONF_DITHER_TYPE_SP (0<<2)
 #define   PIPECONF_DITHER_TYPE_ST1 (1<<2)
 #define   PIPECONF_DITHER_TYPE_ST2 (2<<2)
 #define   PIPECONF_DITHER_TYPE_TEMP (3<<2)
-#define _PIPEASTAT             0x70024
+#define _PIPEASTAT             (dev_priv->info->display_mmio_offset + 0x70024)
 #define   PIPE_FIFO_UNDERRUN_STATUS            (1UL<<31)
 #define   SPRITE1_FLIPDONE_INT_EN_VLV          (1UL<<30)
 #define   PIPE_CRC_ERROR_ENABLE                        (1UL<<29)
 #define   PIPE_VSYNC_INTERRUPT_ENABLE          (1UL<<25)
 #define   PIPE_DISPLAY_LINE_COMPARE_ENABLE     (1UL<<24)
 #define   PIPE_DPST_EVENT_ENABLE               (1UL<<23)
-#define   SPRITE0_FLIP_DONE_INT_EN_VLV         (1UL<<26)
+#define   SPRITE0_FLIP_DONE_INT_EN_VLV         (1UL<<22)
 #define   PIPE_LEGACY_BLC_EVENT_ENABLE         (1UL<<22)
 #define   PIPE_ODD_FIELD_INTERRUPT_ENABLE      (1UL<<21)
 #define   PIPE_EVEN_FIELD_INTERRUPT_ENABLE     (1UL<<20)
 #define   PIPEA_HBLANK_INT_EN_VLV              (1UL<<16)
 #define   PIPE_OVERLAY_UPDATED_ENABLE          (1UL<<16)
 #define   SPRITE1_FLIPDONE_INT_STATUS_VLV      (1UL<<15)
-#define   SPRITE0_FLIPDONE_INT_STATUS_VLV      (1UL<<15)
+#define   SPRITE0_FLIPDONE_INT_STATUS_VLV      (1UL<<14)
 #define   PIPE_CRC_ERROR_INTERRUPT_STATUS      (1UL<<13)
 #define   PIPE_CRC_DONE_INTERRUPT_STATUS       (1UL<<12)
 #define   PIPE_GMBUS_INTERRUPT_STATUS          (1UL<<11)
 #define   PIPE_START_VBLANK_INTERRUPT_STATUS   (1UL<<2) /* 965 or later */
 #define   PIPE_VBLANK_INTERRUPT_STATUS         (1UL<<1)
 #define   PIPE_OVERLAY_UPDATED_STATUS          (1UL<<0)
-#define   PIPE_BPC_MASK                                (7 << 5) /* Ironlake */
-#define   PIPE_8BPC                            (0 << 5)
-#define   PIPE_10BPC                           (1 << 5)
-#define   PIPE_6BPC                            (2 << 5)
-#define   PIPE_12BPC                           (3 << 5)
 
 #define PIPESRC(pipe) _PIPE(pipe, _PIPEASRC, _PIPEBSRC)
 #define PIPECONF(tran) _TRANSCODER(tran, _PIPEACONF, _PIPEBCONF)