sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64
  65 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  66 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  67
  68 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  69     int tiling_mode);
  70 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  71     uint32_t size, int tiling_mode);
  72 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  73     unsigned alignment, bool map_and_fenceable);
  74 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  75     int flags);
  76 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj);
  77 static int i915_gem_object_set_cpu_read_domain_range(
  78     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  79 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  80 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  81 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  82 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  83 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  84 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  85 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  86     uint32_t flush_domains);
  87 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  88     struct drm_i915_fence_reg *reg);
  89 static void i915_gem_reset_fences(struct drm_device *dev);
  90 static int i915_gem_phys_pwrite(struct drm_device *dev,
  91     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  92     uint64_t size, struct drm_file *file_priv);
  93 static void i915_gem_lowmem(void *arg);
  94
  95 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
  96 long i915_gem_wired_pages_cnt;
  97
  98 static void
  99 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size)
 100 {
 101
 102         dev_priv->mm.object_count++;
 103         dev_priv->mm.object_memory += size;
 104 }
 105
 106 static void
 107 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size)
 108 {
 109
 110         dev_priv->mm.object_count--;
 111         dev_priv->mm.object_memory -= size;
 112 }
 113
 114 static int
 115 i915_gem_wait_for_error(struct drm_device *dev)
 116 {
 117         struct drm_i915_private *dev_priv = dev->dev_private;
 118         int ret;
 119
 120         if (!atomic_read(&dev_priv->mm.wedged))
 121                 return 0;
 122
 123         lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 124         while (dev_priv->error_completion == 0) {
 125                 ret = -lksleep(&dev_priv->error_completion,
 126                     &dev_priv->error_completion_lock, PCATCH, "915wco", 0);
 127                 if (ret != 0) {
 128                         lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 129                         return (ret);
 130                 }
 131         }
 132         lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 133
 134         if (atomic_read(&dev_priv->mm.wedged)) {
 135                 /* GPU is hung, bump the completion count to account for
 136                  * the token we just consumed so that we never hit zero and
 137                  * end up waiting upon a subsequent completion event that
 138                  * will never happen.
 139                  */
 140                 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 141                 dev_priv->error_completion++;
 142                 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 143         }
 144         return 0;
 145 }
 146
 147 int
 148 i915_mutex_lock_interruptible(struct drm_device *dev)
 149 {
 150         struct drm_i915_private *dev_priv;
 151         int ret;
 152
 153         dev_priv = dev->dev_private;
 154         ret = i915_gem_wait_for_error(dev);
 155         if (ret != 0)
 156                 return (ret);
 157
 158         /*
 159          * interruptible shall it be. might indeed be if dev_lock is
 160          * changed to sx
 161          */
 162         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 163         if (ret != 0)
 164                 return (-ret);
 165
 166         return (0);
 167 }
 168
 169
 170 static void
 171 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
 172 {
 173         struct drm_device *dev;
 174         drm_i915_private_t *dev_priv;
 175         int ret;
 176
 177         dev = obj->base.dev;
 178         dev_priv = dev->dev_private;
 179
 180         ret = i915_gem_object_unbind(obj);
 181         if (ret == -ERESTART) {
 182                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
 183                 return;
 184         }
 185
 186         drm_gem_free_mmap_offset(&obj->base);
 187         drm_gem_object_release(&obj->base);
 188         i915_gem_info_remove_obj(dev_priv, obj->base.size);
 189
 190         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
 191         drm_free(obj->bit_17, DRM_I915_GEM);
 192         drm_free(obj, DRM_I915_GEM);
 193 }
 194
 195 void
 196 i915_gem_free_object(struct drm_gem_object *gem_obj)
 197 {
 198         struct drm_i915_gem_object *obj;
 199         struct drm_device *dev;
 200
 201         obj = to_intel_bo(gem_obj);
 202         dev = obj->base.dev;
 203
 204         while (obj->pin_count > 0)
 205                 i915_gem_object_unpin(obj);
 206
 207         if (obj->phys_obj != NULL)
 208                 i915_gem_detach_phys_object(dev, obj);
 209
 210         i915_gem_free_object_tail(obj);
 211 }
 212
 213 int
 214 i915_gem_do_init(struct drm_device *dev, unsigned long start,
 215     unsigned long mappable_end, unsigned long end)
 216 {
 217         drm_i915_private_t *dev_priv;
 218         unsigned long mappable;
 219         int error;
 220
 221         dev_priv = dev->dev_private;
 222         mappable = min(end, mappable_end) - start;
 223
 224         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
 225
 226         dev_priv->mm.gtt_start = start;
 227         dev_priv->mm.gtt_mappable_end = mappable_end;
 228         dev_priv->mm.gtt_end = end;
 229         dev_priv->mm.gtt_total = end - start;
 230         dev_priv->mm.mappable_gtt_total = mappable;
 231
 232         /* Take over this portion of the GTT */
 233         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 234         device_printf(dev->dev,
 235             "taking over the fictitious range 0x%lx-0x%lx\n",
 236             dev->agp->base + start, dev->agp->base + start + mappable);
 237         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
 238             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
 239         return (error);
 240 }
 241
 242 int
 243 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 244     struct drm_file *file)
 245 {
 246         struct drm_i915_gem_init *args;
 247         drm_i915_private_t *dev_priv;
 248
 249         dev_priv = dev->dev_private;
 250         args = data;
 251
 252         if (args->gtt_start >= args->gtt_end ||
 253             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 254                 return (-EINVAL);
 255
 256         /*
 257          * XXXKIB. The second-time initialization should be guarded
 258          * against.
 259          */
 260         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 261         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 262         lockmgr(&dev->dev_lock, LK_RELEASE);
 263
 264         return 0;
 265 }
 266
 267 int
 268 i915_gem_idle(struct drm_device *dev)
 269 {
 270         drm_i915_private_t *dev_priv;
 271         int ret;
 272
 273         dev_priv = dev->dev_private;
 274         if (dev_priv->mm.suspended)
 275                 return (0);
 276
 277         ret = i915_gpu_idle(dev);
 278         if (ret != 0)
 279                 return (ret);
 280
 281         /* Under UMS, be paranoid and evict. */
 282         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
 283                 ret = i915_gem_evict_inactive(dev, false);
 284                 if (ret != 0)
 285                         return ret;
 286         }
 287
 288         i915_gem_reset_fences(dev);
 289
 290         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
 291          * We need to replace this with a semaphore, or something.
 292          * And not confound mm.suspended!
 293          */
 294         dev_priv->mm.suspended = 1;
 295         del_timer_sync(&dev_priv->hangcheck_timer);
 296
 297         i915_kernel_lost_context(dev);
 298         i915_gem_cleanup_ringbuffer(dev);
 299
 300         /* Cancel the retire work handler, which should be idle now. */
 301         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
 302
 303         return (ret);
 304 }
 305
 306 void
 307 i915_gem_init_swizzling(struct drm_device *dev)
 308 {
 309         drm_i915_private_t *dev_priv;
 310
 311         dev_priv = dev->dev_private;
 312
 313         if (INTEL_INFO(dev)->gen < 5 ||
 314             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
 315                 return;
 316
 317         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 318                                  DISP_TILE_SURFACE_SWIZZLING);
 319
 320         if (IS_GEN5(dev))
 321                 return;
 322
 323         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
 324         if (IS_GEN6(dev))
 325                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
 326         else
 327                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
 328 }
 329
 330 int
 331 i915_gem_init_hw(struct drm_device *dev)
 332 {
 333         drm_i915_private_t *dev_priv;
 334         int ret;
 335
 336         dev_priv = dev->dev_private;
 337
 338         i915_gem_init_swizzling(dev);
 339
 340         ret = intel_init_render_ring_buffer(dev);
 341         if (ret != 0)
 342                 return (ret);
 343
 344         if (HAS_BSD(dev)) {
 345                 ret = intel_init_bsd_ring_buffer(dev);
 346                 if (ret != 0)
 347                         goto cleanup_render_ring;
 348         }
 349
 350         if (HAS_BLT(dev)) {
 351                 ret = intel_init_blt_ring_buffer(dev);
 352                 if (ret != 0)
 353                         goto cleanup_bsd_ring;
 354         }
 355
 356         dev_priv->next_seqno = 1;
 357         i915_gem_init_ppgtt(dev);
 358         return (0);
 359
 360 cleanup_bsd_ring:
 361         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
 362 cleanup_render_ring:
 363         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
 364         return (ret);
 365 }
 366
 367 int
 368 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 369     struct drm_file *file)
 370 {
 371         struct drm_i915_private *dev_priv;
 372         struct drm_i915_gem_get_aperture *args;
 373         struct drm_i915_gem_object *obj;
 374         size_t pinned;
 375
 376         dev_priv = dev->dev_private;
 377         args = data;
 378
 379         if (!(dev->driver->driver_features & DRIVER_GEM))
 380                 return (-ENODEV);
 381
 382         pinned = 0;
 383         DRM_LOCK(dev);
 384         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 385                 pinned += obj->gtt_space->size;
 386         DRM_UNLOCK(dev);
 387
 388         args->aper_size = dev_priv->mm.gtt_total;
 389         args->aper_available_size = args->aper_size - pinned;
 390
 391         return (0);
 392 }
 393
 394 int
 395 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
 396      bool map_and_fenceable)
 397 {
 398         struct drm_device *dev;
 399         struct drm_i915_private *dev_priv;
 400         int ret;
 401
 402         dev = obj->base.dev;
 403         dev_priv = dev->dev_private;
 404
 405         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
 406             ("Max pin count"));
 407
 408         if (obj->gtt_space != NULL) {
 409                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
 410                     (map_and_fenceable && !obj->map_and_fenceable)) {
 411                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
 412                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
 413                              " obj->map_and_fenceable=%d\n",
 414                              obj->gtt_offset, alignment,
 415                              map_and_fenceable,
 416                              obj->map_and_fenceable);
 417                         ret = i915_gem_object_unbind(obj);
 418                         if (ret != 0)
 419                                 return (ret);
 420                 }
 421         }
 422
 423         if (obj->gtt_space == NULL) {
 424                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
 425                     map_and_fenceable);
 426                 if (ret)
 427                         return (ret);
 428         }
 429
 430         if (obj->pin_count++ == 0 && !obj->active)
 431                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
 432         obj->pin_mappable |= map_and_fenceable;
 433
 434 #if 1
 435         KIB_NOTYET();
 436 #else
 437         WARN_ON(i915_verify_lists(dev));
 438 #endif
 439         return (0);
 440 }
 441
 442 void
 443 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
 444 {
 445         struct drm_device *dev;
 446         drm_i915_private_t *dev_priv;
 447
 448         dev = obj->base.dev;
 449         dev_priv = dev->dev_private;
 450
 451 #if 1
 452         KIB_NOTYET();
 453 #else
 454         WARN_ON(i915_verify_lists(dev));
 455 #endif
 456
 457         KASSERT(obj->pin_count != 0, ("zero pin count"));
 458         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
 459
 460         if (--obj->pin_count == 0) {
 461                 if (!obj->active)
 462                         list_move_tail(&obj->mm_list,
 463                             &dev_priv->mm.inactive_list);
 464                 obj->pin_mappable = false;
 465         }
 466 #if 1
 467         KIB_NOTYET();
 468 #else
 469         WARN_ON(i915_verify_lists(dev));
 470 #endif
 471 }
 472
 473 int
 474 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 475     struct drm_file *file)
 476 {
 477         struct drm_i915_gem_pin *args;
 478         struct drm_i915_gem_object *obj;
 479         struct drm_gem_object *gobj;
 480         int ret;
 481
 482         args = data;
 483
 484         ret = i915_mutex_lock_interruptible(dev);
 485         if (ret != 0)
 486                 return ret;
 487
 488         gobj = drm_gem_object_lookup(dev, file, args->handle);
 489         if (gobj == NULL) {
 490                 ret = -ENOENT;
 491                 goto unlock;
 492         }
 493         obj = to_intel_bo(gobj);
 494
 495         if (obj->madv != I915_MADV_WILLNEED) {
 496                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
 497                 ret = -EINVAL;
 498                 goto out;
 499         }
 500
 501         if (obj->pin_filp != NULL && obj->pin_filp != file) {
 502                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
 503                     args->handle);
 504                 ret = -EINVAL;
 505                 goto out;
 506         }
 507
 508         obj->user_pin_count++;
 509         obj->pin_filp = file;
 510         if (obj->user_pin_count == 1) {
 511                 ret = i915_gem_object_pin(obj, args->alignment, true);
 512                 if (ret != 0)
 513                         goto out;
 514         }
 515
 516         /* XXX - flush the CPU caches for pinned objects
 517          * as the X server doesn't manage domains yet
 518          */
 519         i915_gem_object_flush_cpu_write_domain(obj);
 520         args->offset = obj->gtt_offset;
 521 out:
 522         drm_gem_object_unreference(&obj->base);
 523 unlock:
 524         DRM_UNLOCK(dev);
 525         return (ret);
 526 }
 527
 528 int
 529 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
 530     struct drm_file *file)
 531 {
 532         struct drm_i915_gem_pin *args;
 533         struct drm_i915_gem_object *obj;
 534         int ret;
 535
 536         args = data;
 537         ret = i915_mutex_lock_interruptible(dev);
 538         if (ret != 0)
 539                 return (ret);
 540
 541         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 542         if (&obj->base == NULL) {
 543                 ret = -ENOENT;
 544                 goto unlock;
 545         }
 546
 547         if (obj->pin_filp != file) {
 548                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
 549                     args->handle);
 550                 ret = -EINVAL;
 551                 goto out;
 552         }
 553         obj->user_pin_count--;
 554         if (obj->user_pin_count == 0) {
 555                 obj->pin_filp = NULL;
 556                 i915_gem_object_unpin(obj);
 557         }
 558
 559 out:
 560         drm_gem_object_unreference(&obj->base);
 561 unlock:
 562         DRM_UNLOCK(dev);
 563         return (ret);
 564 }
 565
 566 int
 567 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 568     struct drm_file *file)
 569 {
 570         struct drm_i915_gem_busy *args;
 571         struct drm_i915_gem_object *obj;
 572         struct drm_i915_gem_request *request;
 573         int ret;
 574
 575         args = data;
 576
 577         ret = i915_mutex_lock_interruptible(dev);
 578         if (ret != 0)
 579                 return ret;
 580
 581         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 582         if (&obj->base == NULL) {
 583                 ret = -ENOENT;
 584                 goto unlock;
 585         }
 586
 587         args->busy = obj->active;
 588         if (args->busy) {
 589                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
 590                         ret = i915_gem_flush_ring(obj->ring,
 591                             0, obj->base.write_domain);
 592                 } else if (obj->ring->outstanding_lazy_request ==
 593                     obj->last_rendering_seqno) {
 594                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
 595                             M_WAITOK | M_ZERO);
 596                         ret = i915_add_request(obj->ring, NULL, request);
 597                         if (ret != 0)
 598                                 drm_free(request, DRM_I915_GEM);
 599                 }
 600
 601                 i915_gem_retire_requests_ring(obj->ring);
 602                 args->busy = obj->active;
 603         }
 604
 605         drm_gem_object_unreference(&obj->base);
 606 unlock:
 607         DRM_UNLOCK(dev);
 608         return (ret);
 609 }
 610
 611 static int
 612 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
 613 {
 614
 615         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
 616                 return (0);
 617         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
 618 }
 619
 620 /**
 621  * Moves a single object to the CPU read, and possibly write domain.
 622  *
 623  * This function returns when the move is complete, including waiting on
 624  * flushes to occur.
 625  */
 626 int
 627 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 628 {
 629         uint32_t old_write_domain, old_read_domains;
 630         int ret;
 631
 632         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
 633                 return 0;
 634
 635         ret = i915_gem_object_flush_gpu_write_domain(obj);
 636         if (ret != 0)
 637                 return (ret);
 638
 639         ret = i915_gem_object_wait_rendering(obj);
 640         if (ret)
 641                 return ret;
 642
 643         i915_gem_object_flush_gtt_write_domain(obj);
 644
 645         old_write_domain = obj->base.write_domain;
 646         old_read_domains = obj->base.read_domains;
 647
 648         /* Flush the CPU cache if it's still invalid. */
 649         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
 650                 i915_gem_clflush_object(obj);
 651
 652                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
 653         }
 654
 655         /* It should now be out of any other write domains, and we can update
 656          * the domain values for our changes.
 657          */
 658         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 659
 660         /* If we're writing through the CPU, then the GPU read domains will
 661          * need to be invalidated at next use.
 662          */
 663         if (write) {
 664                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 665                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 666         }
 667
 668         return 0;
 669 }
 670
 671 /* Throttle our rendering by waiting until the ring has completed our requests
 672  * emitted over 20 msec ago.
 673  *
 674  * Note that if we were to use the current jiffies each time around the loop,
 675  * we wouldn't escape the function with any frames outstanding if the time to
 676  * render a frame was over 20ms.
 677  *
 678  * This should get us reasonable parallelism between CPU and GPU but also
 679  * relatively low latency when blocking on a particular request to finish.
 680  */
 681 static int
 682 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 683 {
 684         struct drm_i915_private *dev_priv = dev->dev_private;
 685         struct drm_i915_file_private *file_priv = file->driver_priv;
 686         unsigned long recent_enough = ticks - (20 * hz / 1000);
 687         struct drm_i915_gem_request *request;
 688         struct intel_ring_buffer *ring = NULL;
 689         u32 seqno = 0;
 690         int ret;
 691
 692         dev_priv = dev->dev_private;
 693         if (atomic_read(&dev_priv->mm.wedged))
 694                 return -EIO;
 695
 696         recent_enough = ticks - (20 * hz / 1000);
 697         ring = NULL;
 698         seqno = 0;
 699
 700         spin_lock(&file_priv->mm.lock);
 701         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
 702                 if (time_after_eq(request->emitted_jiffies, recent_enough))
 703                         break;
 704
 705                 ring = request->ring;
 706                 seqno = request->seqno;
 707         }
 708         spin_unlock(&file_priv->mm.lock);
 709
 710         if (seqno == 0)
 711                 return 0;
 712
 713         ret = 0;
 714         lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
 715         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
 716                 if (ring->irq_get(ring)) {
 717                         while (ret == 0 &&
 718                             !(i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
 719                             atomic_read(&dev_priv->mm.wedged)))
 720                                 ret = -lksleep(ring, &ring->irq_lock, PCATCH,
 721                                     "915thr", 0);
 722                         ring->irq_put(ring);
 723                         if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
 724                                 ret = -EIO;
 725                 } else if (_intel_wait_for(dev,
 726                     i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
 727                     atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
 728                         ret = -EBUSY;
 729                 }
 730         }
 731         lockmgr(&ring->irq_lock, LK_RELEASE);
 732
 733         if (ret == 0)
 734                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 735
 736         return ret;
 737 }
 738
 739 int
 740 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
 741     struct drm_file *file_priv)
 742 {
 743
 744         return (i915_gem_ring_throttle(dev, file_priv));
 745 }
 746
 747 int
 748 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 749                        struct drm_file *file_priv)
 750 {
 751         struct drm_i915_gem_madvise *args = data;
 752         struct drm_i915_gem_object *obj;
 753         int ret;
 754
 755         switch (args->madv) {
 756         case I915_MADV_DONTNEED:
 757         case I915_MADV_WILLNEED:
 758             break;
 759         default:
 760             return -EINVAL;
 761         }
 762
 763         ret = i915_mutex_lock_interruptible(dev);
 764         if (ret)
 765                 return ret;
 766
 767         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
 768         if (&obj->base == NULL) {
 769                 ret = -ENOENT;
 770                 goto unlock;
 771         }
 772
 773         if (obj->pin_count) {
 774                 ret = -EINVAL;
 775                 goto out;
 776         }
 777
 778         if (obj->madv != __I915_MADV_PURGED)
 779                 obj->madv = args->madv;
 780
 781         /* if the object is no longer attached, discard its backing storage */
 782         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
 783                 i915_gem_object_truncate(obj);
 784
 785         args->retained = obj->madv != __I915_MADV_PURGED;
 786
 787 out:
 788         drm_gem_object_unreference(&obj->base);
 789 unlock:
 790         DRM_UNLOCK(dev);
 791         return ret;
 792 }
 793
 794 void
 795 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 796 {
 797         drm_i915_private_t *dev_priv;
 798         int i;
 799
 800         dev_priv = dev->dev_private;
 801         for (i = 0; i < I915_NUM_RINGS; i++)
 802                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
 803 }
 804
 805 int
 806 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 807                        struct drm_file *file_priv)
 808 {
 809         drm_i915_private_t *dev_priv = dev->dev_private;
 810         int ret;
 811
 812         if (drm_core_check_feature(dev, DRIVER_MODESET))
 813                 return 0;
 814
 815         if (atomic_read(&dev_priv->mm.wedged)) {
 816                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
 817                 atomic_set(&dev_priv->mm.wedged, 0);
 818         }
 819
 820         DRM_LOCK(dev);
 821         dev_priv->mm.suspended = 0;
 822
 823         ret = i915_gem_init_hw(dev);
 824         if (ret != 0) {
 825                 DRM_UNLOCK(dev);
 826                 return ret;
 827         }
 828
 829         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
 830         DRM_UNLOCK(dev);
 831
 832         ret = drm_irq_install(dev);
 833         if (ret)
 834                 goto cleanup_ringbuffer;
 835
 836         return 0;
 837
 838 cleanup_ringbuffer:
 839         DRM_LOCK(dev);
 840         i915_gem_cleanup_ringbuffer(dev);
 841         dev_priv->mm.suspended = 1;
 842         DRM_UNLOCK(dev);
 843
 844         return ret;
 845 }
 846
 847 int
 848 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
 849     struct drm_file *file_priv)
 850 {
 851
 852         if (drm_core_check_feature(dev, DRIVER_MODESET))
 853                 return 0;
 854
 855         drm_irq_uninstall(dev);
 856         return (i915_gem_idle(dev));
 857 }
 858
 859 int
 860 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 861     uint32_t *handle_p)
 862 {
 863         struct drm_i915_gem_object *obj;
 864         uint32_t handle;
 865         int ret;
 866
 867         size = roundup(size, PAGE_SIZE);
 868         if (size == 0)
 869                 return (-EINVAL);
 870
 871         obj = i915_gem_alloc_object(dev, size);
 872         if (obj == NULL)
 873                 return (-ENOMEM);
 874
 875         handle = 0;
 876         ret = drm_gem_handle_create(file, &obj->base, &handle);
 877         if (ret != 0) {
 878                 drm_gem_object_release(&obj->base);
 879                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 880                 drm_free(obj, DRM_I915_GEM);
 881                 return (-ret);
 882         }
 883
 884         /* drop reference from allocate - handle holds it now */
 885         drm_gem_object_unreference(&obj->base);
 886         *handle_p = handle;
 887         return (0);
 888 }
 889
 890 int
 891 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 892     struct drm_mode_create_dumb *args)
 893 {
 894
 895         /* have to work out size/pitch and return them */
 896         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 897         args->size = args->pitch * args->height;
 898         return (i915_gem_create(file, dev, args->size, &args->handle));
 899 }
 900
 901 int
 902 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
 903     uint32_t handle)
 904 {
 905
 906         return (drm_gem_handle_delete(file, handle));
 907 }
 908
 909 int
 910 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 911     struct drm_file *file)
 912 {
 913         struct drm_i915_gem_create *args = data;
 914
 915         return (i915_gem_create(file, dev, args->size, &args->handle));
 916 }
 917
 918 static inline void vm_page_reference(vm_page_t m)
 919 {
 920         vm_page_flag_set(m, PG_REFERENCED);
 921 }
 922
 923 static int
 924 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
 925     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
 926     struct drm_file *file)
 927 {
 928         vm_object_t vm_obj;
 929         vm_page_t m;
 930         struct sf_buf *sf;
 931         vm_offset_t mkva;
 932         vm_pindex_t obj_pi;
 933         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
 934
 935         if (obj->gtt_offset != 0 && rw == UIO_READ)
 936                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 937         else
 938                 do_bit17_swizzling = 0;
 939
 940         obj->dirty = 1;
 941         vm_obj = obj->base.vm_obj;
 942         ret = 0;
 943
 944         VM_OBJECT_LOCK(vm_obj);
 945         vm_object_pip_add(vm_obj, 1);
 946         while (size > 0) {
 947                 obj_pi = OFF_TO_IDX(offset);
 948                 obj_po = offset & PAGE_MASK;
 949
 950                 m = i915_gem_wire_page(vm_obj, obj_pi);
 951                 VM_OBJECT_UNLOCK(vm_obj);
 952
 953                 sf = sf_buf_alloc(m);
 954                 mkva = sf_buf_kva(sf);
 955                 length = min(size, PAGE_SIZE - obj_po);
 956                 while (length > 0) {
 957                         if (do_bit17_swizzling &&
 958                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
 959                                 cnt = roundup2(obj_po + 1, 64);
 960                                 cnt = min(cnt - obj_po, length);
 961                                 swizzled_po = obj_po ^ 64;
 962                         } else {
 963                                 cnt = length;
 964                                 swizzled_po = obj_po;
 965                         }
 966                         if (rw == UIO_READ)
 967                                 ret = -copyout_nofault(
 968                                     (char *)mkva + swizzled_po,
 969                                     (void *)(uintptr_t)data_ptr, cnt);
 970                         else
 971                                 ret = -copyin_nofault(
 972                                     (void *)(uintptr_t)data_ptr,
 973                                     (char *)mkva + swizzled_po, cnt);
 974                         if (ret != 0)
 975                                 break;
 976                         data_ptr += cnt;
 977                         size -= cnt;
 978                         length -= cnt;
 979                         offset += cnt;
 980                         obj_po += cnt;
 981                 }
 982                 sf_buf_free(sf);
 983                 VM_OBJECT_LOCK(vm_obj);
 984                 if (rw == UIO_WRITE)
 985                         vm_page_dirty(m);
 986                 vm_page_reference(m);
 987                 vm_page_busy_wait(m, FALSE, "i915gem");
 988                 vm_page_unwire(m, 1);
 989                 vm_page_wakeup(m);
 990                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 991
 992                 if (ret != 0)
 993                         break;
 994         }
 995         vm_object_pip_wakeup(vm_obj);
 996         VM_OBJECT_UNLOCK(vm_obj);
 997
 998         return (ret);
 999 }
1000
1001 static int
1002 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
1003     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
1004 {
1005         vm_offset_t mkva;
1006         int ret;
1007
1008         /*
1009          * Pass the unaligned physical address and size to pmap_mapdev_attr()
1010          * so it can properly calculate whether an extra page needs to be
1011          * mapped or not to cover the requested range.  The function will
1012          * add the page offset into the returned mkva for us.
1013          */
1014         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
1015             offset, size, PAT_WRITE_COMBINING);
1016         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
1017         pmap_unmapdev(mkva, size);
1018         return (ret);
1019 }
1020
1021 static int
1022 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
1023     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
1024 {
1025         struct drm_i915_gem_object *obj;
1026         vm_page_t *ma;
1027         vm_offset_t start, end;
1028         int npages, ret;
1029
1030         if (size == 0)
1031                 return (0);
1032         start = trunc_page(data_ptr);
1033         end = round_page(data_ptr + size);
1034         npages = howmany(end - start, PAGE_SIZE);
1035         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
1036             M_ZERO);
1037         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
1038             (vm_offset_t)data_ptr, size,
1039             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
1040         if (npages == -1) {
1041                 ret = -EFAULT;
1042                 goto free_ma;
1043         }
1044
1045         ret = i915_mutex_lock_interruptible(dev);
1046         if (ret != 0)
1047                 goto unlocked;
1048
1049         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1050         if (&obj->base == NULL) {
1051                 ret = -ENOENT;
1052                 goto unlock;
1053         }
1054         if (offset > obj->base.size || size > obj->base.size - offset) {
1055                 ret = -EINVAL;
1056                 goto out;
1057         }
1058
1059         if (rw == UIO_READ) {
1060                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
1061                     offset, size);
1062                 if (ret != 0)
1063                         goto out;
1064                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1065                     UIO_READ, file);
1066         } else {
1067                 if (obj->phys_obj) {
1068                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
1069                             size, file);
1070                 } else if (obj->gtt_space &&
1071                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1072                         ret = i915_gem_object_pin(obj, 0, true);
1073                         if (ret != 0)
1074                                 goto out;
1075                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1076                         if (ret != 0)
1077                                 goto out_unpin;
1078                         ret = i915_gem_object_put_fence(obj);
1079                         if (ret != 0)
1080                                 goto out_unpin;
1081                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
1082                             offset, file);
1083 out_unpin:
1084                         i915_gem_object_unpin(obj);
1085                 } else {
1086                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1087                         if (ret != 0)
1088                                 goto out;
1089                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1090                             UIO_WRITE, file);
1091                 }
1092         }
1093 out:
1094         drm_gem_object_unreference(&obj->base);
1095 unlock:
1096         DRM_UNLOCK(dev);
1097 unlocked:
1098         vm_page_unhold_pages(ma, npages);
1099 free_ma:
1100         drm_free(ma, DRM_I915_GEM);
1101         return (ret);
1102 }
1103
1104 int
1105 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1106 {
1107         struct drm_i915_gem_pread *args;
1108
1109         args = data;
1110         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1111             args->offset, UIO_READ, file));
1112 }
1113
1114 int
1115 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1116 {
1117         struct drm_i915_gem_pwrite *args;
1118
1119         args = data;
1120         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1121             args->offset, UIO_WRITE, file));
1122 }
1123
1124 int
1125 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1126     struct drm_file *file)
1127 {
1128         struct drm_i915_gem_set_domain *args;
1129         struct drm_i915_gem_object *obj;
1130         uint32_t read_domains;
1131         uint32_t write_domain;
1132         int ret;
1133
1134         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1135                 return (-ENODEV);
1136
1137         args = data;
1138         read_domains = args->read_domains;
1139         write_domain = args->write_domain;
1140
1141         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
1142             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
1143             (write_domain != 0 && read_domains != write_domain))
1144                 return (-EINVAL);
1145
1146         ret = i915_mutex_lock_interruptible(dev);
1147         if (ret != 0)
1148                 return (ret);
1149
1150         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1151         if (&obj->base == NULL) {
1152                 ret = -ENOENT;
1153                 goto unlock;
1154         }
1155
1156         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
1157                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1158                 if (ret == -EINVAL)
1159                         ret = 0;
1160         } else
1161                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1162
1163         drm_gem_object_unreference(&obj->base);
1164 unlock:
1165         DRM_UNLOCK(dev);
1166         return (ret);
1167 }
1168
1169 int
1170 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1171     struct drm_file *file)
1172 {
1173         struct drm_i915_gem_sw_finish *args;
1174         struct drm_i915_gem_object *obj;
1175         int ret;
1176
1177         args = data;
1178         ret = 0;
1179         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1180                 return (ENODEV);
1181         ret = i915_mutex_lock_interruptible(dev);
1182         if (ret != 0)
1183                 return (ret);
1184         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1185         if (&obj->base == NULL) {
1186                 ret = -ENOENT;
1187                 goto unlock;
1188         }
1189         if (obj->pin_count != 0)
1190                 i915_gem_object_flush_cpu_write_domain(obj);
1191         drm_gem_object_unreference(&obj->base);
1192 unlock:
1193         DRM_UNLOCK(dev);
1194         return (ret);
1195 }
1196
1197 int
1198 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1199     struct drm_file *file)
1200 {
1201         struct drm_i915_gem_mmap *args;
1202         struct drm_gem_object *obj;
1203         struct proc *p;
1204         vm_map_t map;
1205         vm_offset_t addr;
1206         vm_size_t size;
1207         int error, rv;
1208
1209         args = data;
1210
1211         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1212                 return (-ENODEV);
1213
1214         obj = drm_gem_object_lookup(dev, file, args->handle);
1215         if (obj == NULL)
1216                 return (-ENOENT);
1217         error = 0;
1218         if (args->size == 0)
1219                 goto out;
1220         p = curproc;
1221         map = &p->p_vmspace->vm_map;
1222         size = round_page(args->size);
1223         PROC_LOCK(p);
1224         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
1225                 PROC_UNLOCK(p);
1226                 error = ENOMEM;
1227                 goto out;
1228         }
1229         PROC_UNLOCK(p);
1230
1231         addr = 0;
1232         vm_object_hold(obj->vm_obj);
1233         vm_object_reference_locked(obj->vm_obj);
1234         vm_object_drop(obj->vm_obj);
1235         DRM_UNLOCK(dev);
1236         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
1237             PAGE_SIZE, /* align */
1238             TRUE, /* fitit */
1239             VM_MAPTYPE_NORMAL, /* maptype */
1240             VM_PROT_READ | VM_PROT_WRITE, /* prot */
1241             VM_PROT_READ | VM_PROT_WRITE, /* max */
1242             MAP_SHARED /* cow */);
1243         if (rv != KERN_SUCCESS) {
1244                 vm_object_deallocate(obj->vm_obj);
1245                 error = -vm_mmap_to_errno(rv);
1246         } else {
1247                 args->addr_ptr = (uint64_t)addr;
1248         }
1249         DRM_LOCK(dev);
1250 out:
1251         drm_gem_object_unreference(obj);
1252         return (error);
1253 }
1254
1255 static int
1256 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
1257     vm_ooffset_t foff, struct ucred *cred, u_short *color)
1258 {
1259
1260         *color = 0; /* XXXKIB */
1261         return (0);
1262 }
1263
1264 int i915_intr_pf;
1265
1266 static int
1267 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
1268     vm_page_t *mres)
1269 {
1270         struct drm_gem_object *gem_obj;
1271         struct drm_i915_gem_object *obj;
1272         struct drm_device *dev;
1273         drm_i915_private_t *dev_priv;
1274         vm_page_t m, oldm;
1275         int cause, ret;
1276         bool write;
1277
1278         gem_obj = vm_obj->handle;
1279         obj = to_intel_bo(gem_obj);
1280         dev = obj->base.dev;
1281         dev_priv = dev->dev_private;
1282 #if 0
1283         write = (prot & VM_PROT_WRITE) != 0;
1284 #else
1285         write = true;
1286 #endif
1287         vm_object_pip_add(vm_obj, 1);
1288
1289         /*
1290          * Remove the placeholder page inserted by vm_fault() from the
1291          * object before dropping the object lock. If
1292          * i915_gem_release_mmap() is active in parallel on this gem
1293          * object, then it owns the drm device sx and might find the
1294          * placeholder already. Then, since the page is busy,
1295          * i915_gem_release_mmap() sleeps waiting for the busy state
1296          * of the page cleared. We will be not able to acquire drm
1297          * device lock until i915_gem_release_mmap() is able to make a
1298          * progress.
1299          */
1300         if (*mres != NULL) {
1301                 oldm = *mres;
1302                 vm_page_remove(oldm);
1303                 *mres = NULL;
1304         } else
1305                 oldm = NULL;
1306 retry:
1307         VM_OBJECT_UNLOCK(vm_obj);
1308 unlocked_vmobj:
1309         cause = ret = 0;
1310         m = NULL;
1311
1312         if (i915_intr_pf) {
1313                 ret = i915_mutex_lock_interruptible(dev);
1314                 if (ret != 0) {
1315                         cause = 10;
1316                         goto out;
1317                 }
1318         } else
1319                 DRM_LOCK(dev);
1320
1321         /*
1322          * Since the object lock was dropped, other thread might have
1323          * faulted on the same GTT address and instantiated the
1324          * mapping for the page.  Recheck.
1325          */
1326         VM_OBJECT_LOCK(vm_obj);
1327         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
1328         if (m != NULL) {
1329                 if ((m->flags & PG_BUSY) != 0) {
1330                         DRM_UNLOCK(dev);
1331 #if 0 /* XXX */
1332                         vm_page_sleep(m, "915pee");
1333 #endif
1334                         goto retry;
1335                 }
1336                 goto have_page;
1337         } else
1338                 VM_OBJECT_UNLOCK(vm_obj);
1339
1340         /* Now bind it into the GTT if needed */
1341         if (!obj->map_and_fenceable) {
1342                 ret = i915_gem_object_unbind(obj);
1343                 if (ret != 0) {
1344                         cause = 20;
1345                         goto unlock;
1346                 }
1347         }
1348         if (!obj->gtt_space) {
1349                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1350                 if (ret != 0) {
1351                         cause = 30;
1352                         goto unlock;
1353                 }
1354
1355                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1356                 if (ret != 0) {
1357                         cause = 40;
1358                         goto unlock;
1359                 }
1360         }
1361
1362         if (obj->tiling_mode == I915_TILING_NONE)
1363                 ret = i915_gem_object_put_fence(obj);
1364         else
1365                 ret = i915_gem_object_get_fence(obj, NULL);
1366         if (ret != 0) {
1367                 cause = 50;
1368                 goto unlock;
1369         }
1370
1371         if (i915_gem_object_is_inactive(obj))
1372                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1373
1374         obj->fault_mappable = true;
1375         VM_OBJECT_LOCK(vm_obj);
1376         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
1377             offset);
1378         if (m == NULL) {
1379                 cause = 60;
1380                 ret = -EFAULT;
1381                 goto unlock;
1382         }
1383         KASSERT((m->flags & PG_FICTITIOUS) != 0,
1384             ("not fictitious %p", m));
1385         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
1386
1387         if ((m->flags & PG_BUSY) != 0) {
1388                 DRM_UNLOCK(dev);
1389 #if 0 /* XXX */
1390                 vm_page_sleep(m, "915pbs");
1391 #endif
1392                 goto retry;
1393         }
1394         m->valid = VM_PAGE_BITS_ALL;
1395         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
1396 have_page:
1397         *mres = m;
1398         vm_page_busy_try(m, false);
1399
1400         DRM_UNLOCK(dev);
1401         if (oldm != NULL) {
1402                 vm_page_free(oldm);
1403         }
1404         vm_object_pip_wakeup(vm_obj);
1405         return (VM_PAGER_OK);
1406
1407 unlock:
1408         DRM_UNLOCK(dev);
1409 out:
1410         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
1411         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
1412                 goto unlocked_vmobj;
1413         }
1414         VM_OBJECT_LOCK(vm_obj);
1415         vm_object_pip_wakeup(vm_obj);
1416         return (VM_PAGER_ERROR);
1417 }
1418
1419 static void
1420 i915_gem_pager_dtor(void *handle)
1421 {
1422         struct drm_gem_object *obj;
1423         struct drm_device *dev;
1424
1425         obj = handle;
1426         dev = obj->dev;
1427
1428         DRM_LOCK(dev);
1429         drm_gem_free_mmap_offset(obj);
1430         i915_gem_release_mmap(to_intel_bo(obj));
1431         drm_gem_object_unreference(obj);
1432         DRM_UNLOCK(dev);
1433 }
1434
1435 struct cdev_pager_ops i915_gem_pager_ops = {
1436         .cdev_pg_fault  = i915_gem_pager_fault,
1437         .cdev_pg_ctor   = i915_gem_pager_ctor,
1438         .cdev_pg_dtor   = i915_gem_pager_dtor
1439 };
1440
1441 int
1442 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev,
1443     uint32_t handle, uint64_t *offset)
1444 {
1445         struct drm_i915_private *dev_priv;
1446         struct drm_i915_gem_object *obj;
1447         int ret;
1448
1449         if (!(dev->driver->driver_features & DRIVER_GEM))
1450                 return (-ENODEV);
1451
1452         dev_priv = dev->dev_private;
1453
1454         ret = i915_mutex_lock_interruptible(dev);
1455         if (ret != 0)
1456                 return (ret);
1457
1458         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1459         if (&obj->base == NULL) {
1460                 ret = -ENOENT;
1461                 goto unlock;
1462         }
1463
1464         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1465                 ret = -E2BIG;
1466                 goto out;
1467         }
1468
1469         if (obj->madv != I915_MADV_WILLNEED) {
1470                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1471                 ret = -EINVAL;
1472                 goto out;
1473         }
1474
1475         ret = drm_gem_create_mmap_offset(&obj->base);
1476         if (ret != 0)
1477                 goto out;
1478
1479         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
1480             DRM_GEM_MAPPING_KEY;
1481 out:
1482         drm_gem_object_unreference(&obj->base);
1483 unlock:
1484         DRM_UNLOCK(dev);
1485         return (ret);
1486 }
1487
1488 int
1489 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1490     struct drm_file *file)
1491 {
1492         struct drm_i915_private *dev_priv;
1493         struct drm_i915_gem_mmap_gtt *args;
1494
1495         dev_priv = dev->dev_private;
1496         args = data;
1497
1498         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
1499 }
1500
1501 struct drm_i915_gem_object *
1502 i915_gem_alloc_object(struct drm_device *dev, size_t size)
1503 {
1504         struct drm_i915_private *dev_priv;
1505         struct drm_i915_gem_object *obj;
1506
1507         dev_priv = dev->dev_private;
1508
1509         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
1510
1511         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
1512                 drm_free(obj, DRM_I915_GEM);
1513                 return (NULL);
1514         }
1515
1516         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1517         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1518
1519         if (HAS_LLC(dev))
1520                 obj->cache_level = I915_CACHE_LLC;
1521         else
1522                 obj->cache_level = I915_CACHE_NONE;
1523         obj->base.driver_private = NULL;
1524         obj->fence_reg = I915_FENCE_REG_NONE;
1525         INIT_LIST_HEAD(&obj->mm_list);
1526         INIT_LIST_HEAD(&obj->gtt_list);
1527         INIT_LIST_HEAD(&obj->ring_list);
1528         INIT_LIST_HEAD(&obj->exec_list);
1529         INIT_LIST_HEAD(&obj->gpu_write_list);
1530         obj->madv = I915_MADV_WILLNEED;
1531         /* Avoid an unnecessary call to unbind on the first bind. */
1532         obj->map_and_fenceable = true;
1533
1534         i915_gem_info_add_obj(dev_priv, size);
1535
1536         return (obj);
1537 }
1538
1539 void
1540 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1541 {
1542
1543         /* If we don't have a page list set up, then we're not pinned
1544          * to GPU, and we can ignore the cache flush because it'll happen
1545          * again at bind time.
1546          */
1547         if (obj->pages == NULL)
1548                 return;
1549
1550         /* If the GPU is snooping the contents of the CPU cache,
1551          * we do not need to manually clear the CPU cache lines.  However,
1552          * the caches are only snooped when the render cache is
1553          * flushed/invalidated.  As we always have to emit invalidations
1554          * and flushes when moving into and out of the RENDER domain, correct
1555          * snooping behaviour occurs naturally as the result of our domain
1556          * tracking.
1557          */
1558         if (obj->cache_level != I915_CACHE_NONE)
1559                 return;
1560
1561         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
1562 }
1563
1564 static void
1565 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
1566 {
1567         uint32_t old_write_domain;
1568
1569         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
1570                 return;
1571
1572         i915_gem_clflush_object(obj);
1573         intel_gtt_chipset_flush();
1574         old_write_domain = obj->base.write_domain;
1575         obj->base.write_domain = 0;
1576 }
1577
1578 /** Flushes the GTT write domain for the object if it's dirty. */
1579 static void
1580 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
1581 {
1582         uint32_t old_write_domain;
1583
1584         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
1585                 return;
1586
1587         /* No actual flushing is required for the GTT write domain.  Writes
1588          * to it immediately go to main memory as far as we know, so there's
1589          * no chipset flush.  It also doesn't land in render cache.
1590          *
1591          * However, we do have to enforce the order so that all writes through
1592          * the GTT land before any writes to the device, such as updates to
1593          * the GATT itself.
1594          */
1595         cpu_sfence();
1596
1597         old_write_domain = obj->base.write_domain;
1598         obj->base.write_domain = 0;
1599 }
1600
1601 int
1602 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
1603 {
1604         uint32_t old_write_domain, old_read_domains;
1605         int ret;
1606
1607         if (obj->gtt_space == NULL)
1608                 return (-EINVAL);
1609
1610         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
1611                 return 0;
1612
1613         ret = i915_gem_object_flush_gpu_write_domain(obj);
1614         if (ret != 0)
1615                 return (ret);
1616
1617         if (obj->pending_gpu_write || write) {
1618                 ret = i915_gem_object_wait_rendering(obj);
1619                 if (ret != 0)
1620                         return (ret);
1621         }
1622
1623         i915_gem_object_flush_cpu_write_domain(obj);
1624
1625         old_write_domain = obj->base.write_domain;
1626         old_read_domains = obj->base.read_domains;
1627
1628         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1629             ("In GTT write domain"));
1630         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1631         if (write) {
1632                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
1633                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
1634                 obj->dirty = 1;
1635         }
1636
1637         return (0);
1638 }
1639
1640 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
1641                                     enum i915_cache_level cache_level)
1642 {
1643         struct drm_device *dev = obj->base.dev;
1644         drm_i915_private_t *dev_priv = dev->dev_private;
1645         int ret;
1646
1647         if (obj->cache_level == cache_level)
1648                 return 0;
1649
1650         if (obj->pin_count) {
1651                 DRM_DEBUG("can not change the cache level of pinned objects\n");
1652                 return -EBUSY;
1653         }
1654
1655         if (obj->gtt_space) {
1656                 ret = i915_gem_object_finish_gpu(obj);
1657                 if (ret != 0)
1658                         return (ret);
1659
1660                 i915_gem_object_finish_gtt(obj);
1661
1662                 /* Before SandyBridge, you could not use tiling or fence
1663                  * registers with snooped memory, so relinquish any fences
1664                  * currently pointing to our region in the aperture.
1665                  */
1666                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
1667                         ret = i915_gem_object_put_fence(obj);
1668                         if (ret)
1669                                 return ret;
1670                 }
1671
1672                 if (obj->has_global_gtt_mapping)
1673                         i915_gem_gtt_bind_object(obj, cache_level);
1674                 if (obj->has_aliasing_ppgtt_mapping)
1675                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
1676                                                obj, cache_level);
1677         }
1678
1679         if (cache_level == I915_CACHE_NONE) {
1680                 u32 old_read_domains, old_write_domain;
1681
1682                 /* If we're coming from LLC cached, then we haven't
1683                  * actually been tracking whether the data is in the
1684                  * CPU cache or not, since we only allow one bit set
1685                  * in obj->write_domain and have been skipping the clflushes.
1686                  * Just set it to the CPU cache for now.
1687                  */
1688                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1689                     ("obj %p in CPU write domain", obj));
1690                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
1691                     ("obj %p in CPU read domain", obj));
1692
1693                 old_read_domains = obj->base.read_domains;
1694                 old_write_domain = obj->base.write_domain;
1695
1696                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1697                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1698
1699         }
1700
1701         obj->cache_level = cache_level;
1702         return (0);
1703 }
1704
1705 int
1706 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
1707     u32 alignment, struct intel_ring_buffer *pipelined)
1708 {
1709         u32 old_read_domains, old_write_domain;
1710         int ret;
1711
1712         ret = i915_gem_object_flush_gpu_write_domain(obj);
1713         if (ret != 0)
1714                 return (ret);
1715
1716         if (pipelined != obj->ring) {
1717                 ret = i915_gem_object_wait_rendering(obj);
1718                 if (ret == -ERESTART || ret == -EINTR)
1719                         return (ret);
1720         }
1721
1722         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
1723         if (ret != 0)
1724                 return (ret);
1725
1726         ret = i915_gem_object_pin(obj, alignment, true);
1727         if (ret != 0)
1728                 return (ret);
1729
1730         i915_gem_object_flush_cpu_write_domain(obj);
1731
1732         old_write_domain = obj->base.write_domain;
1733         old_read_domains = obj->base.read_domains;
1734
1735         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1736             ("obj %p in GTT write domain", obj));
1737         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1738
1739         return (0);
1740 }
1741
1742 int
1743 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
1744 {
1745         int ret;
1746
1747         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
1748                 return (0);
1749
1750         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1751                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
1752                 if (ret != 0)
1753                         return (ret);
1754         }
1755
1756         ret = i915_gem_object_wait_rendering(obj);
1757         if (ret != 0)
1758                 return (ret);
1759
1760         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1761
1762         return (0);
1763 }
1764
1765 static int
1766 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
1767     uint64_t offset, uint64_t size)
1768 {
1769         uint32_t old_read_domains;
1770         int i, ret;
1771
1772         if (offset == 0 && size == obj->base.size)
1773                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
1774
1775         ret = i915_gem_object_flush_gpu_write_domain(obj);
1776         if (ret != 0)
1777                 return (ret);
1778         ret = i915_gem_object_wait_rendering(obj);
1779         if (ret != 0)
1780                 return (ret);
1781
1782         i915_gem_object_flush_gtt_write_domain(obj);
1783
1784         if (obj->page_cpu_valid == NULL &&
1785             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
1786                 return (0);
1787
1788         if (obj->page_cpu_valid == NULL) {
1789                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
1790                     DRM_I915_GEM, M_WAITOK | M_ZERO);
1791         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1792                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
1793
1794         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1795              i++) {
1796                 if (obj->page_cpu_valid[i])
1797                         continue;
1798                 drm_clflush_pages(obj->pages + i, 1);
1799                 obj->page_cpu_valid[i] = 1;
1800         }
1801
1802         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1803             ("In gpu write domain"));
1804
1805         old_read_domains = obj->base.read_domains;
1806         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
1807
1808         return (0);
1809 }
1810
1811 static uint32_t
1812 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1813 {
1814         uint32_t gtt_size;
1815
1816         if (INTEL_INFO(dev)->gen >= 4 ||
1817             tiling_mode == I915_TILING_NONE)
1818                 return (size);
1819
1820         /* Previous chips need a power-of-two fence region when tiling */
1821         if (INTEL_INFO(dev)->gen == 3)
1822                 gtt_size = 1024*1024;
1823         else
1824                 gtt_size = 512*1024;
1825
1826         while (gtt_size < size)
1827                 gtt_size <<= 1;
1828
1829         return (gtt_size);
1830 }
1831
1832 /**
1833  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1834  * @obj: object to check
1835  *
1836  * Return the required GTT alignment for an object, taking into account
1837  * potential fence register mapping.
1838  */
1839 static uint32_t
1840 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1841      int tiling_mode)
1842 {
1843
1844         /*
1845          * Minimum alignment is 4k (GTT page size), but might be greater
1846          * if a fence register is needed for the object.
1847          */
1848         if (INTEL_INFO(dev)->gen >= 4 ||
1849             tiling_mode == I915_TILING_NONE)
1850                 return (4096);
1851
1852         /*
1853          * Previous chips need to be aligned to the size of the smallest
1854          * fence register that can contain the object.
1855          */
1856         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
1857 }
1858
1859 uint32_t
1860 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size,
1861     int tiling_mode)
1862 {
1863
1864         if (tiling_mode == I915_TILING_NONE)
1865                 return (4096);
1866
1867         /*
1868          * Minimum alignment is 4k (GTT page size) for sane hw.
1869          */
1870         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
1871                 return (4096);
1872
1873         /*
1874          * Previous hardware however needs to be aligned to a power-of-two
1875          * tile height. The simplest method for determining this is to reuse
1876          * the power-of-tile object size.
1877          */
1878         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
1879 }
1880
1881 static int
1882 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1883     unsigned alignment, bool map_and_fenceable)
1884 {
1885         struct drm_device *dev;
1886         struct drm_i915_private *dev_priv;
1887         struct drm_mm_node *free_space;
1888         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1889         bool mappable, fenceable;
1890         int ret;
1891
1892         dev = obj->base.dev;
1893         dev_priv = dev->dev_private;
1894
1895         if (obj->madv != I915_MADV_WILLNEED) {
1896                 DRM_ERROR("Attempting to bind a purgeable object\n");
1897                 return (-EINVAL);
1898         }
1899
1900         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1901             obj->tiling_mode);
1902         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1903             obj->tiling_mode);
1904         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1905             obj->base.size, obj->tiling_mode);
1906         if (alignment == 0)
1907                 alignment = map_and_fenceable ? fence_alignment :
1908                     unfenced_alignment;
1909         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1910                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1911                 return (-EINVAL);
1912         }
1913
1914         size = map_and_fenceable ? fence_size : obj->base.size;
1915
1916         /* If the object is bigger than the entire aperture, reject it early
1917          * before evicting everything in a vain attempt to find space.
1918          */
1919         if (obj->base.size > (map_and_fenceable ?
1920             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1921                 DRM_ERROR(
1922 "Attempting to bind an object larger than the aperture\n");
1923                 return (-E2BIG);
1924         }
1925
1926  search_free:
1927         if (map_and_fenceable)
1928                 free_space = drm_mm_search_free_in_range(
1929                     &dev_priv->mm.gtt_space, size, alignment, 0,
1930                     dev_priv->mm.gtt_mappable_end, 0);
1931         else
1932                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1933                     size, alignment, 0);
1934         if (free_space != NULL) {
1935                 int color = 0;
1936                 if (map_and_fenceable)
1937                         obj->gtt_space = drm_mm_get_block_range_generic(
1938                             free_space, size, alignment, color, 0,
1939                             dev_priv->mm.gtt_mappable_end, 1);
1940                 else
1941                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1942                             size, alignment, color, 1);
1943         }
1944         if (obj->gtt_space == NULL) {
1945                 ret = i915_gem_evict_something(dev, size, alignment,
1946                     map_and_fenceable);
1947                 if (ret != 0)
1948                         return (ret);
1949                 goto search_free;
1950         }
1951         ret = i915_gem_object_get_pages_gtt(obj, 0);
1952         if (ret != 0) {
1953                 drm_mm_put_block(obj->gtt_space);
1954                 obj->gtt_space = NULL;
1955                 /*
1956                  * i915_gem_object_get_pages_gtt() cannot return
1957                  * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY)
1958                  * (which does not support operation without a flag
1959                  * anyway).
1960                  */
1961                 return (ret);
1962         }
1963
1964         i915_gem_gtt_bind_object(obj, obj->cache_level);
1965         if (ret != 0) {
1966                 i915_gem_object_put_pages_gtt(obj);
1967                 drm_mm_put_block(obj->gtt_space);
1968                 obj->gtt_space = NULL;
1969                 if (i915_gem_evict_everything(dev, false))
1970                         return (ret);
1971                 goto search_free;
1972         }
1973
1974         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1975         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1976
1977         obj->gtt_offset = obj->gtt_space->start;
1978
1979         fenceable =
1980                 obj->gtt_space->size == fence_size &&
1981                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1982
1983         mappable =
1984                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1985         obj->map_and_fenceable = mappable && fenceable;
1986
1987         return (0);
1988 }
1989
1990 static void
1991 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1992 {
1993         u32 old_write_domain, old_read_domains;
1994
1995         /* Act a barrier for all accesses through the GTT */
1996         cpu_mfence();
1997
1998         /* Force a pagefault for domain tracking on next user access */
1999         i915_gem_release_mmap(obj);
2000
2001         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2002                 return;
2003
2004         old_read_domains = obj->base.read_domains;
2005         old_write_domain = obj->base.write_domain;
2006
2007         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2008         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2009
2010 }
2011
2012 int
2013 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2014 {
2015         drm_i915_private_t *dev_priv;
2016         int ret;
2017
2018         dev_priv = obj->base.dev->dev_private;
2019         ret = 0;
2020         if (obj->gtt_space == NULL)
2021                 return (0);
2022         if (obj->pin_count != 0) {
2023                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2024                 return (-EINVAL);
2025         }
2026
2027         ret = i915_gem_object_finish_gpu(obj);
2028         if (ret == -ERESTART || ret == -EINTR)
2029                 return (ret);
2030
2031         i915_gem_object_finish_gtt(obj);
2032
2033         if (ret == 0)
2034                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2035         if (ret == -ERESTART || ret == -EINTR)
2036                 return (ret);
2037         if (ret != 0) {
2038                 i915_gem_clflush_object(obj);
2039                 obj->base.read_domains = obj->base.write_domain =
2040                     I915_GEM_DOMAIN_CPU;
2041         }
2042
2043         ret = i915_gem_object_put_fence(obj);
2044         if (ret == -ERESTART)
2045                 return (ret);
2046
2047         i915_gem_gtt_unbind_object(obj);
2048         if (obj->has_aliasing_ppgtt_mapping) {
2049                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2050                 obj->has_aliasing_ppgtt_mapping = 0;
2051         }
2052         i915_gem_object_put_pages_gtt(obj);
2053
2054         list_del_init(&obj->gtt_list);
2055         list_del_init(&obj->mm_list);
2056         obj->map_and_fenceable = true;
2057
2058         drm_mm_put_block(obj->gtt_space);
2059         obj->gtt_space = NULL;
2060         obj->gtt_offset = 0;
2061
2062         if (i915_gem_object_is_purgeable(obj))
2063                 i915_gem_object_truncate(obj);
2064
2065         return (ret);
2066 }
2067
2068 int i915_gpu_idle(struct drm_device *dev)
2069 {
2070         drm_i915_private_t *dev_priv = dev->dev_private;
2071         struct intel_ring_buffer *ring;
2072         int ret, i;
2073
2074         /* Flush everything onto the inactive list. */
2075         for_each_ring(ring, dev_priv, i) {
2076                 ret = intel_ring_idle(ring);
2077                 if (ret)
2078                         return ret;
2079         }
2080
2081         return 0;
2082 }
2083
2084 static int
2085 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
2086     int flags)
2087 {
2088         struct drm_device *dev;
2089         vm_object_t vm_obj;
2090         vm_page_t m;
2091         int page_count, i, j;
2092
2093         dev = obj->base.dev;
2094         KASSERT(obj->pages == NULL, ("Obj already has pages"));
2095         page_count = obj->base.size / PAGE_SIZE;
2096         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
2097             M_WAITOK);
2098         vm_obj = obj->base.vm_obj;
2099         VM_OBJECT_LOCK(vm_obj);
2100         for (i = 0; i < page_count; i++) {
2101                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
2102                         goto failed;
2103         }
2104         VM_OBJECT_UNLOCK(vm_obj);
2105         if (i915_gem_object_needs_bit17_swizzle(obj))
2106                 i915_gem_object_do_bit_17_swizzle(obj);
2107         return (0);
2108
2109 failed:
2110         for (j = 0; j < i; j++) {
2111                 m = obj->pages[j];
2112                 vm_page_busy_wait(m, FALSE, "i915gem");
2113                 vm_page_unwire(m, 0);
2114                 vm_page_wakeup(m);
2115                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2116         }
2117         VM_OBJECT_UNLOCK(vm_obj);
2118         drm_free(obj->pages, DRM_I915_GEM);
2119         obj->pages = NULL;
2120         return (-EIO);
2121 }
2122
2123 #define GEM_PARANOID_CHECK_GTT 0
2124 #if GEM_PARANOID_CHECK_GTT
2125 static void
2126 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
2127     int page_count)
2128 {
2129         struct drm_i915_private *dev_priv;
2130         vm_paddr_t pa;
2131         unsigned long start, end;
2132         u_int i;
2133         int j;
2134
2135         dev_priv = dev->dev_private;
2136         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
2137         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
2138         for (i = start; i < end; i++) {
2139                 pa = intel_gtt_read_pte_paddr(i);
2140                 for (j = 0; j < page_count; j++) {
2141                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
2142                                 panic("Page %p in GTT pte index %d pte %x",
2143                                     ma[i], i, intel_gtt_read_pte(i));
2144                         }
2145                 }
2146         }
2147 }
2148 #endif
2149
2150 static void
2151 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2152 {
2153         vm_page_t m;
2154         int page_count, i;
2155
2156         BUG_ON(obj->madv == __I915_MADV_PURGED);
2157
2158         if (obj->tiling_mode != I915_TILING_NONE)
2159                 i915_gem_object_save_bit_17_swizzle(obj);
2160         if (obj->madv == I915_MADV_DONTNEED)
2161                 obj->dirty = 0;
2162         page_count = obj->base.size / PAGE_SIZE;
2163         VM_OBJECT_LOCK(obj->base.vm_obj);
2164 #if GEM_PARANOID_CHECK_GTT
2165         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
2166 #endif
2167         for (i = 0; i < page_count; i++) {
2168                 m = obj->pages[i];
2169                 if (obj->dirty)
2170                         vm_page_dirty(m);
2171                 if (obj->madv == I915_MADV_WILLNEED)
2172                         vm_page_reference(m);
2173                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
2174                 vm_page_unwire(obj->pages[i], 1);
2175                 vm_page_wakeup(obj->pages[i]);
2176                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2177         }
2178         VM_OBJECT_UNLOCK(obj->base.vm_obj);
2179         obj->dirty = 0;
2180         drm_free(obj->pages, DRM_I915_GEM);
2181         obj->pages = NULL;
2182 }
2183
2184 void
2185 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2186 {
2187         vm_object_t devobj;
2188         vm_page_t m;
2189         int i, page_count;
2190
2191         if (!obj->fault_mappable)
2192                 return;
2193
2194         devobj = cdev_pager_lookup(obj);
2195         if (devobj != NULL) {
2196                 page_count = OFF_TO_IDX(obj->base.size);
2197
2198                 VM_OBJECT_LOCK(devobj);
2199                 for (i = 0; i < page_count; i++) {
2200                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
2201                         if (m == NULL)
2202                                 continue;
2203                         cdev_pager_free_page(devobj, m);
2204                 }
2205                 VM_OBJECT_UNLOCK(devobj);
2206                 vm_object_deallocate(devobj);
2207         }
2208
2209         obj->fault_mappable = false;
2210 }
2211
2212 int
2213 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
2214 {
2215         u32 seqno;
2216         int ret;
2217
2218
2219         seqno = obj->last_rendering_seqno;
2220         if (seqno == 0)
2221                 return 0;
2222
2223         if (obj->active) {
2224                 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
2225                 if (ret != 0)
2226                         return (ret);
2227         }
2228
2229         /* Manually manage the write flush as we may have not yet
2230          * retired the buffer.
2231          */
2232         if (obj->last_rendering_seqno &&
2233             i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
2234                 obj->last_rendering_seqno = 0;
2235                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
2236         }
2237
2238         return 0;
2239 }
2240
2241 void
2242 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2243     struct intel_ring_buffer *ring, uint32_t seqno)
2244 {
2245         struct drm_device *dev = obj->base.dev;
2246         struct drm_i915_private *dev_priv = dev->dev_private;
2247         struct drm_i915_fence_reg *reg;
2248
2249         obj->ring = ring;
2250         KASSERT(ring != NULL, ("NULL ring"));
2251
2252         /* Add a reference if we're newly entering the active list. */
2253         if (!obj->active) {
2254                 drm_gem_object_reference(&obj->base);
2255                 obj->active = 1;
2256         }
2257
2258         /* Move from whatever list we were on to the tail of execution. */
2259         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
2260         list_move_tail(&obj->ring_list, &ring->active_list);
2261
2262         obj->last_rendering_seqno = seqno;
2263         if (obj->fenced_gpu_access) {
2264                 obj->last_fenced_seqno = seqno;
2265                 obj->last_fenced_ring = ring;
2266
2267                 /* Bump MRU to take account of the delayed flush */
2268                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2269                         reg = &dev_priv->fence_regs[obj->fence_reg];
2270                         list_move_tail(&reg->lru_list,
2271                                        &dev_priv->mm.fence_list);
2272                 }
2273         }
2274 }
2275
2276 static void
2277 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
2278 {
2279         list_del_init(&obj->ring_list);
2280         obj->last_rendering_seqno = 0;
2281         obj->last_fenced_seqno = 0;
2282 }
2283
2284 static void
2285 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
2286 {
2287         struct drm_device *dev = obj->base.dev;
2288         drm_i915_private_t *dev_priv = dev->dev_private;
2289
2290         KASSERT(obj->active, ("Object not active"));
2291         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
2292
2293         i915_gem_object_move_off_active(obj);
2294 }
2295
2296 static void
2297 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2298 {
2299         struct drm_device *dev = obj->base.dev;
2300         struct drm_i915_private *dev_priv = dev->dev_private;
2301
2302         if (obj->pin_count != 0)
2303                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2304         else
2305                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2306
2307         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
2308         KASSERT(obj->active, ("Object not active"));
2309         obj->ring = NULL;
2310         obj->last_fenced_ring = NULL;
2311
2312         i915_gem_object_move_off_active(obj);
2313         obj->fenced_gpu_access = false;
2314
2315         obj->active = 0;
2316         obj->pending_gpu_write = false;
2317         drm_gem_object_unreference(&obj->base);
2318
2319 #if 1
2320         KIB_NOTYET();
2321 #else
2322         WARN_ON(i915_verify_lists(dev));
2323 #endif
2324 }
2325
2326 /* Immediately discard the backing storage */
2327 static void
2328 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2329 {
2330         vm_object_t vm_obj;
2331
2332         vm_obj = obj->base.vm_obj;
2333         VM_OBJECT_LOCK(vm_obj);
2334         vm_object_page_remove(vm_obj, 0, 0, false);
2335         VM_OBJECT_UNLOCK(vm_obj);
2336         obj->madv = __I915_MADV_PURGED;
2337 }
2338
2339 static inline int
2340 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
2341 {
2342
2343         return (obj->madv == I915_MADV_DONTNEED);
2344 }
2345
2346 static void
2347 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
2348     uint32_t flush_domains)
2349 {
2350         struct drm_i915_gem_object *obj, *next;
2351         uint32_t old_write_domain;
2352
2353         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
2354             gpu_write_list) {
2355                 if (obj->base.write_domain & flush_domains) {
2356                         old_write_domain = obj->base.write_domain;
2357                         obj->base.write_domain = 0;
2358                         list_del_init(&obj->gpu_write_list);
2359                         i915_gem_object_move_to_active(obj, ring,
2360                             i915_gem_next_request_seqno(ring));
2361                 }
2362         }
2363 }
2364
2365 static int
2366 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
2367 {
2368         drm_i915_private_t *dev_priv;
2369
2370         dev_priv = obj->base.dev->dev_private;
2371         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
2372             obj->tiling_mode != I915_TILING_NONE);
2373 }
2374
2375 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
2376
2377 static vm_page_t
2378 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
2379 {
2380         vm_page_t m;
2381         int rv;
2382
2383         VM_OBJECT_LOCK_ASSERT_OWNED(object);
2384         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
2385         if (m->valid != VM_PAGE_BITS_ALL) {
2386                 if (vm_pager_has_page(object, pindex)) {
2387                         rv = vm_pager_get_page(object, &m, 1);
2388                         m = vm_page_lookup(object, pindex);
2389                         if (m == NULL)
2390                                 return (NULL);
2391                         if (rv != VM_PAGER_OK) {
2392                                 vm_page_free(m);
2393                                 return (NULL);
2394                         }
2395                 } else {
2396                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
2397                         m->valid = VM_PAGE_BITS_ALL;
2398                         m->dirty = 0;
2399                 }
2400         }
2401         vm_page_wire(m);
2402         vm_page_wakeup(m);
2403         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
2404         return (m);
2405 }
2406
2407 int
2408 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
2409     uint32_t flush_domains)
2410 {
2411         int ret;
2412
2413         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2414                 return 0;
2415
2416         ret = ring->flush(ring, invalidate_domains, flush_domains);
2417         if (ret)
2418                 return ret;
2419
2420         if (flush_domains & I915_GEM_GPU_DOMAINS)
2421                 i915_gem_process_flushing_list(ring, flush_domains);
2422         return 0;
2423 }
2424
2425 /**
2426  * Waits for a sequence number to be signaled, and cleans up the
2427  * request and object lists appropriately for that event.
2428  */
2429 int
2430 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
2431 {
2432         drm_i915_private_t *dev_priv;
2433         struct drm_i915_gem_request *request;
2434         uint32_t ier;
2435         int flags, ret;
2436         bool recovery_complete;
2437         bool do_retire = true;
2438
2439         KASSERT(seqno != 0, ("Zero seqno"));
2440
2441         dev_priv = ring->dev->dev_private;
2442         ret = 0;
2443
2444         if (atomic_read(&dev_priv->mm.wedged) != 0) {
2445                 /* Give the error handler a chance to run. */
2446                 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
2447                 recovery_complete = (&dev_priv->error_completion) > 0;
2448                 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
2449                 return (recovery_complete ? -EIO : -EAGAIN);
2450         }
2451
2452         if (seqno == ring->outstanding_lazy_request) {
2453                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
2454                     M_WAITOK | M_ZERO);
2455                 if (request == NULL)
2456                         return (-ENOMEM);
2457
2458                 ret = i915_add_request(ring, NULL, request);
2459                 if (ret != 0) {
2460                         drm_free(request, DRM_I915_GEM);
2461                         return (ret);
2462                 }
2463
2464                 seqno = request->seqno;
2465         }
2466
2467         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
2468                 if (HAS_PCH_SPLIT(ring->dev))
2469                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2470                 else
2471                         ier = I915_READ(IER);
2472                 if (!ier) {
2473                         DRM_ERROR("something (likely vbetool) disabled "
2474                                   "interrupts, re-enabling\n");
2475                         ring->dev->driver->irq_preinstall(ring->dev);
2476                         ring->dev->driver->irq_postinstall(ring->dev);
2477                 }
2478
2479                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2480                 if (ring->irq_get(ring)) {
2481                         flags = dev_priv->mm.interruptible ? PCATCH : 0;
2482                         while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)
2483                             && !atomic_read(&dev_priv->mm.wedged) &&
2484                             ret == 0) {
2485                                 ret = -lksleep(ring, &ring->irq_lock, flags,
2486                                     "915gwr", 0);
2487                         }
2488                         ring->irq_put(ring);
2489                         lockmgr(&ring->irq_lock, LK_RELEASE);
2490                 } else {
2491                         lockmgr(&ring->irq_lock, LK_RELEASE);
2492                         if (_intel_wait_for(ring->dev,
2493                             i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2494                             atomic_read(&dev_priv->mm.wedged), 3000,
2495                             0, "i915wrq") != 0)
2496                                 ret = -EBUSY;
2497                 }
2498
2499         }
2500         if (atomic_read(&dev_priv->mm.wedged))
2501                 ret = -EAGAIN;
2502
2503         /* Directly dispatch request retiring.  While we have the work queue
2504          * to handle this, the waiter on a request often wants an associated
2505          * buffer to have made it to the inactive list, and we would need
2506          * a separate wait queue to handle that.
2507          */
2508         if (ret == 0 && do_retire)
2509                 i915_gem_retire_requests_ring(ring);
2510
2511         return (ret);
2512 }
2513
2514 static u32
2515 i915_gem_get_seqno(struct drm_device *dev)
2516 {
2517         drm_i915_private_t *dev_priv = dev->dev_private;
2518         u32 seqno = dev_priv->next_seqno;
2519
2520         /* reserve 0 for non-seqno */
2521         if (++dev_priv->next_seqno == 0)
2522                 dev_priv->next_seqno = 1;
2523
2524         return seqno;
2525 }
2526
2527 u32
2528 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
2529 {
2530         if (ring->outstanding_lazy_request == 0)
2531                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
2532
2533         return ring->outstanding_lazy_request;
2534 }
2535
2536 int
2537 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
2538      struct drm_i915_gem_request *request)
2539 {
2540         drm_i915_private_t *dev_priv;
2541         struct drm_i915_file_private *file_priv;
2542         uint32_t seqno;
2543         u32 request_ring_position;
2544         int was_empty;
2545         int ret;
2546
2547         KASSERT(request != NULL, ("NULL request in add"));
2548         DRM_LOCK_ASSERT(ring->dev);
2549         dev_priv = ring->dev->dev_private;
2550
2551         seqno = i915_gem_next_request_seqno(ring);
2552         request_ring_position = intel_ring_get_tail(ring);
2553
2554         ret = ring->add_request(ring, &seqno);
2555         if (ret != 0)
2556             return ret;
2557
2558         request->seqno = seqno;
2559         request->ring = ring;
2560         request->tail = request_ring_position;
2561         request->emitted_jiffies = ticks;
2562         was_empty = list_empty(&ring->request_list);
2563         list_add_tail(&request->list, &ring->request_list);
2564
2565         if (file != NULL) {
2566                 file_priv = file->driver_priv;
2567
2568                 spin_lock(&file_priv->mm.lock);
2569                 request->file_priv = file_priv;
2570                 list_add_tail(&request->client_list,
2571                     &file_priv->mm.request_list);
2572                 spin_unlock(&file_priv->mm.lock);
2573         }
2574
2575         ring->outstanding_lazy_request = 0;
2576
2577         if (!dev_priv->mm.suspended) {
2578                 if (i915_enable_hangcheck) {
2579                         mod_timer(&dev_priv->hangcheck_timer,
2580                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2581                 }
2582                 if (was_empty) {
2583                         queue_delayed_work(dev_priv->wq,
2584                                            &dev_priv->mm.retire_work,
2585                                            round_jiffies_up_relative(hz));
2586                         intel_mark_busy(dev_priv->dev);
2587                 }
2588         }
2589         return (0);
2590 }
2591
2592 static inline void
2593 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2594 {
2595         struct drm_i915_file_private *file_priv = request->file_priv;
2596
2597         if (!file_priv)
2598                 return;
2599
2600         DRM_LOCK_ASSERT(request->ring->dev);
2601
2602         spin_lock(&file_priv->mm.lock);
2603         if (request->file_priv != NULL) {
2604                 list_del(&request->client_list);
2605                 request->file_priv = NULL;
2606         }
2607         spin_unlock(&file_priv->mm.lock);
2608 }
2609
2610 void
2611 i915_gem_release(struct drm_device *dev, struct drm_file *file)
2612 {
2613         struct drm_i915_file_private *file_priv;
2614         struct drm_i915_gem_request *request;
2615
2616         file_priv = file->driver_priv;
2617
2618         /* Clean up our request list when the client is going away, so that
2619          * later retire_requests won't dereference our soon-to-be-gone
2620          * file_priv.
2621          */
2622         spin_lock(&file_priv->mm.lock);
2623         while (!list_empty(&file_priv->mm.request_list)) {
2624                 request = list_first_entry(&file_priv->mm.request_list,
2625                                            struct drm_i915_gem_request,
2626                                            client_list);
2627                 list_del(&request->client_list);
2628                 request->file_priv = NULL;
2629         }
2630         spin_unlock(&file_priv->mm.lock);
2631 }
2632
2633 static void
2634 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2635     struct intel_ring_buffer *ring)
2636 {
2637
2638         if (ring->dev != NULL)
2639                 DRM_LOCK_ASSERT(ring->dev);
2640
2641         while (!list_empty(&ring->request_list)) {
2642                 struct drm_i915_gem_request *request;
2643
2644                 request = list_first_entry(&ring->request_list,
2645                     struct drm_i915_gem_request, list);
2646
2647                 list_del(&request->list);
2648                 i915_gem_request_remove_from_client(request);
2649                 drm_free(request, DRM_I915_GEM);
2650         }
2651
2652         while (!list_empty(&ring->active_list)) {
2653                 struct drm_i915_gem_object *obj;
2654
2655                 obj = list_first_entry(&ring->active_list,
2656                     struct drm_i915_gem_object, ring_list);
2657
2658                 obj->base.write_domain = 0;
2659                 list_del_init(&obj->gpu_write_list);
2660                 i915_gem_object_move_to_inactive(obj);
2661         }
2662 }
2663
2664 static void
2665 i915_gem_reset_fences(struct drm_device *dev)
2666 {
2667         struct drm_i915_private *dev_priv = dev->dev_private;
2668         int i;
2669
2670         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2671                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2672                 struct drm_i915_gem_object *obj = reg->obj;
2673
2674                 if (!obj)
2675                         continue;
2676
2677                 if (obj->tiling_mode)
2678                         i915_gem_release_mmap(obj);
2679
2680                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
2681                 reg->obj->fenced_gpu_access = false;
2682                 reg->obj->last_fenced_seqno = 0;
2683                 reg->obj->last_fenced_ring = NULL;
2684                 i915_gem_clear_fence_reg(dev, reg);
2685         }
2686 }
2687
2688 void
2689 i915_gem_reset(struct drm_device *dev)
2690 {
2691         struct drm_i915_private *dev_priv = dev->dev_private;
2692         struct drm_i915_gem_object *obj;
2693         int i;
2694
2695         for (i = 0; i < I915_NUM_RINGS; i++)
2696                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
2697
2698         /* Remove anything from the flushing lists. The GPU cache is likely
2699          * to be lost on reset along with the data, so simply move the
2700          * lost bo to the inactive list.
2701          */
2702         while (!list_empty(&dev_priv->mm.flushing_list)) {
2703                 obj = list_first_entry(&dev_priv->mm.flushing_list,
2704                                       struct drm_i915_gem_object,
2705                                       mm_list);
2706
2707                 obj->base.write_domain = 0;
2708                 list_del_init(&obj->gpu_write_list);
2709                 i915_gem_object_move_to_inactive(obj);
2710         }
2711
2712         /* Move everything out of the GPU domains to ensure we do any
2713          * necessary invalidation upon reuse.
2714          */
2715         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
2716                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2717         }
2718
2719         /* The fence registers are invalidated so clear them out */
2720         i915_gem_reset_fences(dev);
2721 }
2722
2723 /**
2724  * This function clears the request list as sequence numbers are passed.
2725  */
2726 void
2727 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2728 {
2729         uint32_t seqno;
2730
2731         if (list_empty(&ring->request_list))
2732                 return;
2733
2734         seqno = ring->get_seqno(ring, true);
2735
2736         while (!list_empty(&ring->request_list)) {
2737                 struct drm_i915_gem_request *request;
2738
2739                 request = list_first_entry(&ring->request_list,
2740                                            struct drm_i915_gem_request,
2741                                            list);
2742
2743                 if (!i915_seqno_passed(seqno, request->seqno))
2744                         break;
2745
2746                 /* We know the GPU must have read the request to have
2747                  * sent us the seqno + interrupt, so use the position
2748                  * of tail of the request to update the last known position
2749                  * of the GPU head.
2750                  */
2751                 ring->last_retired_head = request->tail;
2752
2753                 list_del(&request->list);
2754                 i915_gem_request_remove_from_client(request);
2755                 drm_free(request, DRM_I915_GEM);
2756         }
2757
2758         /* Move any buffers on the active list that are no longer referenced
2759          * by the ringbuffer to the flushing/inactive lists as appropriate.
2760          */
2761         while (!list_empty(&ring->active_list)) {
2762                 struct drm_i915_gem_object *obj;
2763
2764                 obj = list_first_entry(&ring->active_list,
2765                                       struct drm_i915_gem_object,
2766                                       ring_list);
2767
2768                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
2769                         break;
2770
2771                 if (obj->base.write_domain != 0)
2772                         i915_gem_object_move_to_flushing(obj);
2773                 else
2774                         i915_gem_object_move_to_inactive(obj);
2775         }
2776
2777         if (unlikely(ring->trace_irq_seqno &&
2778                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2779                 ring->irq_put(ring);
2780                 ring->trace_irq_seqno = 0;
2781         }
2782
2783 }
2784
2785 void
2786 i915_gem_retire_requests(struct drm_device *dev)
2787 {
2788         drm_i915_private_t *dev_priv = dev->dev_private;
2789         struct drm_i915_gem_object *obj, *next;
2790         int i;
2791
2792         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2793                 list_for_each_entry_safe(obj, next,
2794                     &dev_priv->mm.deferred_free_list, mm_list)
2795                         i915_gem_free_object_tail(obj);
2796         }
2797
2798         for (i = 0; i < I915_NUM_RINGS; i++)
2799                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
2800 }
2801
2802 static int
2803 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2804     struct intel_ring_buffer *pipelined)
2805 {
2806         struct drm_device *dev = obj->base.dev;
2807         drm_i915_private_t *dev_priv = dev->dev_private;
2808         u32 size = obj->gtt_space->size;
2809         int regnum = obj->fence_reg;
2810         uint64_t val;
2811
2812         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2813                          0xfffff000) << 32;
2814         val |= obj->gtt_offset & 0xfffff000;
2815         val |= (uint64_t)((obj->stride / 128) - 1) <<
2816                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2817
2818         if (obj->tiling_mode == I915_TILING_Y)
2819                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2820         val |= I965_FENCE_REG_VALID;
2821
2822         if (pipelined) {
2823                 int ret = intel_ring_begin(pipelined, 6);
2824                 if (ret)
2825                         return ret;
2826
2827                 intel_ring_emit(pipelined, MI_NOOP);
2828                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2829                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2830                 intel_ring_emit(pipelined, (u32)val);
2831                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2832                 intel_ring_emit(pipelined, (u32)(val >> 32));
2833                 intel_ring_advance(pipelined);
2834         } else
2835                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2836
2837         return 0;
2838 }
2839
2840 static int
2841 i965_write_fence_reg(struct drm_i915_gem_object *obj,
2842     struct intel_ring_buffer *pipelined)
2843 {
2844         struct drm_device *dev = obj->base.dev;
2845         drm_i915_private_t *dev_priv = dev->dev_private;
2846         u32 size = obj->gtt_space->size;
2847         int regnum = obj->fence_reg;
2848         uint64_t val;
2849
2850         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2851                     0xfffff000) << 32;
2852         val |= obj->gtt_offset & 0xfffff000;
2853         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2854         if (obj->tiling_mode == I915_TILING_Y)
2855                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2856         val |= I965_FENCE_REG_VALID;
2857
2858         if (pipelined) {
2859                 int ret = intel_ring_begin(pipelined, 6);
2860                 if (ret)
2861                         return ret;
2862
2863                 intel_ring_emit(pipelined, MI_NOOP);
2864                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2865                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2866                 intel_ring_emit(pipelined, (u32)val);
2867                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2868                 intel_ring_emit(pipelined, (u32)(val >> 32));
2869                 intel_ring_advance(pipelined);
2870         } else
2871                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2872
2873         return 0;
2874 }
2875
2876 static int
2877 i915_write_fence_reg(struct drm_i915_gem_object *obj,
2878     struct intel_ring_buffer *pipelined)
2879 {
2880         struct drm_device *dev = obj->base.dev;
2881         drm_i915_private_t *dev_priv = dev->dev_private;
2882         u32 size = obj->gtt_space->size;
2883         u32 fence_reg, val, pitch_val;
2884         int tile_width;
2885
2886         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2887             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
2888                 kprintf(
2889 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2890                  obj->gtt_offset, obj->map_and_fenceable, size);
2891                 return -EINVAL;
2892         }
2893
2894         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2895                 tile_width = 128;
2896         else
2897                 tile_width = 512;
2898
2899         /* Note: pitch better be a power of two tile widths */
2900         pitch_val = obj->stride / tile_width;
2901         pitch_val = ffs(pitch_val) - 1;
2902
2903         val = obj->gtt_offset;
2904         if (obj->tiling_mode == I915_TILING_Y)
2905                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2906         val |= I915_FENCE_SIZE_BITS(size);
2907         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2908         val |= I830_FENCE_REG_VALID;
2909
2910         fence_reg = obj->fence_reg;
2911         if (fence_reg < 8)
2912                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2913         else
2914                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2915
2916         if (pipelined) {
2917                 int ret = intel_ring_begin(pipelined, 4);
2918                 if (ret)
2919                         return ret;
2920
2921                 intel_ring_emit(pipelined, MI_NOOP);
2922                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2923                 intel_ring_emit(pipelined, fence_reg);
2924                 intel_ring_emit(pipelined, val);
2925                 intel_ring_advance(pipelined);
2926         } else
2927                 I915_WRITE(fence_reg, val);
2928
2929         return 0;
2930 }
2931
2932 static int
2933 i830_write_fence_reg(struct drm_i915_gem_object *obj,
2934     struct intel_ring_buffer *pipelined)
2935 {
2936         struct drm_device *dev = obj->base.dev;
2937         drm_i915_private_t *dev_priv = dev->dev_private;
2938         u32 size = obj->gtt_space->size;
2939         int regnum = obj->fence_reg;
2940         uint32_t val;
2941         uint32_t pitch_val;
2942
2943         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2944             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
2945                 kprintf(
2946 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2947                     obj->gtt_offset, size);
2948                 return -EINVAL;
2949         }
2950
2951         pitch_val = obj->stride / 128;
2952         pitch_val = ffs(pitch_val) - 1;
2953
2954         val = obj->gtt_offset;
2955         if (obj->tiling_mode == I915_TILING_Y)
2956                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2957         val |= I830_FENCE_SIZE_BITS(size);
2958         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2959         val |= I830_FENCE_REG_VALID;
2960
2961         if (pipelined) {
2962                 int ret = intel_ring_begin(pipelined, 4);
2963                 if (ret)
2964                         return ret;
2965
2966                 intel_ring_emit(pipelined, MI_NOOP);
2967                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2968                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2969                 intel_ring_emit(pipelined, val);
2970                 intel_ring_advance(pipelined);
2971         } else
2972                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2973
2974         return 0;
2975 }
2976
2977 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2978 {
2979         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
2980 }
2981
2982 static int
2983 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2984     struct intel_ring_buffer *pipelined)
2985 {
2986         int ret;
2987
2988         if (obj->fenced_gpu_access) {
2989                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2990                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
2991                             obj->base.write_domain);
2992                         if (ret)
2993                                 return ret;
2994                 }
2995
2996                 obj->fenced_gpu_access = false;
2997         }
2998
2999         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
3000                 if (!ring_passed_seqno(obj->last_fenced_ring,
3001                                        obj->last_fenced_seqno)) {
3002                         ret = i915_wait_seqno(obj->last_fenced_ring,
3003                                                 obj->last_fenced_seqno);
3004                         if (ret)
3005                                 return ret;
3006                 }
3007
3008                 obj->last_fenced_seqno = 0;
3009                 obj->last_fenced_ring = NULL;
3010         }
3011
3012         /* Ensure that all CPU reads are completed before installing a fence
3013          * and all writes before removing the fence.
3014          */
3015         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
3016                 cpu_mfence();
3017
3018         return 0;
3019 }
3020
3021 int
3022 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3023 {
3024         int ret;
3025
3026         if (obj->tiling_mode)
3027                 i915_gem_release_mmap(obj);
3028
3029         ret = i915_gem_object_flush_fence(obj, NULL);
3030         if (ret)
3031                 return ret;
3032
3033         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3034                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3035
3036                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
3037                         kprintf("%s: pin_count %d\n", __func__,
3038                             dev_priv->fence_regs[obj->fence_reg].pin_count);
3039                 i915_gem_clear_fence_reg(obj->base.dev,
3040                                          &dev_priv->fence_regs[obj->fence_reg]);
3041
3042                 obj->fence_reg = I915_FENCE_REG_NONE;
3043         }
3044
3045         return 0;
3046 }
3047
3048 static struct drm_i915_fence_reg *
3049 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
3050 {
3051         struct drm_i915_private *dev_priv = dev->dev_private;
3052         struct drm_i915_fence_reg *reg, *first, *avail;
3053         int i;
3054
3055         /* First try to find a free reg */
3056         avail = NULL;
3057         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3058                 reg = &dev_priv->fence_regs[i];
3059                 if (!reg->obj)
3060                         return reg;
3061
3062                 if (!reg->pin_count)
3063                         avail = reg;
3064         }
3065
3066         if (avail == NULL)
3067                 return NULL;
3068
3069         /* None available, try to steal one or wait for a user to finish */
3070         avail = first = NULL;
3071         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3072                 if (reg->pin_count)
3073                         continue;
3074
3075                 if (first == NULL)
3076                         first = reg;
3077
3078                 if (!pipelined ||
3079                     !reg->obj->last_fenced_ring ||
3080                     reg->obj->last_fenced_ring == pipelined) {
3081                         avail = reg;
3082                         break;
3083                 }
3084         }
3085
3086         if (avail == NULL)
3087                 avail = first;
3088
3089         return avail;
3090 }
3091
3092 int
3093 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
3094     struct intel_ring_buffer *pipelined)
3095 {
3096         struct drm_device *dev = obj->base.dev;
3097         struct drm_i915_private *dev_priv = dev->dev_private;
3098         struct drm_i915_fence_reg *reg;
3099         int ret;
3100
3101         pipelined = NULL;
3102         ret = 0;
3103
3104         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3105                 reg = &dev_priv->fence_regs[obj->fence_reg];
3106                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
3107
3108                 if (obj->tiling_changed) {
3109                         ret = i915_gem_object_flush_fence(obj, pipelined);
3110                         if (ret)
3111                                 return ret;
3112
3113                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3114                                 pipelined = NULL;
3115
3116                         if (pipelined) {
3117                                 reg->setup_seqno =
3118                                         i915_gem_next_request_seqno(pipelined);
3119                                 obj->last_fenced_seqno = reg->setup_seqno;
3120                                 obj->last_fenced_ring = pipelined;
3121                         }
3122
3123                         goto update;
3124                 }
3125
3126                 if (!pipelined) {
3127                         if (reg->setup_seqno) {
3128                                 if (!ring_passed_seqno(obj->last_fenced_ring,
3129                                     reg->setup_seqno)) {
3130                                         ret = i915_wait_seqno(
3131                                             obj->last_fenced_ring,
3132                                             reg->setup_seqno);
3133                                         if (ret)
3134                                                 return ret;
3135                                 }
3136
3137                                 reg->setup_seqno = 0;
3138                         }
3139                 } else if (obj->last_fenced_ring &&
3140                            obj->last_fenced_ring != pipelined) {
3141                         ret = i915_gem_object_flush_fence(obj, pipelined);
3142                         if (ret)
3143                                 return ret;
3144                 }
3145
3146                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3147                         pipelined = NULL;
3148                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
3149
3150                 if (obj->tiling_changed) {
3151                         if (pipelined) {
3152                                 reg->setup_seqno =
3153                                         i915_gem_next_request_seqno(pipelined);
3154                                 obj->last_fenced_seqno = reg->setup_seqno;
3155                                 obj->last_fenced_ring = pipelined;
3156                         }
3157                         goto update;
3158                 }
3159
3160                 return 0;
3161         }
3162
3163         reg = i915_find_fence_reg(dev, pipelined);
3164         if (reg == NULL)
3165                 return -EDEADLK;
3166
3167         ret = i915_gem_object_flush_fence(obj, pipelined);
3168         if (ret)
3169                 return ret;
3170
3171         if (reg->obj) {
3172                 struct drm_i915_gem_object *old = reg->obj;
3173
3174                 drm_gem_object_reference(&old->base);
3175
3176                 if (old->tiling_mode)
3177                         i915_gem_release_mmap(old);
3178
3179                 ret = i915_gem_object_flush_fence(old, pipelined);
3180                 if (ret) {
3181                         drm_gem_object_unreference(&old->base);
3182                         return ret;
3183                 }
3184
3185                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
3186                         pipelined = NULL;
3187
3188                 old->fence_reg = I915_FENCE_REG_NONE;
3189                 old->last_fenced_ring = pipelined;
3190                 old->last_fenced_seqno =
3191                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3192
3193                 drm_gem_object_unreference(&old->base);
3194         } else if (obj->last_fenced_seqno == 0)
3195                 pipelined = NULL;
3196
3197         reg->obj = obj;
3198         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
3199         obj->fence_reg = reg - dev_priv->fence_regs;
3200         obj->last_fenced_ring = pipelined;
3201
3202         reg->setup_seqno =
3203                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3204         obj->last_fenced_seqno = reg->setup_seqno;
3205
3206 update:
3207         obj->tiling_changed = false;
3208         switch (INTEL_INFO(dev)->gen) {
3209         case 7:
3210         case 6:
3211                 ret = sandybridge_write_fence_reg(obj, pipelined);
3212                 break;
3213         case 5:
3214         case 4:
3215                 ret = i965_write_fence_reg(obj, pipelined);
3216                 break;
3217         case 3:
3218                 ret = i915_write_fence_reg(obj, pipelined);
3219                 break;
3220         case 2:
3221                 ret = i830_write_fence_reg(obj, pipelined);
3222                 break;
3223         }
3224
3225         return ret;
3226 }
3227
3228 static void
3229 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3230 {
3231         drm_i915_private_t *dev_priv = dev->dev_private;
3232         uint32_t fence_reg = reg - dev_priv->fence_regs;
3233
3234         switch (INTEL_INFO(dev)->gen) {
3235         case 7:
3236         case 6:
3237                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3238                 break;
3239         case 5:
3240         case 4:
3241                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3242                 break;
3243         case 3:
3244                 if (fence_reg >= 8)
3245                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3246                 else
3247         case 2:
3248                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3249
3250                 I915_WRITE(fence_reg, 0);
3251                 break;
3252         }
3253
3254         list_del_init(&reg->lru_list);
3255         reg->obj = NULL;
3256         reg->setup_seqno = 0;
3257         reg->pin_count = 0;
3258 }
3259
3260 int
3261 i915_gem_init_object(struct drm_gem_object *obj)
3262 {
3263
3264         kprintf("i915_gem_init_object called\n");
3265         return (0);
3266 }
3267
3268 static bool
3269 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
3270 {
3271
3272         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
3273 }
3274
3275 static void
3276 i915_gem_retire_work_handler(struct work_struct *work)
3277 {
3278         drm_i915_private_t *dev_priv;
3279         struct drm_device *dev;
3280         struct intel_ring_buffer *ring;
3281         bool idle;
3282         int i;
3283
3284         dev_priv = container_of(work, drm_i915_private_t,
3285                                 mm.retire_work.work);
3286         dev = dev_priv->dev;
3287
3288         /* Come back later if the device is busy... */
3289         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
3290                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3291                                    round_jiffies_up_relative(hz));
3292                 return;
3293         }
3294
3295         i915_gem_retire_requests(dev);
3296
3297         /* Send a periodic flush down the ring so we don't hold onto GEM
3298          * objects indefinitely.
3299          */
3300         idle = true;
3301         for_each_ring(ring, dev_priv, i) {
3302                 if (ring->gpu_caches_dirty)
3303                         i915_add_request(ring, NULL, NULL);
3304
3305                 idle &= list_empty(&ring->request_list);
3306         }
3307
3308         if (!dev_priv->mm.suspended && !idle)
3309                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3310                                    round_jiffies_up_relative(hz));
3311         if (idle)
3312                 intel_mark_idle(dev);
3313
3314         DRM_UNLOCK(dev);
3315 }
3316
3317 void
3318 i915_gem_lastclose(struct drm_device *dev)
3319 {
3320         int ret;
3321
3322         if (drm_core_check_feature(dev, DRIVER_MODESET))
3323                 return;
3324
3325         ret = i915_gem_idle(dev);
3326         if (ret != 0)
3327                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3328 }
3329
3330 static int
3331 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
3332 {
3333         drm_i915_private_t *dev_priv;
3334         struct drm_i915_gem_phys_object *phys_obj;
3335         int ret;
3336
3337         dev_priv = dev->dev_private;
3338         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
3339                 return (0);
3340
3341         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3342             M_WAITOK | M_ZERO);
3343
3344         phys_obj->id = id;
3345
3346         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3347         if (phys_obj->handle == NULL) {
3348                 ret = -ENOMEM;
3349                 goto free_obj;
3350         }
3351         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3352             size / PAGE_SIZE, PAT_WRITE_COMBINING);
3353
3354         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3355
3356         return (0);
3357
3358 free_obj:
3359         drm_free(phys_obj, DRM_I915_GEM);
3360         return (ret);
3361 }
3362
3363 static void
3364 i915_gem_free_phys_object(struct drm_device *dev, int id)
3365 {
3366         drm_i915_private_t *dev_priv;
3367         struct drm_i915_gem_phys_object *phys_obj;
3368
3369         dev_priv = dev->dev_private;
3370         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3371                 return;
3372
3373         phys_obj = dev_priv->mm.phys_objs[id - 1];
3374         if (phys_obj->cur_obj != NULL)
3375                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3376
3377         drm_pci_free(dev, phys_obj->handle);
3378         drm_free(phys_obj, DRM_I915_GEM);
3379         dev_priv->mm.phys_objs[id - 1] = NULL;
3380 }
3381
3382 void
3383 i915_gem_free_all_phys_object(struct drm_device *dev)
3384 {
3385         int i;
3386
3387         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3388                 i915_gem_free_phys_object(dev, i);
3389 }
3390
3391 void
3392 i915_gem_detach_phys_object(struct drm_device *dev,
3393     struct drm_i915_gem_object *obj)
3394 {
3395         vm_page_t m;
3396         struct sf_buf *sf;
3397         char *vaddr, *dst;
3398         int i, page_count;
3399
3400         if (obj->phys_obj == NULL)
3401                 return;
3402         vaddr = obj->phys_obj->handle->vaddr;
3403
3404         page_count = obj->base.size / PAGE_SIZE;
3405         VM_OBJECT_LOCK(obj->base.vm_obj);
3406         for (i = 0; i < page_count; i++) {
3407                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3408                 if (m == NULL)
3409                         continue; /* XXX */
3410
3411                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3412                 sf = sf_buf_alloc(m);
3413                 if (sf != NULL) {
3414                         dst = (char *)sf_buf_kva(sf);
3415                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3416                         sf_buf_free(sf);
3417                 }
3418                 drm_clflush_pages(&m, 1);
3419
3420                 VM_OBJECT_LOCK(obj->base.vm_obj);
3421                 vm_page_reference(m);
3422                 vm_page_dirty(m);
3423                 vm_page_busy_wait(m, FALSE, "i915gem");
3424                 vm_page_unwire(m, 0);
3425                 vm_page_wakeup(m);
3426                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3427         }
3428         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3429         intel_gtt_chipset_flush();
3430
3431         obj->phys_obj->cur_obj = NULL;
3432         obj->phys_obj = NULL;
3433 }
3434
3435 static void
3436 init_ring_lists(struct intel_ring_buffer *ring)
3437 {
3438
3439         INIT_LIST_HEAD(&ring->active_list);
3440         INIT_LIST_HEAD(&ring->request_list);
3441         INIT_LIST_HEAD(&ring->gpu_write_list);
3442 }
3443
3444 void
3445 i915_gem_load(struct drm_device *dev)
3446 {
3447         int i;
3448         drm_i915_private_t *dev_priv = dev->dev_private;
3449
3450         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3451         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3452         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3453         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3454         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3455         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3456         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3457         for (i = 0; i < I915_NUM_RINGS; i++)
3458                 init_ring_lists(&dev_priv->ring[i]);
3459         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3460                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3461         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3462                           i915_gem_retire_work_handler);
3463         dev_priv->error_completion = 0;
3464
3465         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3466         if (IS_GEN3(dev)) {
3467                 I915_WRITE(MI_ARB_STATE,
3468                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3469         }
3470
3471         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3472
3473         /* Old X drivers will take 0-2 for front, back, depth buffers */
3474         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3475                 dev_priv->fence_reg_start = 3;
3476
3477         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3478                 dev_priv->num_fence_regs = 16;
3479         else
3480                 dev_priv->num_fence_regs = 8;
3481
3482         /* Initialize fence registers to zero */
3483         i915_gem_reset_fences(dev);
3484
3485         i915_gem_detect_bit_6_swizzle(dev);
3486
3487         dev_priv->mm.interruptible = true;
3488
3489         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
3490             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
3491 }
3492
3493 int
3494 i915_gem_attach_phys_object(struct drm_device *dev,
3495     struct drm_i915_gem_object *obj, int id, int align)
3496 {
3497         drm_i915_private_t *dev_priv;
3498         vm_page_t m;
3499         struct sf_buf *sf;
3500         char *dst, *src;
3501         int i, page_count, ret;
3502
3503         if (id > I915_MAX_PHYS_OBJECT)
3504                 return (-EINVAL);
3505
3506         if (obj->phys_obj != NULL) {
3507                 if (obj->phys_obj->id == id)
3508                         return (0);
3509                 i915_gem_detach_phys_object(dev, obj);
3510         }
3511
3512         dev_priv = dev->dev_private;
3513         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3514                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3515                 if (ret != 0) {
3516                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3517                                   id, obj->base.size);
3518                         return (ret);
3519                 }
3520         }
3521
3522         /* bind to the object */
3523         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3524         obj->phys_obj->cur_obj = obj;
3525
3526         page_count = obj->base.size / PAGE_SIZE;
3527
3528         VM_OBJECT_LOCK(obj->base.vm_obj);
3529         ret = 0;
3530         for (i = 0; i < page_count; i++) {
3531                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3532                 if (m == NULL) {
3533                         ret = -EIO;
3534                         break;
3535                 }
3536                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3537                 sf = sf_buf_alloc(m);
3538                 src = (char *)sf_buf_kva(sf);
3539                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3540                 memcpy(dst, src, PAGE_SIZE);
3541                 sf_buf_free(sf);
3542
3543                 VM_OBJECT_LOCK(obj->base.vm_obj);
3544
3545                 vm_page_reference(m);
3546                 vm_page_busy_wait(m, FALSE, "i915gem");
3547                 vm_page_unwire(m, 0);
3548                 vm_page_wakeup(m);
3549                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3550         }
3551         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3552
3553         return (0);
3554 }
3555
3556 static int
3557 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3558     uint64_t data_ptr, uint64_t offset, uint64_t size,
3559     struct drm_file *file_priv)
3560 {
3561         char *user_data, *vaddr;
3562         int ret;
3563
3564         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3565         user_data = (char *)(uintptr_t)data_ptr;
3566
3567         if (copyin_nofault(user_data, vaddr, size) != 0) {
3568                 /* The physical object once assigned is fixed for the lifetime
3569                  * of the obj, so we can safely drop the lock and continue
3570                  * to access vaddr.
3571                  */
3572                 DRM_UNLOCK(dev);
3573                 ret = -copyin(user_data, vaddr, size);
3574                 DRM_LOCK(dev);
3575                 if (ret != 0)
3576                         return (ret);
3577         }
3578
3579         intel_gtt_chipset_flush();
3580         return (0);
3581 }
3582
3583 static int
3584 i915_gpu_is_active(struct drm_device *dev)
3585 {
3586         drm_i915_private_t *dev_priv;
3587
3588         dev_priv = dev->dev_private;
3589         return (!list_empty(&dev_priv->mm.flushing_list) ||
3590             !list_empty(&dev_priv->mm.active_list));
3591 }
3592
3593 static void
3594 i915_gem_lowmem(void *arg)
3595 {
3596         struct drm_device *dev;
3597         struct drm_i915_private *dev_priv;
3598         struct drm_i915_gem_object *obj, *next;
3599         int cnt, cnt_fail, cnt_total;
3600
3601         dev = arg;
3602         dev_priv = dev->dev_private;
3603
3604         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3605                 return;
3606
3607 rescan:
3608         /* first scan for clean buffers */
3609         i915_gem_retire_requests(dev);
3610
3611         cnt_total = cnt_fail = cnt = 0;
3612
3613         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3614             mm_list) {
3615                 if (i915_gem_object_is_purgeable(obj)) {
3616                         if (i915_gem_object_unbind(obj) != 0)
3617                                 cnt_total++;
3618                 } else
3619                         cnt_total++;
3620         }
3621
3622         /* second pass, evict/count anything still on the inactive list */
3623         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3624             mm_list) {
3625                 if (i915_gem_object_unbind(obj) == 0)
3626                         cnt++;
3627                 else
3628                         cnt_fail++;
3629         }
3630
3631         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3632                 /*
3633                  * We are desperate for pages, so as a last resort, wait
3634                  * for the GPU to finish and discard whatever we can.
3635                  * This has a dramatic impact to reduce the number of
3636                  * OOM-killer events whilst running the GPU aggressively.
3637                  */
3638                 if (i915_gpu_idle(dev) == 0)
3639                         goto rescan;
3640         }
3641         DRM_UNLOCK(dev);
3642 }
3643
3644 void
3645 i915_gem_unload(struct drm_device *dev)
3646 {
3647         struct drm_i915_private *dev_priv;
3648
3649         dev_priv = dev->dev_private;
3650         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3651 }