sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64
  65 static void i915_gem_object_flush_cpu_write_domain(
  66     struct drm_i915_gem_object *obj);
  67 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  68     int tiling_mode);
  69 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  70     uint32_t size, int tiling_mode);
  71 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  72     unsigned alignment, bool map_and_fenceable);
  73 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  74     int flags);
  75 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj);
  76 static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
  77     bool write);
  78 static void i915_gem_object_set_to_full_cpu_read_domain(
  79     struct drm_i915_gem_object *obj);
  80 static int i915_gem_object_set_cpu_read_domain_range(
  81     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  82 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  83 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  84 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  85 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  86 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  87 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  88 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  89     uint32_t flush_domains);
  90 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  91     struct drm_i915_fence_reg *reg);
  92 static void i915_gem_reset_fences(struct drm_device *dev);
  93 static int i915_gem_phys_pwrite(struct drm_device *dev,
  94     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  95     uint64_t size, struct drm_file *file_priv);
  96 static void i915_gem_lowmem(void *arg);
  97
  98 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
  99 long i915_gem_wired_pages_cnt;
 100
 101 static void
 102 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size)
 103 {
 104
 105         dev_priv->mm.object_count++;
 106         dev_priv->mm.object_memory += size;
 107 }
 108
 109 static void
 110 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size)
 111 {
 112
 113         dev_priv->mm.object_count--;
 114         dev_priv->mm.object_memory -= size;
 115 }
 116
 117 static int
 118 i915_gem_wait_for_error(struct drm_device *dev)
 119 {
 120         struct drm_i915_private *dev_priv = dev->dev_private;
 121         int ret;
 122
 123         if (!atomic_read(&dev_priv->mm.wedged))
 124                 return 0;
 125
 126         lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 127         while (dev_priv->error_completion == 0) {
 128                 ret = -lksleep(&dev_priv->error_completion,
 129                     &dev_priv->error_completion_lock, PCATCH, "915wco", 0);
 130                 if (ret != 0) {
 131                         lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 132                         return (ret);
 133                 }
 134         }
 135         lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 136
 137         if (atomic_read(&dev_priv->mm.wedged)) {
 138                 /* GPU is hung, bump the completion count to account for
 139                  * the token we just consumed so that we never hit zero and
 140                  * end up waiting upon a subsequent completion event that
 141                  * will never happen.
 142                  */
 143                 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 144                 dev_priv->error_completion++;
 145                 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 146         }
 147         return 0;
 148 }
 149
 150 int
 151 i915_mutex_lock_interruptible(struct drm_device *dev)
 152 {
 153         struct drm_i915_private *dev_priv;
 154         int ret;
 155
 156         dev_priv = dev->dev_private;
 157         ret = i915_gem_wait_for_error(dev);
 158         if (ret != 0)
 159                 return (ret);
 160
 161         /*
 162          * interruptible shall it be. might indeed be if dev_lock is
 163          * changed to sx
 164          */
 165         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 166         if (ret != 0)
 167                 return (-ret);
 168
 169         return (0);
 170 }
 171
 172
 173 static void
 174 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
 175 {
 176         struct drm_device *dev;
 177         drm_i915_private_t *dev_priv;
 178         int ret;
 179
 180         dev = obj->base.dev;
 181         dev_priv = dev->dev_private;
 182
 183         ret = i915_gem_object_unbind(obj);
 184         if (ret == -ERESTART) {
 185                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
 186                 return;
 187         }
 188
 189         drm_gem_free_mmap_offset(&obj->base);
 190         drm_gem_object_release(&obj->base);
 191         i915_gem_info_remove_obj(dev_priv, obj->base.size);
 192
 193         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
 194         drm_free(obj->bit_17, DRM_I915_GEM);
 195         drm_free(obj, DRM_I915_GEM);
 196 }
 197
 198 void
 199 i915_gem_free_object(struct drm_gem_object *gem_obj)
 200 {
 201         struct drm_i915_gem_object *obj;
 202         struct drm_device *dev;
 203
 204         obj = to_intel_bo(gem_obj);
 205         dev = obj->base.dev;
 206
 207         while (obj->pin_count > 0)
 208                 i915_gem_object_unpin(obj);
 209
 210         if (obj->phys_obj != NULL)
 211                 i915_gem_detach_phys_object(dev, obj);
 212
 213         i915_gem_free_object_tail(obj);
 214 }
 215
 216 int
 217 i915_gem_do_init(struct drm_device *dev, unsigned long start,
 218     unsigned long mappable_end, unsigned long end)
 219 {
 220         drm_i915_private_t *dev_priv;
 221         unsigned long mappable;
 222         int error;
 223
 224         dev_priv = dev->dev_private;
 225         mappable = min(end, mappable_end) - start;
 226
 227         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
 228
 229         dev_priv->mm.gtt_start = start;
 230         dev_priv->mm.gtt_mappable_end = mappable_end;
 231         dev_priv->mm.gtt_end = end;
 232         dev_priv->mm.gtt_total = end - start;
 233         dev_priv->mm.mappable_gtt_total = mappable;
 234
 235         /* Take over this portion of the GTT */
 236         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
 237         device_printf(dev->dev,
 238             "taking over the fictitious range 0x%lx-0x%lx\n",
 239             dev->agp->base + start, dev->agp->base + start + mappable);
 240         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
 241             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
 242         return (error);
 243 }
 244
 245 int
 246 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 247     struct drm_file *file)
 248 {
 249         struct drm_i915_gem_init *args;
 250         drm_i915_private_t *dev_priv;
 251
 252         dev_priv = dev->dev_private;
 253         args = data;
 254
 255         if (args->gtt_start >= args->gtt_end ||
 256             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 257                 return (-EINVAL);
 258
 259         /*
 260          * XXXKIB. The second-time initialization should be guarded
 261          * against.
 262          */
 263         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 264         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 265         lockmgr(&dev->dev_lock, LK_RELEASE);
 266
 267         return 0;
 268 }
 269
 270 int
 271 i915_gem_idle(struct drm_device *dev)
 272 {
 273         drm_i915_private_t *dev_priv;
 274         int ret;
 275
 276         dev_priv = dev->dev_private;
 277         if (dev_priv->mm.suspended)
 278                 return (0);
 279
 280         ret = i915_gpu_idle(dev, true);
 281         if (ret != 0)
 282                 return (ret);
 283
 284         /* Under UMS, be paranoid and evict. */
 285         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
 286                 ret = i915_gem_evict_inactive(dev, false);
 287                 if (ret != 0)
 288                         return ret;
 289         }
 290
 291         i915_gem_reset_fences(dev);
 292
 293         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
 294          * We need to replace this with a semaphore, or something.
 295          * And not confound mm.suspended!
 296          */
 297         dev_priv->mm.suspended = 1;
 298         del_timer_sync(&dev_priv->hangcheck_timer);
 299
 300         i915_kernel_lost_context(dev);
 301         i915_gem_cleanup_ringbuffer(dev);
 302
 303         /* Cancel the retire work handler, which should be idle now. */
 304         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
 305
 306         return (ret);
 307 }
 308
 309 void
 310 i915_gem_init_swizzling(struct drm_device *dev)
 311 {
 312         drm_i915_private_t *dev_priv;
 313
 314         dev_priv = dev->dev_private;
 315
 316         if (INTEL_INFO(dev)->gen < 5 ||
 317             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
 318                 return;
 319
 320         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 321                                  DISP_TILE_SURFACE_SWIZZLING);
 322
 323         if (IS_GEN5(dev))
 324                 return;
 325
 326         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
 327         if (IS_GEN6(dev))
 328                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
 329         else
 330                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
 331 }
 332
 333 int
 334 i915_gem_init_hw(struct drm_device *dev)
 335 {
 336         drm_i915_private_t *dev_priv;
 337         int ret;
 338
 339         dev_priv = dev->dev_private;
 340
 341         i915_gem_init_swizzling(dev);
 342
 343         ret = intel_init_render_ring_buffer(dev);
 344         if (ret != 0)
 345                 return (ret);
 346
 347         if (HAS_BSD(dev)) {
 348                 ret = intel_init_bsd_ring_buffer(dev);
 349                 if (ret != 0)
 350                         goto cleanup_render_ring;
 351         }
 352
 353         if (HAS_BLT(dev)) {
 354                 ret = intel_init_blt_ring_buffer(dev);
 355                 if (ret != 0)
 356                         goto cleanup_bsd_ring;
 357         }
 358
 359         dev_priv->next_seqno = 1;
 360         i915_gem_init_ppgtt(dev);
 361         return (0);
 362
 363 cleanup_bsd_ring:
 364         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
 365 cleanup_render_ring:
 366         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
 367         return (ret);
 368 }
 369
 370 int
 371 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 372     struct drm_file *file)
 373 {
 374         struct drm_i915_private *dev_priv;
 375         struct drm_i915_gem_get_aperture *args;
 376         struct drm_i915_gem_object *obj;
 377         size_t pinned;
 378
 379         dev_priv = dev->dev_private;
 380         args = data;
 381
 382         if (!(dev->driver->driver_features & DRIVER_GEM))
 383                 return (-ENODEV);
 384
 385         pinned = 0;
 386         DRM_LOCK(dev);
 387         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 388                 pinned += obj->gtt_space->size;
 389         DRM_UNLOCK(dev);
 390
 391         args->aper_size = dev_priv->mm.gtt_total;
 392         args->aper_available_size = args->aper_size - pinned;
 393
 394         return (0);
 395 }
 396
 397 int
 398 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
 399      bool map_and_fenceable)
 400 {
 401         struct drm_device *dev;
 402         struct drm_i915_private *dev_priv;
 403         int ret;
 404
 405         dev = obj->base.dev;
 406         dev_priv = dev->dev_private;
 407
 408         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
 409             ("Max pin count"));
 410
 411         if (obj->gtt_space != NULL) {
 412                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
 413                     (map_and_fenceable && !obj->map_and_fenceable)) {
 414                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
 415                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
 416                              " obj->map_and_fenceable=%d\n",
 417                              obj->gtt_offset, alignment,
 418                              map_and_fenceable,
 419                              obj->map_and_fenceable);
 420                         ret = i915_gem_object_unbind(obj);
 421                         if (ret != 0)
 422                                 return (ret);
 423                 }
 424         }
 425
 426         if (obj->gtt_space == NULL) {
 427                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
 428                     map_and_fenceable);
 429                 if (ret)
 430                         return (ret);
 431         }
 432
 433         if (obj->pin_count++ == 0 && !obj->active)
 434                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
 435         obj->pin_mappable |= map_and_fenceable;
 436
 437 #if 1
 438         KIB_NOTYET();
 439 #else
 440         WARN_ON(i915_verify_lists(dev));
 441 #endif
 442         return (0);
 443 }
 444
 445 void
 446 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
 447 {
 448         struct drm_device *dev;
 449         drm_i915_private_t *dev_priv;
 450
 451         dev = obj->base.dev;
 452         dev_priv = dev->dev_private;
 453
 454 #if 1
 455         KIB_NOTYET();
 456 #else
 457         WARN_ON(i915_verify_lists(dev));
 458 #endif
 459
 460         KASSERT(obj->pin_count != 0, ("zero pin count"));
 461         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
 462
 463         if (--obj->pin_count == 0) {
 464                 if (!obj->active)
 465                         list_move_tail(&obj->mm_list,
 466                             &dev_priv->mm.inactive_list);
 467                 obj->pin_mappable = false;
 468         }
 469 #if 1
 470         KIB_NOTYET();
 471 #else
 472         WARN_ON(i915_verify_lists(dev));
 473 #endif
 474 }
 475
 476 int
 477 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 478     struct drm_file *file)
 479 {
 480         struct drm_i915_gem_pin *args;
 481         struct drm_i915_gem_object *obj;
 482         struct drm_gem_object *gobj;
 483         int ret;
 484
 485         args = data;
 486
 487         ret = i915_mutex_lock_interruptible(dev);
 488         if (ret != 0)
 489                 return ret;
 490
 491         gobj = drm_gem_object_lookup(dev, file, args->handle);
 492         if (gobj == NULL) {
 493                 ret = -ENOENT;
 494                 goto unlock;
 495         }
 496         obj = to_intel_bo(gobj);
 497
 498         if (obj->madv != I915_MADV_WILLNEED) {
 499                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
 500                 ret = -EINVAL;
 501                 goto out;
 502         }
 503
 504         if (obj->pin_filp != NULL && obj->pin_filp != file) {
 505                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
 506                     args->handle);
 507                 ret = -EINVAL;
 508                 goto out;
 509         }
 510
 511         obj->user_pin_count++;
 512         obj->pin_filp = file;
 513         if (obj->user_pin_count == 1) {
 514                 ret = i915_gem_object_pin(obj, args->alignment, true);
 515                 if (ret != 0)
 516                         goto out;
 517         }
 518
 519         /* XXX - flush the CPU caches for pinned objects
 520          * as the X server doesn't manage domains yet
 521          */
 522         i915_gem_object_flush_cpu_write_domain(obj);
 523         args->offset = obj->gtt_offset;
 524 out:
 525         drm_gem_object_unreference(&obj->base);
 526 unlock:
 527         DRM_UNLOCK(dev);
 528         return (ret);
 529 }
 530
 531 int
 532 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
 533     struct drm_file *file)
 534 {
 535         struct drm_i915_gem_pin *args;
 536         struct drm_i915_gem_object *obj;
 537         int ret;
 538
 539         args = data;
 540         ret = i915_mutex_lock_interruptible(dev);
 541         if (ret != 0)
 542                 return (ret);
 543
 544         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 545         if (&obj->base == NULL) {
 546                 ret = -ENOENT;
 547                 goto unlock;
 548         }
 549
 550         if (obj->pin_filp != file) {
 551                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
 552                     args->handle);
 553                 ret = -EINVAL;
 554                 goto out;
 555         }
 556         obj->user_pin_count--;
 557         if (obj->user_pin_count == 0) {
 558                 obj->pin_filp = NULL;
 559                 i915_gem_object_unpin(obj);
 560         }
 561
 562 out:
 563         drm_gem_object_unreference(&obj->base);
 564 unlock:
 565         DRM_UNLOCK(dev);
 566         return (ret);
 567 }
 568
 569 int
 570 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 571     struct drm_file *file)
 572 {
 573         struct drm_i915_gem_busy *args;
 574         struct drm_i915_gem_object *obj;
 575         struct drm_i915_gem_request *request;
 576         int ret;
 577
 578         args = data;
 579
 580         ret = i915_mutex_lock_interruptible(dev);
 581         if (ret != 0)
 582                 return ret;
 583
 584         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 585         if (&obj->base == NULL) {
 586                 ret = -ENOENT;
 587                 goto unlock;
 588         }
 589
 590         args->busy = obj->active;
 591         if (args->busy) {
 592                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
 593                         ret = i915_gem_flush_ring(obj->ring,
 594                             0, obj->base.write_domain);
 595                 } else if (obj->ring->outstanding_lazy_request ==
 596                     obj->last_rendering_seqno) {
 597                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
 598                             M_WAITOK | M_ZERO);
 599                         ret = i915_add_request(obj->ring, NULL, request);
 600                         if (ret != 0)
 601                                 drm_free(request, DRM_I915_GEM);
 602                 }
 603
 604                 i915_gem_retire_requests_ring(obj->ring);
 605                 args->busy = obj->active;
 606         }
 607
 608         drm_gem_object_unreference(&obj->base);
 609 unlock:
 610         DRM_UNLOCK(dev);
 611         return (ret);
 612 }
 613
 614 /* Throttle our rendering by waiting until the ring has completed our requests
 615  * emitted over 20 msec ago.
 616  *
 617  * Note that if we were to use the current jiffies each time around the loop,
 618  * we wouldn't escape the function with any frames outstanding if the time to
 619  * render a frame was over 20ms.
 620  *
 621  * This should get us reasonable parallelism between CPU and GPU but also
 622  * relatively low latency when blocking on a particular request to finish.
 623  */
 624 static int
 625 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 626 {
 627         struct drm_i915_private *dev_priv = dev->dev_private;
 628         struct drm_i915_file_private *file_priv = file->driver_priv;
 629         unsigned long recent_enough = ticks - (20 * hz / 1000);
 630         struct drm_i915_gem_request *request;
 631         struct intel_ring_buffer *ring = NULL;
 632         u32 seqno = 0;
 633         int ret;
 634
 635         dev_priv = dev->dev_private;
 636         if (atomic_read(&dev_priv->mm.wedged))
 637                 return -EIO;
 638
 639         recent_enough = ticks - (20 * hz / 1000);
 640         ring = NULL;
 641         seqno = 0;
 642
 643         spin_lock(&file_priv->mm.lock);
 644         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
 645                 if (time_after_eq(request->emitted_jiffies, recent_enough))
 646                         break;
 647
 648                 ring = request->ring;
 649                 seqno = request->seqno;
 650         }
 651         spin_unlock(&file_priv->mm.lock);
 652
 653         if (seqno == 0)
 654                 return 0;
 655
 656         ret = 0;
 657         lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
 658         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
 659                 if (ring->irq_get(ring)) {
 660                         while (ret == 0 &&
 661                             !(i915_seqno_passed(ring->get_seqno(ring), seqno) ||
 662                             atomic_read(&dev_priv->mm.wedged)))
 663                                 ret = -lksleep(ring, &ring->irq_lock, PCATCH,
 664                                     "915thr", 0);
 665                         ring->irq_put(ring);
 666                         if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
 667                                 ret = -EIO;
 668                 } else if (_intel_wait_for(dev,
 669                     i915_seqno_passed(ring->get_seqno(ring), seqno) ||
 670                     atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
 671                         ret = -EBUSY;
 672                 }
 673         }
 674         lockmgr(&ring->irq_lock, LK_RELEASE);
 675
 676         if (ret == 0)
 677                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 678
 679         return ret;
 680 }
 681
 682 int
 683 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
 684     struct drm_file *file_priv)
 685 {
 686
 687         return (i915_gem_ring_throttle(dev, file_priv));
 688 }
 689
 690 int
 691 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 692                        struct drm_file *file_priv)
 693 {
 694         struct drm_i915_gem_madvise *args = data;
 695         struct drm_i915_gem_object *obj;
 696         int ret;
 697
 698         switch (args->madv) {
 699         case I915_MADV_DONTNEED:
 700         case I915_MADV_WILLNEED:
 701             break;
 702         default:
 703             return -EINVAL;
 704         }
 705
 706         ret = i915_mutex_lock_interruptible(dev);
 707         if (ret)
 708                 return ret;
 709
 710         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
 711         if (&obj->base == NULL) {
 712                 ret = -ENOENT;
 713                 goto unlock;
 714         }
 715
 716         if (obj->pin_count) {
 717                 ret = -EINVAL;
 718                 goto out;
 719         }
 720
 721         if (obj->madv != __I915_MADV_PURGED)
 722                 obj->madv = args->madv;
 723
 724         /* if the object is no longer attached, discard its backing storage */
 725         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
 726                 i915_gem_object_truncate(obj);
 727
 728         args->retained = obj->madv != __I915_MADV_PURGED;
 729
 730 out:
 731         drm_gem_object_unreference(&obj->base);
 732 unlock:
 733         DRM_UNLOCK(dev);
 734         return ret;
 735 }
 736
 737 void
 738 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 739 {
 740         drm_i915_private_t *dev_priv;
 741         int i;
 742
 743         dev_priv = dev->dev_private;
 744         for (i = 0; i < I915_NUM_RINGS; i++)
 745                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
 746 }
 747
 748 int
 749 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 750                        struct drm_file *file_priv)
 751 {
 752         drm_i915_private_t *dev_priv = dev->dev_private;
 753         int ret;
 754
 755         if (drm_core_check_feature(dev, DRIVER_MODESET))
 756                 return 0;
 757
 758         if (atomic_read(&dev_priv->mm.wedged)) {
 759                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
 760                 atomic_set(&dev_priv->mm.wedged, 0);
 761         }
 762
 763         DRM_LOCK(dev);
 764         dev_priv->mm.suspended = 0;
 765
 766         ret = i915_gem_init_hw(dev);
 767         if (ret != 0) {
 768                 DRM_UNLOCK(dev);
 769                 return ret;
 770         }
 771
 772         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
 773         DRM_UNLOCK(dev);
 774
 775         ret = drm_irq_install(dev);
 776         if (ret)
 777                 goto cleanup_ringbuffer;
 778
 779         return 0;
 780
 781 cleanup_ringbuffer:
 782         DRM_LOCK(dev);
 783         i915_gem_cleanup_ringbuffer(dev);
 784         dev_priv->mm.suspended = 1;
 785         DRM_UNLOCK(dev);
 786
 787         return ret;
 788 }
 789
 790 int
 791 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
 792     struct drm_file *file_priv)
 793 {
 794
 795         if (drm_core_check_feature(dev, DRIVER_MODESET))
 796                 return 0;
 797
 798         drm_irq_uninstall(dev);
 799         return (i915_gem_idle(dev));
 800 }
 801
 802 int
 803 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 804     uint32_t *handle_p)
 805 {
 806         struct drm_i915_gem_object *obj;
 807         uint32_t handle;
 808         int ret;
 809
 810         size = roundup(size, PAGE_SIZE);
 811         if (size == 0)
 812                 return (-EINVAL);
 813
 814         obj = i915_gem_alloc_object(dev, size);
 815         if (obj == NULL)
 816                 return (-ENOMEM);
 817
 818         handle = 0;
 819         ret = drm_gem_handle_create(file, &obj->base, &handle);
 820         if (ret != 0) {
 821                 drm_gem_object_release(&obj->base);
 822                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 823                 drm_free(obj, DRM_I915_GEM);
 824                 return (-ret);
 825         }
 826
 827         /* drop reference from allocate - handle holds it now */
 828         drm_gem_object_unreference(&obj->base);
 829         *handle_p = handle;
 830         return (0);
 831 }
 832
 833 int
 834 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 835     struct drm_mode_create_dumb *args)
 836 {
 837
 838         /* have to work out size/pitch and return them */
 839         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 840         args->size = args->pitch * args->height;
 841         return (i915_gem_create(file, dev, args->size, &args->handle));
 842 }
 843
 844 int
 845 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
 846     uint32_t handle)
 847 {
 848
 849         return (drm_gem_handle_delete(file, handle));
 850 }
 851
 852 int
 853 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 854     struct drm_file *file)
 855 {
 856         struct drm_i915_gem_create *args = data;
 857
 858         return (i915_gem_create(file, dev, args->size, &args->handle));
 859 }
 860
 861 static inline void vm_page_reference(vm_page_t m)
 862 {
 863         vm_page_flag_set(m, PG_REFERENCED);
 864 }
 865
 866 static int
 867 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
 868     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
 869     struct drm_file *file)
 870 {
 871         vm_object_t vm_obj;
 872         vm_page_t m;
 873         struct sf_buf *sf;
 874         vm_offset_t mkva;
 875         vm_pindex_t obj_pi;
 876         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
 877
 878         if (obj->gtt_offset != 0 && rw == UIO_READ)
 879                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 880         else
 881                 do_bit17_swizzling = 0;
 882
 883         obj->dirty = 1;
 884         vm_obj = obj->base.vm_obj;
 885         ret = 0;
 886
 887         VM_OBJECT_LOCK(vm_obj);
 888         vm_object_pip_add(vm_obj, 1);
 889         while (size > 0) {
 890                 obj_pi = OFF_TO_IDX(offset);
 891                 obj_po = offset & PAGE_MASK;
 892
 893                 m = i915_gem_wire_page(vm_obj, obj_pi);
 894                 VM_OBJECT_UNLOCK(vm_obj);
 895
 896                 sf = sf_buf_alloc(m);
 897                 mkva = sf_buf_kva(sf);
 898                 length = min(size, PAGE_SIZE - obj_po);
 899                 while (length > 0) {
 900                         if (do_bit17_swizzling &&
 901                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
 902                                 cnt = roundup2(obj_po + 1, 64);
 903                                 cnt = min(cnt - obj_po, length);
 904                                 swizzled_po = obj_po ^ 64;
 905                         } else {
 906                                 cnt = length;
 907                                 swizzled_po = obj_po;
 908                         }
 909                         if (rw == UIO_READ)
 910                                 ret = -copyout_nofault(
 911                                     (char *)mkva + swizzled_po,
 912                                     (void *)(uintptr_t)data_ptr, cnt);
 913                         else
 914                                 ret = -copyin_nofault(
 915                                     (void *)(uintptr_t)data_ptr,
 916                                     (char *)mkva + swizzled_po, cnt);
 917                         if (ret != 0)
 918                                 break;
 919                         data_ptr += cnt;
 920                         size -= cnt;
 921                         length -= cnt;
 922                         offset += cnt;
 923                         obj_po += cnt;
 924                 }
 925                 sf_buf_free(sf);
 926                 VM_OBJECT_LOCK(vm_obj);
 927                 if (rw == UIO_WRITE)
 928                         vm_page_dirty(m);
 929                 vm_page_reference(m);
 930                 vm_page_busy_wait(m, FALSE, "i915gem");
 931                 vm_page_unwire(m, 1);
 932                 vm_page_wakeup(m);
 933                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 934
 935                 if (ret != 0)
 936                         break;
 937         }
 938         vm_object_pip_wakeup(vm_obj);
 939         VM_OBJECT_UNLOCK(vm_obj);
 940
 941         return (ret);
 942 }
 943
 944 static int
 945 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
 946     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
 947 {
 948         vm_offset_t mkva;
 949         int ret;
 950
 951         /*
 952          * Pass the unaligned physical address and size to pmap_mapdev_attr()
 953          * so it can properly calculate whether an extra page needs to be
 954          * mapped or not to cover the requested range.  The function will
 955          * add the page offset into the returned mkva for us.
 956          */
 957         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
 958             offset, size, PAT_WRITE_COMBINING);
 959         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
 960         pmap_unmapdev(mkva, size);
 961         return (ret);
 962 }
 963
 964 static int
 965 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
 966     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
 967 {
 968         struct drm_i915_gem_object *obj;
 969         vm_page_t *ma;
 970         vm_offset_t start, end;
 971         int npages, ret;
 972
 973         if (size == 0)
 974                 return (0);
 975         start = trunc_page(data_ptr);
 976         end = round_page(data_ptr + size);
 977         npages = howmany(end - start, PAGE_SIZE);
 978         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
 979             M_ZERO);
 980         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
 981             (vm_offset_t)data_ptr, size,
 982             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
 983         if (npages == -1) {
 984                 ret = -EFAULT;
 985                 goto free_ma;
 986         }
 987
 988         ret = i915_mutex_lock_interruptible(dev);
 989         if (ret != 0)
 990                 goto unlocked;
 991
 992         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 993         if (&obj->base == NULL) {
 994                 ret = -ENOENT;
 995                 goto unlock;
 996         }
 997         if (offset > obj->base.size || size > obj->base.size - offset) {
 998                 ret = -EINVAL;
 999                 goto out;
1000         }
1001
1002         if (rw == UIO_READ) {
1003                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
1004                     offset, size);
1005                 if (ret != 0)
1006                         goto out;
1007                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1008                     UIO_READ, file);
1009         } else {
1010                 if (obj->phys_obj) {
1011                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
1012                             size, file);
1013                 } else if (obj->gtt_space &&
1014                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1015                         ret = i915_gem_object_pin(obj, 0, true);
1016                         if (ret != 0)
1017                                 goto out;
1018                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1019                         if (ret != 0)
1020                                 goto out_unpin;
1021                         ret = i915_gem_object_put_fence(obj);
1022                         if (ret != 0)
1023                                 goto out_unpin;
1024                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
1025                             offset, file);
1026 out_unpin:
1027                         i915_gem_object_unpin(obj);
1028                 } else {
1029                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
1030                         if (ret != 0)
1031                                 goto out;
1032                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1033                             UIO_WRITE, file);
1034                 }
1035         }
1036 out:
1037         drm_gem_object_unreference(&obj->base);
1038 unlock:
1039         DRM_UNLOCK(dev);
1040 unlocked:
1041         vm_page_unhold_pages(ma, npages);
1042 free_ma:
1043         drm_free(ma, DRM_I915_GEM);
1044         return (ret);
1045 }
1046
1047 int
1048 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1049 {
1050         struct drm_i915_gem_pread *args;
1051
1052         args = data;
1053         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1054             args->offset, UIO_READ, file));
1055 }
1056
1057 int
1058 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1059 {
1060         struct drm_i915_gem_pwrite *args;
1061
1062         args = data;
1063         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1064             args->offset, UIO_WRITE, file));
1065 }
1066
1067 int
1068 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1069     struct drm_file *file)
1070 {
1071         struct drm_i915_gem_set_domain *args;
1072         struct drm_i915_gem_object *obj;
1073         uint32_t read_domains;
1074         uint32_t write_domain;
1075         int ret;
1076
1077         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1078                 return (-ENODEV);
1079
1080         args = data;
1081         read_domains = args->read_domains;
1082         write_domain = args->write_domain;
1083
1084         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
1085             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
1086             (write_domain != 0 && read_domains != write_domain))
1087                 return (-EINVAL);
1088
1089         ret = i915_mutex_lock_interruptible(dev);
1090         if (ret != 0)
1091                 return (ret);
1092
1093         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1094         if (&obj->base == NULL) {
1095                 ret = -ENOENT;
1096                 goto unlock;
1097         }
1098
1099         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
1100                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1101                 if (ret == -EINVAL)
1102                         ret = 0;
1103         } else
1104                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1105
1106         drm_gem_object_unreference(&obj->base);
1107 unlock:
1108         DRM_UNLOCK(dev);
1109         return (ret);
1110 }
1111
1112 int
1113 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1114     struct drm_file *file)
1115 {
1116         struct drm_i915_gem_sw_finish *args;
1117         struct drm_i915_gem_object *obj;
1118         int ret;
1119
1120         args = data;
1121         ret = 0;
1122         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1123                 return (ENODEV);
1124         ret = i915_mutex_lock_interruptible(dev);
1125         if (ret != 0)
1126                 return (ret);
1127         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1128         if (&obj->base == NULL) {
1129                 ret = -ENOENT;
1130                 goto unlock;
1131         }
1132         if (obj->pin_count != 0)
1133                 i915_gem_object_flush_cpu_write_domain(obj);
1134         drm_gem_object_unreference(&obj->base);
1135 unlock:
1136         DRM_UNLOCK(dev);
1137         return (ret);
1138 }
1139
1140 int
1141 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1142     struct drm_file *file)
1143 {
1144         struct drm_i915_gem_mmap *args;
1145         struct drm_gem_object *obj;
1146         struct proc *p;
1147         vm_map_t map;
1148         vm_offset_t addr;
1149         vm_size_t size;
1150         int error, rv;
1151
1152         args = data;
1153
1154         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1155                 return (-ENODEV);
1156
1157         obj = drm_gem_object_lookup(dev, file, args->handle);
1158         if (obj == NULL)
1159                 return (-ENOENT);
1160         error = 0;
1161         if (args->size == 0)
1162                 goto out;
1163         p = curproc;
1164         map = &p->p_vmspace->vm_map;
1165         size = round_page(args->size);
1166         PROC_LOCK(p);
1167         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
1168                 PROC_UNLOCK(p);
1169                 error = ENOMEM;
1170                 goto out;
1171         }
1172         PROC_UNLOCK(p);
1173
1174         addr = 0;
1175         vm_object_hold(obj->vm_obj);
1176         vm_object_reference_locked(obj->vm_obj);
1177         vm_object_drop(obj->vm_obj);
1178         DRM_UNLOCK(dev);
1179         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
1180             PAGE_SIZE, /* align */
1181             TRUE, /* fitit */
1182             VM_MAPTYPE_NORMAL, /* maptype */
1183             VM_PROT_READ | VM_PROT_WRITE, /* prot */
1184             VM_PROT_READ | VM_PROT_WRITE, /* max */
1185             MAP_SHARED /* cow */);
1186         if (rv != KERN_SUCCESS) {
1187                 vm_object_deallocate(obj->vm_obj);
1188                 error = -vm_mmap_to_errno(rv);
1189         } else {
1190                 args->addr_ptr = (uint64_t)addr;
1191         }
1192         DRM_LOCK(dev);
1193 out:
1194         drm_gem_object_unreference(obj);
1195         return (error);
1196 }
1197
1198 static int
1199 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
1200     vm_ooffset_t foff, struct ucred *cred, u_short *color)
1201 {
1202
1203         *color = 0; /* XXXKIB */
1204         return (0);
1205 }
1206
1207 int i915_intr_pf;
1208
1209 static int
1210 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
1211     vm_page_t *mres)
1212 {
1213         struct drm_gem_object *gem_obj;
1214         struct drm_i915_gem_object *obj;
1215         struct drm_device *dev;
1216         drm_i915_private_t *dev_priv;
1217         vm_page_t m, oldm;
1218         int cause, ret;
1219         bool write;
1220
1221         gem_obj = vm_obj->handle;
1222         obj = to_intel_bo(gem_obj);
1223         dev = obj->base.dev;
1224         dev_priv = dev->dev_private;
1225 #if 0
1226         write = (prot & VM_PROT_WRITE) != 0;
1227 #else
1228         write = true;
1229 #endif
1230         vm_object_pip_add(vm_obj, 1);
1231
1232         /*
1233          * Remove the placeholder page inserted by vm_fault() from the
1234          * object before dropping the object lock. If
1235          * i915_gem_release_mmap() is active in parallel on this gem
1236          * object, then it owns the drm device sx and might find the
1237          * placeholder already. Then, since the page is busy,
1238          * i915_gem_release_mmap() sleeps waiting for the busy state
1239          * of the page cleared. We will be not able to acquire drm
1240          * device lock until i915_gem_release_mmap() is able to make a
1241          * progress.
1242          */
1243         if (*mres != NULL) {
1244                 oldm = *mres;
1245                 vm_page_remove(oldm);
1246                 *mres = NULL;
1247         } else
1248                 oldm = NULL;
1249 retry:
1250         VM_OBJECT_UNLOCK(vm_obj);
1251 unlocked_vmobj:
1252         cause = ret = 0;
1253         m = NULL;
1254
1255         if (i915_intr_pf) {
1256                 ret = i915_mutex_lock_interruptible(dev);
1257                 if (ret != 0) {
1258                         cause = 10;
1259                         goto out;
1260                 }
1261         } else
1262                 DRM_LOCK(dev);
1263
1264         /*
1265          * Since the object lock was dropped, other thread might have
1266          * faulted on the same GTT address and instantiated the
1267          * mapping for the page.  Recheck.
1268          */
1269         VM_OBJECT_LOCK(vm_obj);
1270         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
1271         if (m != NULL) {
1272                 if ((m->flags & PG_BUSY) != 0) {
1273                         DRM_UNLOCK(dev);
1274 #if 0 /* XXX */
1275                         vm_page_sleep(m, "915pee");
1276 #endif
1277                         goto retry;
1278                 }
1279                 goto have_page;
1280         } else
1281                 VM_OBJECT_UNLOCK(vm_obj);
1282
1283         /* Now bind it into the GTT if needed */
1284         if (!obj->map_and_fenceable) {
1285                 ret = i915_gem_object_unbind(obj);
1286                 if (ret != 0) {
1287                         cause = 20;
1288                         goto unlock;
1289                 }
1290         }
1291         if (!obj->gtt_space) {
1292                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1293                 if (ret != 0) {
1294                         cause = 30;
1295                         goto unlock;
1296                 }
1297
1298                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1299                 if (ret != 0) {
1300                         cause = 40;
1301                         goto unlock;
1302                 }
1303         }
1304
1305         if (obj->tiling_mode == I915_TILING_NONE)
1306                 ret = i915_gem_object_put_fence(obj);
1307         else
1308                 ret = i915_gem_object_get_fence(obj, NULL);
1309         if (ret != 0) {
1310                 cause = 50;
1311                 goto unlock;
1312         }
1313
1314         if (i915_gem_object_is_inactive(obj))
1315                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1316
1317         obj->fault_mappable = true;
1318         VM_OBJECT_LOCK(vm_obj);
1319         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
1320             offset);
1321         if (m == NULL) {
1322                 cause = 60;
1323                 ret = -EFAULT;
1324                 goto unlock;
1325         }
1326         KASSERT((m->flags & PG_FICTITIOUS) != 0,
1327             ("not fictitious %p", m));
1328         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
1329
1330         if ((m->flags & PG_BUSY) != 0) {
1331                 DRM_UNLOCK(dev);
1332 #if 0 /* XXX */
1333                 vm_page_sleep(m, "915pbs");
1334 #endif
1335                 goto retry;
1336         }
1337         m->valid = VM_PAGE_BITS_ALL;
1338         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
1339 have_page:
1340         *mres = m;
1341         vm_page_busy_try(m, false);
1342
1343         DRM_UNLOCK(dev);
1344         if (oldm != NULL) {
1345                 vm_page_free(oldm);
1346         }
1347         vm_object_pip_wakeup(vm_obj);
1348         return (VM_PAGER_OK);
1349
1350 unlock:
1351         DRM_UNLOCK(dev);
1352 out:
1353         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
1354         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
1355                 goto unlocked_vmobj;
1356         }
1357         VM_OBJECT_LOCK(vm_obj);
1358         vm_object_pip_wakeup(vm_obj);
1359         return (VM_PAGER_ERROR);
1360 }
1361
1362 static void
1363 i915_gem_pager_dtor(void *handle)
1364 {
1365         struct drm_gem_object *obj;
1366         struct drm_device *dev;
1367
1368         obj = handle;
1369         dev = obj->dev;
1370
1371         DRM_LOCK(dev);
1372         drm_gem_free_mmap_offset(obj);
1373         i915_gem_release_mmap(to_intel_bo(obj));
1374         drm_gem_object_unreference(obj);
1375         DRM_UNLOCK(dev);
1376 }
1377
1378 struct cdev_pager_ops i915_gem_pager_ops = {
1379         .cdev_pg_fault  = i915_gem_pager_fault,
1380         .cdev_pg_ctor   = i915_gem_pager_ctor,
1381         .cdev_pg_dtor   = i915_gem_pager_dtor
1382 };
1383
1384 int
1385 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev,
1386     uint32_t handle, uint64_t *offset)
1387 {
1388         struct drm_i915_private *dev_priv;
1389         struct drm_i915_gem_object *obj;
1390         int ret;
1391
1392         if (!(dev->driver->driver_features & DRIVER_GEM))
1393                 return (-ENODEV);
1394
1395         dev_priv = dev->dev_private;
1396
1397         ret = i915_mutex_lock_interruptible(dev);
1398         if (ret != 0)
1399                 return (ret);
1400
1401         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1402         if (&obj->base == NULL) {
1403                 ret = -ENOENT;
1404                 goto unlock;
1405         }
1406
1407         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1408                 ret = -E2BIG;
1409                 goto out;
1410         }
1411
1412         if (obj->madv != I915_MADV_WILLNEED) {
1413                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1414                 ret = -EINVAL;
1415                 goto out;
1416         }
1417
1418         ret = drm_gem_create_mmap_offset(&obj->base);
1419         if (ret != 0)
1420                 goto out;
1421
1422         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
1423             DRM_GEM_MAPPING_KEY;
1424 out:
1425         drm_gem_object_unreference(&obj->base);
1426 unlock:
1427         DRM_UNLOCK(dev);
1428         return (ret);
1429 }
1430
1431 int
1432 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1433     struct drm_file *file)
1434 {
1435         struct drm_i915_private *dev_priv;
1436         struct drm_i915_gem_mmap_gtt *args;
1437
1438         dev_priv = dev->dev_private;
1439         args = data;
1440
1441         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
1442 }
1443
1444 struct drm_i915_gem_object *
1445 i915_gem_alloc_object(struct drm_device *dev, size_t size)
1446 {
1447         struct drm_i915_private *dev_priv;
1448         struct drm_i915_gem_object *obj;
1449
1450         dev_priv = dev->dev_private;
1451
1452         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
1453
1454         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
1455                 drm_free(obj, DRM_I915_GEM);
1456                 return (NULL);
1457         }
1458
1459         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1460         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1461
1462         if (HAS_LLC(dev))
1463                 obj->cache_level = I915_CACHE_LLC;
1464         else
1465                 obj->cache_level = I915_CACHE_NONE;
1466         obj->base.driver_private = NULL;
1467         obj->fence_reg = I915_FENCE_REG_NONE;
1468         INIT_LIST_HEAD(&obj->mm_list);
1469         INIT_LIST_HEAD(&obj->gtt_list);
1470         INIT_LIST_HEAD(&obj->ring_list);
1471         INIT_LIST_HEAD(&obj->exec_list);
1472         INIT_LIST_HEAD(&obj->gpu_write_list);
1473         obj->madv = I915_MADV_WILLNEED;
1474         /* Avoid an unnecessary call to unbind on the first bind. */
1475         obj->map_and_fenceable = true;
1476
1477         i915_gem_info_add_obj(dev_priv, size);
1478
1479         return (obj);
1480 }
1481
1482 void
1483 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1484 {
1485
1486         /* If we don't have a page list set up, then we're not pinned
1487          * to GPU, and we can ignore the cache flush because it'll happen
1488          * again at bind time.
1489          */
1490         if (obj->pages == NULL)
1491                 return;
1492
1493         /* If the GPU is snooping the contents of the CPU cache,
1494          * we do not need to manually clear the CPU cache lines.  However,
1495          * the caches are only snooped when the render cache is
1496          * flushed/invalidated.  As we always have to emit invalidations
1497          * and flushes when moving into and out of the RENDER domain, correct
1498          * snooping behaviour occurs naturally as the result of our domain
1499          * tracking.
1500          */
1501         if (obj->cache_level != I915_CACHE_NONE)
1502                 return;
1503
1504         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
1505 }
1506
1507 static void
1508 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
1509 {
1510         uint32_t old_write_domain;
1511
1512         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
1513                 return;
1514
1515         i915_gem_clflush_object(obj);
1516         intel_gtt_chipset_flush();
1517         old_write_domain = obj->base.write_domain;
1518         obj->base.write_domain = 0;
1519 }
1520
1521 static int
1522 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
1523 {
1524
1525         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
1526                 return (0);
1527         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
1528 }
1529
1530 static void
1531 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
1532 {
1533         uint32_t old_write_domain;
1534
1535         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
1536                 return;
1537
1538         cpu_sfence();
1539
1540         old_write_domain = obj->base.write_domain;
1541         obj->base.write_domain = 0;
1542 }
1543
1544 int
1545 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
1546 {
1547         uint32_t old_write_domain, old_read_domains;
1548         int ret;
1549
1550         if (obj->gtt_space == NULL)
1551                 return (-EINVAL);
1552
1553         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
1554                 return 0;
1555
1556         ret = i915_gem_object_flush_gpu_write_domain(obj);
1557         if (ret != 0)
1558                 return (ret);
1559
1560         if (obj->pending_gpu_write || write) {
1561                 ret = i915_gem_object_wait_rendering(obj);
1562                 if (ret != 0)
1563                         return (ret);
1564         }
1565
1566         i915_gem_object_flush_cpu_write_domain(obj);
1567
1568         old_write_domain = obj->base.write_domain;
1569         old_read_domains = obj->base.read_domains;
1570
1571         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1572             ("In GTT write domain"));
1573         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1574         if (write) {
1575                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
1576                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
1577                 obj->dirty = 1;
1578         }
1579
1580         return (0);
1581 }
1582
1583 int
1584 i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
1585     enum i915_cache_level cache_level)
1586 {
1587         struct drm_device *dev;
1588         drm_i915_private_t *dev_priv;
1589         int ret;
1590
1591         if (obj->cache_level == cache_level)
1592                 return 0;
1593
1594         if (obj->pin_count) {
1595                 DRM_DEBUG("can not change the cache level of pinned objects\n");
1596                 return (-EBUSY);
1597         }
1598
1599         dev = obj->base.dev;
1600         dev_priv = dev->dev_private;
1601         if (obj->gtt_space) {
1602                 ret = i915_gem_object_finish_gpu(obj);
1603                 if (ret != 0)
1604                         return (ret);
1605
1606                 i915_gem_object_finish_gtt(obj);
1607
1608                 /* Before SandyBridge, you could not use tiling or fence
1609                  * registers with snooped memory, so relinquish any fences
1610                  * currently pointing to our region in the aperture.
1611                  */
1612                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
1613                         ret = i915_gem_object_put_fence(obj);
1614                         if (ret != 0)
1615                                 return (ret);
1616                 }
1617
1618                 i915_gem_gtt_rebind_object(obj, cache_level);
1619                 if (obj->has_aliasing_ppgtt_mapping)
1620                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
1621                             obj, cache_level);
1622         }
1623
1624         if (cache_level == I915_CACHE_NONE) {
1625                 u32 old_read_domains, old_write_domain;
1626
1627                 /* If we're coming from LLC cached, then we haven't
1628                  * actually been tracking whether the data is in the
1629                  * CPU cache or not, since we only allow one bit set
1630                  * in obj->write_domain and have been skipping the clflushes.
1631                  * Just set it to the CPU cache for now.
1632                  */
1633                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1634                     ("obj %p in CPU write domain", obj));
1635                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
1636                     ("obj %p in CPU read domain", obj));
1637
1638                 old_read_domains = obj->base.read_domains;
1639                 old_write_domain = obj->base.write_domain;
1640
1641                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1642                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1643
1644         }
1645
1646         obj->cache_level = cache_level;
1647         return (0);
1648 }
1649
1650 int
1651 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
1652     u32 alignment, struct intel_ring_buffer *pipelined)
1653 {
1654         u32 old_read_domains, old_write_domain;
1655         int ret;
1656
1657         ret = i915_gem_object_flush_gpu_write_domain(obj);
1658         if (ret != 0)
1659                 return (ret);
1660
1661         if (pipelined != obj->ring) {
1662                 ret = i915_gem_object_wait_rendering(obj);
1663                 if (ret == -ERESTART || ret == -EINTR)
1664                         return (ret);
1665         }
1666
1667         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
1668         if (ret != 0)
1669                 return (ret);
1670
1671         ret = i915_gem_object_pin(obj, alignment, true);
1672         if (ret != 0)
1673                 return (ret);
1674
1675         i915_gem_object_flush_cpu_write_domain(obj);
1676
1677         old_write_domain = obj->base.write_domain;
1678         old_read_domains = obj->base.read_domains;
1679
1680         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1681             ("obj %p in GTT write domain", obj));
1682         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1683
1684         return (0);
1685 }
1686
1687 int
1688 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
1689 {
1690         int ret;
1691
1692         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
1693                 return (0);
1694
1695         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1696                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
1697                 if (ret != 0)
1698                         return (ret);
1699         }
1700
1701         ret = i915_gem_object_wait_rendering(obj);
1702         if (ret != 0)
1703                 return (ret);
1704
1705         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1706
1707         return (0);
1708 }
1709
1710 static int
1711 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
1712 {
1713         uint32_t old_write_domain, old_read_domains;
1714         int ret;
1715
1716         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
1717                 return 0;
1718
1719         ret = i915_gem_object_flush_gpu_write_domain(obj);
1720         if (ret != 0)
1721                 return (ret);
1722
1723         ret = i915_gem_object_wait_rendering(obj);
1724         if (ret != 0)
1725                 return (ret);
1726
1727         i915_gem_object_flush_gtt_write_domain(obj);
1728         i915_gem_object_set_to_full_cpu_read_domain(obj);
1729
1730         old_write_domain = obj->base.write_domain;
1731         old_read_domains = obj->base.read_domains;
1732
1733         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
1734                 i915_gem_clflush_object(obj);
1735                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
1736         }
1737
1738         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1739             ("In cpu write domain"));
1740
1741         if (write) {
1742                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
1743                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1744         }
1745
1746         return (0);
1747 }
1748
1749 static void
1750 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
1751 {
1752         int i;
1753
1754         if (obj->page_cpu_valid == NULL)
1755                 return;
1756
1757         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) {
1758                 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
1759                         if (obj->page_cpu_valid[i] != 0)
1760                                 continue;
1761                         drm_clflush_pages(obj->pages + i, 1);
1762                 }
1763         }
1764
1765         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
1766         obj->page_cpu_valid = NULL;
1767 }
1768
1769 static int
1770 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
1771     uint64_t offset, uint64_t size)
1772 {
1773         uint32_t old_read_domains;
1774         int i, ret;
1775
1776         if (offset == 0 && size == obj->base.size)
1777                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
1778
1779         ret = i915_gem_object_flush_gpu_write_domain(obj);
1780         if (ret != 0)
1781                 return (ret);
1782         ret = i915_gem_object_wait_rendering(obj);
1783         if (ret != 0)
1784                 return (ret);
1785
1786         i915_gem_object_flush_gtt_write_domain(obj);
1787
1788         if (obj->page_cpu_valid == NULL &&
1789             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
1790                 return (0);
1791
1792         if (obj->page_cpu_valid == NULL) {
1793                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
1794                     DRM_I915_GEM, M_WAITOK | M_ZERO);
1795         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1796                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
1797
1798         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1799              i++) {
1800                 if (obj->page_cpu_valid[i])
1801                         continue;
1802                 drm_clflush_pages(obj->pages + i, 1);
1803                 obj->page_cpu_valid[i] = 1;
1804         }
1805
1806         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1807             ("In gpu write domain"));
1808
1809         old_read_domains = obj->base.read_domains;
1810         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
1811
1812         return (0);
1813 }
1814
1815 static uint32_t
1816 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1817 {
1818         uint32_t gtt_size;
1819
1820         if (INTEL_INFO(dev)->gen >= 4 ||
1821             tiling_mode == I915_TILING_NONE)
1822                 return (size);
1823
1824         /* Previous chips need a power-of-two fence region when tiling */
1825         if (INTEL_INFO(dev)->gen == 3)
1826                 gtt_size = 1024*1024;
1827         else
1828                 gtt_size = 512*1024;
1829
1830         while (gtt_size < size)
1831                 gtt_size <<= 1;
1832
1833         return (gtt_size);
1834 }
1835
1836 /**
1837  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1838  * @obj: object to check
1839  *
1840  * Return the required GTT alignment for an object, taking into account
1841  * potential fence register mapping.
1842  */
1843 static uint32_t
1844 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1845      int tiling_mode)
1846 {
1847
1848         /*
1849          * Minimum alignment is 4k (GTT page size), but might be greater
1850          * if a fence register is needed for the object.
1851          */
1852         if (INTEL_INFO(dev)->gen >= 4 ||
1853             tiling_mode == I915_TILING_NONE)
1854                 return (4096);
1855
1856         /*
1857          * Previous chips need to be aligned to the size of the smallest
1858          * fence register that can contain the object.
1859          */
1860         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
1861 }
1862
1863 uint32_t
1864 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size,
1865     int tiling_mode)
1866 {
1867
1868         if (tiling_mode == I915_TILING_NONE)
1869                 return (4096);
1870
1871         /*
1872          * Minimum alignment is 4k (GTT page size) for sane hw.
1873          */
1874         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
1875                 return (4096);
1876
1877         /*
1878          * Previous hardware however needs to be aligned to a power-of-two
1879          * tile height. The simplest method for determining this is to reuse
1880          * the power-of-tile object size.
1881          */
1882         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
1883 }
1884
1885 static int
1886 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1887     unsigned alignment, bool map_and_fenceable)
1888 {
1889         struct drm_device *dev;
1890         struct drm_i915_private *dev_priv;
1891         struct drm_mm_node *free_space;
1892         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1893         bool mappable, fenceable;
1894         int ret;
1895
1896         dev = obj->base.dev;
1897         dev_priv = dev->dev_private;
1898
1899         if (obj->madv != I915_MADV_WILLNEED) {
1900                 DRM_ERROR("Attempting to bind a purgeable object\n");
1901                 return (-EINVAL);
1902         }
1903
1904         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1905             obj->tiling_mode);
1906         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1907             obj->tiling_mode);
1908         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1909             obj->base.size, obj->tiling_mode);
1910         if (alignment == 0)
1911                 alignment = map_and_fenceable ? fence_alignment :
1912                     unfenced_alignment;
1913         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1914                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1915                 return (-EINVAL);
1916         }
1917
1918         size = map_and_fenceable ? fence_size : obj->base.size;
1919
1920         /* If the object is bigger than the entire aperture, reject it early
1921          * before evicting everything in a vain attempt to find space.
1922          */
1923         if (obj->base.size > (map_and_fenceable ?
1924             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1925                 DRM_ERROR(
1926 "Attempting to bind an object larger than the aperture\n");
1927                 return (-E2BIG);
1928         }
1929
1930  search_free:
1931         if (map_and_fenceable)
1932                 free_space = drm_mm_search_free_in_range(
1933                     &dev_priv->mm.gtt_space, size, alignment, 0,
1934                     dev_priv->mm.gtt_mappable_end, 0);
1935         else
1936                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1937                     size, alignment, 0);
1938         if (free_space != NULL) {
1939                 int color = 0;
1940                 if (map_and_fenceable)
1941                         obj->gtt_space = drm_mm_get_block_range_generic(
1942                             free_space, size, alignment, color, 0,
1943                             dev_priv->mm.gtt_mappable_end, 1);
1944                 else
1945                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1946                             size, alignment, color, 1);
1947         }
1948         if (obj->gtt_space == NULL) {
1949                 ret = i915_gem_evict_something(dev, size, alignment,
1950                     map_and_fenceable);
1951                 if (ret != 0)
1952                         return (ret);
1953                 goto search_free;
1954         }
1955         ret = i915_gem_object_get_pages_gtt(obj, 0);
1956         if (ret != 0) {
1957                 drm_mm_put_block(obj->gtt_space);
1958                 obj->gtt_space = NULL;
1959                 /*
1960                  * i915_gem_object_get_pages_gtt() cannot return
1961                  * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY)
1962                  * (which does not support operation without a flag
1963                  * anyway).
1964                  */
1965                 return (ret);
1966         }
1967
1968         ret = i915_gem_gtt_bind_object(obj);
1969         if (ret != 0) {
1970                 i915_gem_object_put_pages_gtt(obj);
1971                 drm_mm_put_block(obj->gtt_space);
1972                 obj->gtt_space = NULL;
1973                 if (i915_gem_evict_everything(dev, false))
1974                         return (ret);
1975                 goto search_free;
1976         }
1977
1978         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1979         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1980
1981         KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0,
1982             ("Object in gpu read domain"));
1983         KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0,
1984             ("Object in gpu write domain"));
1985
1986         obj->gtt_offset = obj->gtt_space->start;
1987
1988         fenceable =
1989                 obj->gtt_space->size == fence_size &&
1990                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1991
1992         mappable =
1993                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1994         obj->map_and_fenceable = mappable && fenceable;
1995
1996         return (0);
1997 }
1998
1999 static void
2000 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2001 {
2002         u32 old_write_domain, old_read_domains;
2003
2004         /* Act a barrier for all accesses through the GTT */
2005         cpu_mfence();
2006
2007         /* Force a pagefault for domain tracking on next user access */
2008         i915_gem_release_mmap(obj);
2009
2010         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2011                 return;
2012
2013         old_read_domains = obj->base.read_domains;
2014         old_write_domain = obj->base.write_domain;
2015
2016         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2017         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2018
2019 }
2020
2021 int
2022 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2023 {
2024         drm_i915_private_t *dev_priv;
2025         int ret;
2026
2027         dev_priv = obj->base.dev->dev_private;
2028         ret = 0;
2029         if (obj->gtt_space == NULL)
2030                 return (0);
2031         if (obj->pin_count != 0) {
2032                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2033                 return (-EINVAL);
2034         }
2035
2036         ret = i915_gem_object_finish_gpu(obj);
2037         if (ret == -ERESTART || ret == -EINTR)
2038                 return (ret);
2039
2040         i915_gem_object_finish_gtt(obj);
2041
2042         if (ret == 0)
2043                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2044         if (ret == -ERESTART || ret == -EINTR)
2045                 return (ret);
2046         if (ret != 0) {
2047                 i915_gem_clflush_object(obj);
2048                 obj->base.read_domains = obj->base.write_domain =
2049                     I915_GEM_DOMAIN_CPU;
2050         }
2051
2052         ret = i915_gem_object_put_fence(obj);
2053         if (ret == -ERESTART)
2054                 return (ret);
2055
2056         i915_gem_gtt_unbind_object(obj);
2057         if (obj->has_aliasing_ppgtt_mapping) {
2058                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2059                 obj->has_aliasing_ppgtt_mapping = 0;
2060         }
2061         i915_gem_object_put_pages_gtt(obj);
2062
2063         list_del_init(&obj->gtt_list);
2064         list_del_init(&obj->mm_list);
2065         obj->map_and_fenceable = true;
2066
2067         drm_mm_put_block(obj->gtt_space);
2068         obj->gtt_space = NULL;
2069         obj->gtt_offset = 0;
2070
2071         if (i915_gem_object_is_purgeable(obj))
2072                 i915_gem_object_truncate(obj);
2073
2074         return (ret);
2075 }
2076
2077 static int
2078 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
2079     int flags)
2080 {
2081         struct drm_device *dev;
2082         vm_object_t vm_obj;
2083         vm_page_t m;
2084         int page_count, i, j;
2085
2086         dev = obj->base.dev;
2087         KASSERT(obj->pages == NULL, ("Obj already has pages"));
2088         page_count = obj->base.size / PAGE_SIZE;
2089         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
2090             M_WAITOK);
2091         vm_obj = obj->base.vm_obj;
2092         VM_OBJECT_LOCK(vm_obj);
2093         for (i = 0; i < page_count; i++) {
2094                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
2095                         goto failed;
2096         }
2097         VM_OBJECT_UNLOCK(vm_obj);
2098         if (i915_gem_object_needs_bit17_swizzle(obj))
2099                 i915_gem_object_do_bit_17_swizzle(obj);
2100         return (0);
2101
2102 failed:
2103         for (j = 0; j < i; j++) {
2104                 m = obj->pages[j];
2105                 vm_page_busy_wait(m, FALSE, "i915gem");
2106                 vm_page_unwire(m, 0);
2107                 vm_page_wakeup(m);
2108                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2109         }
2110         VM_OBJECT_UNLOCK(vm_obj);
2111         drm_free(obj->pages, DRM_I915_GEM);
2112         obj->pages = NULL;
2113         return (-EIO);
2114 }
2115
2116 #define GEM_PARANOID_CHECK_GTT 0
2117 #if GEM_PARANOID_CHECK_GTT
2118 static void
2119 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
2120     int page_count)
2121 {
2122         struct drm_i915_private *dev_priv;
2123         vm_paddr_t pa;
2124         unsigned long start, end;
2125         u_int i;
2126         int j;
2127
2128         dev_priv = dev->dev_private;
2129         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
2130         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
2131         for (i = start; i < end; i++) {
2132                 pa = intel_gtt_read_pte_paddr(i);
2133                 for (j = 0; j < page_count; j++) {
2134                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
2135                                 panic("Page %p in GTT pte index %d pte %x",
2136                                     ma[i], i, intel_gtt_read_pte(i));
2137                         }
2138                 }
2139         }
2140 }
2141 #endif
2142
2143 static void
2144 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2145 {
2146         vm_page_t m;
2147         int page_count, i;
2148
2149         BUG_ON(obj->madv == __I915_MADV_PURGED);
2150
2151         if (obj->tiling_mode != I915_TILING_NONE)
2152                 i915_gem_object_save_bit_17_swizzle(obj);
2153         if (obj->madv == I915_MADV_DONTNEED)
2154                 obj->dirty = 0;
2155         page_count = obj->base.size / PAGE_SIZE;
2156         VM_OBJECT_LOCK(obj->base.vm_obj);
2157 #if GEM_PARANOID_CHECK_GTT
2158         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
2159 #endif
2160         for (i = 0; i < page_count; i++) {
2161                 m = obj->pages[i];
2162                 if (obj->dirty)
2163                         vm_page_dirty(m);
2164                 if (obj->madv == I915_MADV_WILLNEED)
2165                         vm_page_reference(m);
2166                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
2167                 vm_page_unwire(obj->pages[i], 1);
2168                 vm_page_wakeup(obj->pages[i]);
2169                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2170         }
2171         VM_OBJECT_UNLOCK(obj->base.vm_obj);
2172         obj->dirty = 0;
2173         drm_free(obj->pages, DRM_I915_GEM);
2174         obj->pages = NULL;
2175 }
2176
2177 void
2178 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2179 {
2180         vm_object_t devobj;
2181         vm_page_t m;
2182         int i, page_count;
2183
2184         if (!obj->fault_mappable)
2185                 return;
2186
2187         devobj = cdev_pager_lookup(obj);
2188         if (devobj != NULL) {
2189                 page_count = OFF_TO_IDX(obj->base.size);
2190
2191                 VM_OBJECT_LOCK(devobj);
2192                 for (i = 0; i < page_count; i++) {
2193                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
2194                         if (m == NULL)
2195                                 continue;
2196                         cdev_pager_free_page(devobj, m);
2197                 }
2198                 VM_OBJECT_UNLOCK(devobj);
2199                 vm_object_deallocate(devobj);
2200         }
2201
2202         obj->fault_mappable = false;
2203 }
2204
2205 int
2206 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
2207 {
2208         int ret;
2209
2210         KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0,
2211             ("In GPU write domain"));
2212
2213         if (obj->active) {
2214                 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno,
2215                     true);
2216                 if (ret != 0)
2217                         return (ret);
2218         }
2219         return (0);
2220 }
2221
2222 void
2223 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2224     struct intel_ring_buffer *ring, uint32_t seqno)
2225 {
2226         struct drm_device *dev = obj->base.dev;
2227         struct drm_i915_private *dev_priv = dev->dev_private;
2228         struct drm_i915_fence_reg *reg;
2229
2230         obj->ring = ring;
2231         KASSERT(ring != NULL, ("NULL ring"));
2232
2233         /* Add a reference if we're newly entering the active list. */
2234         if (!obj->active) {
2235                 drm_gem_object_reference(&obj->base);
2236                 obj->active = 1;
2237         }
2238
2239         /* Move from whatever list we were on to the tail of execution. */
2240         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
2241         list_move_tail(&obj->ring_list, &ring->active_list);
2242
2243         obj->last_rendering_seqno = seqno;
2244         if (obj->fenced_gpu_access) {
2245                 obj->last_fenced_seqno = seqno;
2246                 obj->last_fenced_ring = ring;
2247
2248                 /* Bump MRU to take account of the delayed flush */
2249                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2250                         reg = &dev_priv->fence_regs[obj->fence_reg];
2251                         list_move_tail(&reg->lru_list,
2252                                        &dev_priv->mm.fence_list);
2253                 }
2254         }
2255 }
2256
2257 static void
2258 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
2259 {
2260         list_del_init(&obj->ring_list);
2261         obj->last_rendering_seqno = 0;
2262         obj->last_fenced_seqno = 0;
2263 }
2264
2265 static void
2266 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
2267 {
2268         struct drm_device *dev = obj->base.dev;
2269         drm_i915_private_t *dev_priv = dev->dev_private;
2270
2271         KASSERT(obj->active, ("Object not active"));
2272         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
2273
2274         i915_gem_object_move_off_active(obj);
2275 }
2276
2277 static void
2278 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2279 {
2280         struct drm_device *dev = obj->base.dev;
2281         struct drm_i915_private *dev_priv = dev->dev_private;
2282
2283         if (obj->pin_count != 0)
2284                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2285         else
2286                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2287
2288         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
2289         KASSERT(obj->active, ("Object not active"));
2290         obj->ring = NULL;
2291         obj->last_fenced_ring = NULL;
2292
2293         i915_gem_object_move_off_active(obj);
2294         obj->fenced_gpu_access = false;
2295
2296         obj->active = 0;
2297         obj->pending_gpu_write = false;
2298         drm_gem_object_unreference(&obj->base);
2299
2300 #if 1
2301         KIB_NOTYET();
2302 #else
2303         WARN_ON(i915_verify_lists(dev));
2304 #endif
2305 }
2306
2307 /* Immediately discard the backing storage */
2308 static void
2309 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2310 {
2311         vm_object_t vm_obj;
2312
2313         vm_obj = obj->base.vm_obj;
2314         VM_OBJECT_LOCK(vm_obj);
2315         vm_object_page_remove(vm_obj, 0, 0, false);
2316         VM_OBJECT_UNLOCK(vm_obj);
2317         obj->madv = __I915_MADV_PURGED;
2318 }
2319
2320 static inline int
2321 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
2322 {
2323
2324         return (obj->madv == I915_MADV_DONTNEED);
2325 }
2326
2327 static void
2328 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
2329     uint32_t flush_domains)
2330 {
2331         struct drm_i915_gem_object *obj, *next;
2332         uint32_t old_write_domain;
2333
2334         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
2335             gpu_write_list) {
2336                 if (obj->base.write_domain & flush_domains) {
2337                         old_write_domain = obj->base.write_domain;
2338                         obj->base.write_domain = 0;
2339                         list_del_init(&obj->gpu_write_list);
2340                         i915_gem_object_move_to_active(obj, ring,
2341                             i915_gem_next_request_seqno(ring));
2342                 }
2343         }
2344 }
2345
2346 static int
2347 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
2348 {
2349         drm_i915_private_t *dev_priv;
2350
2351         dev_priv = obj->base.dev->dev_private;
2352         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
2353             obj->tiling_mode != I915_TILING_NONE);
2354 }
2355
2356 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
2357
2358 static vm_page_t
2359 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
2360 {
2361         vm_page_t m;
2362         int rv;
2363
2364         VM_OBJECT_LOCK_ASSERT_OWNED(object);
2365         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
2366         if (m->valid != VM_PAGE_BITS_ALL) {
2367                 if (vm_pager_has_page(object, pindex)) {
2368                         rv = vm_pager_get_page(object, &m, 1);
2369                         m = vm_page_lookup(object, pindex);
2370                         if (m == NULL)
2371                                 return (NULL);
2372                         if (rv != VM_PAGER_OK) {
2373                                 vm_page_free(m);
2374                                 return (NULL);
2375                         }
2376                 } else {
2377                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
2378                         m->valid = VM_PAGE_BITS_ALL;
2379                         m->dirty = 0;
2380                 }
2381         }
2382         vm_page_wire(m);
2383         vm_page_wakeup(m);
2384         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
2385         return (m);
2386 }
2387
2388 int
2389 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
2390     uint32_t flush_domains)
2391 {
2392         int ret;
2393
2394         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2395                 return 0;
2396
2397         ret = ring->flush(ring, invalidate_domains, flush_domains);
2398         if (ret)
2399                 return ret;
2400
2401         if (flush_domains & I915_GEM_GPU_DOMAINS)
2402                 i915_gem_process_flushing_list(ring, flush_domains);
2403         return 0;
2404 }
2405
2406 static int
2407 i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire)
2408 {
2409         int ret;
2410
2411         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2412                 return 0;
2413
2414         if (!list_empty(&ring->gpu_write_list)) {
2415                 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS,
2416                     I915_GEM_GPU_DOMAINS);
2417                 if (ret != 0)
2418                         return ret;
2419         }
2420
2421         return (i915_wait_request(ring, i915_gem_next_request_seqno(ring),
2422             do_retire));
2423 }
2424
2425 int
2426 i915_gpu_idle(struct drm_device *dev, bool do_retire)
2427 {
2428         drm_i915_private_t *dev_priv = dev->dev_private;
2429         int ret, i;
2430
2431         /* Flush everything onto the inactive list. */
2432         for (i = 0; i < I915_NUM_RINGS; i++) {
2433                 ret = i915_ring_idle(&dev_priv->ring[i], do_retire);
2434                 if (ret)
2435                         return ret;
2436         }
2437
2438         return 0;
2439 }
2440
2441 int
2442 i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire)
2443 {
2444         drm_i915_private_t *dev_priv;
2445         struct drm_i915_gem_request *request;
2446         uint32_t ier;
2447         int flags, ret;
2448         bool recovery_complete;
2449
2450         KASSERT(seqno != 0, ("Zero seqno"));
2451
2452         dev_priv = ring->dev->dev_private;
2453         ret = 0;
2454
2455         if (atomic_read(&dev_priv->mm.wedged) != 0) {
2456                 /* Give the error handler a chance to run. */
2457                 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
2458                 recovery_complete = (&dev_priv->error_completion) > 0;
2459                 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
2460                 return (recovery_complete ? -EIO : -EAGAIN);
2461         }
2462
2463         if (seqno == ring->outstanding_lazy_request) {
2464                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
2465                     M_WAITOK | M_ZERO);
2466                 if (request == NULL)
2467                         return (-ENOMEM);
2468
2469                 ret = i915_add_request(ring, NULL, request);
2470                 if (ret != 0) {
2471                         drm_free(request, DRM_I915_GEM);
2472                         return (ret);
2473                 }
2474
2475                 seqno = request->seqno;
2476         }
2477
2478         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2479                 if (HAS_PCH_SPLIT(ring->dev))
2480                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2481                 else
2482                         ier = I915_READ(IER);
2483                 if (!ier) {
2484                         DRM_ERROR("something (likely vbetool) disabled "
2485                                   "interrupts, re-enabling\n");
2486                         ring->dev->driver->irq_preinstall(ring->dev);
2487                         ring->dev->driver->irq_postinstall(ring->dev);
2488                 }
2489
2490                 ring->waiting_seqno = seqno;
2491                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2492                 if (ring->irq_get(ring)) {
2493                         flags = dev_priv->mm.interruptible ? PCATCH : 0;
2494                         while (!i915_seqno_passed(ring->get_seqno(ring), seqno)
2495                             && !atomic_read(&dev_priv->mm.wedged) &&
2496                             ret == 0) {
2497                                 ret = -lksleep(ring, &ring->irq_lock, flags,
2498                                     "915gwr", 0);
2499                         }
2500                         ring->irq_put(ring);
2501                         lockmgr(&ring->irq_lock, LK_RELEASE);
2502                 } else {
2503                         lockmgr(&ring->irq_lock, LK_RELEASE);
2504                         if (_intel_wait_for(ring->dev,
2505                             i915_seqno_passed(ring->get_seqno(ring), seqno) ||
2506                             atomic_read(&dev_priv->mm.wedged), 3000,
2507                             0, "i915wrq") != 0)
2508                                 ret = -EBUSY;
2509                 }
2510                 ring->waiting_seqno = 0;
2511
2512         }
2513         if (atomic_read(&dev_priv->mm.wedged))
2514                 ret = -EAGAIN;
2515
2516         /* Directly dispatch request retiring.  While we have the work queue
2517          * to handle this, the waiter on a request often wants an associated
2518          * buffer to have made it to the inactive list, and we would need
2519          * a separate wait queue to handle that.
2520          */
2521         if (ret == 0 && do_retire)
2522                 i915_gem_retire_requests_ring(ring);
2523
2524         return (ret);
2525 }
2526
2527 static u32
2528 i915_gem_get_seqno(struct drm_device *dev)
2529 {
2530         drm_i915_private_t *dev_priv = dev->dev_private;
2531         u32 seqno = dev_priv->next_seqno;
2532
2533         /* reserve 0 for non-seqno */
2534         if (++dev_priv->next_seqno == 0)
2535                 dev_priv->next_seqno = 1;
2536
2537         return seqno;
2538 }
2539
2540 u32
2541 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
2542 {
2543         if (ring->outstanding_lazy_request == 0)
2544                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
2545
2546         return ring->outstanding_lazy_request;
2547 }
2548
2549 int
2550 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
2551      struct drm_i915_gem_request *request)
2552 {
2553         drm_i915_private_t *dev_priv;
2554         struct drm_i915_file_private *file_priv;
2555         uint32_t seqno;
2556         u32 request_ring_position;
2557         int was_empty;
2558         int ret;
2559
2560         KASSERT(request != NULL, ("NULL request in add"));
2561         DRM_LOCK_ASSERT(ring->dev);
2562         dev_priv = ring->dev->dev_private;
2563
2564         seqno = i915_gem_next_request_seqno(ring);
2565         request_ring_position = intel_ring_get_tail(ring);
2566
2567         ret = ring->add_request(ring, &seqno);
2568         if (ret != 0)
2569             return ret;
2570
2571         request->seqno = seqno;
2572         request->ring = ring;
2573         request->tail = request_ring_position;
2574         request->emitted_jiffies = ticks;
2575         was_empty = list_empty(&ring->request_list);
2576         list_add_tail(&request->list, &ring->request_list);
2577
2578         if (file != NULL) {
2579                 file_priv = file->driver_priv;
2580
2581                 spin_lock(&file_priv->mm.lock);
2582                 request->file_priv = file_priv;
2583                 list_add_tail(&request->client_list,
2584                     &file_priv->mm.request_list);
2585                 spin_unlock(&file_priv->mm.lock);
2586         }
2587
2588         ring->outstanding_lazy_request = 0;
2589
2590         if (!dev_priv->mm.suspended) {
2591                 if (i915_enable_hangcheck) {
2592                         mod_timer(&dev_priv->hangcheck_timer,
2593                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2594                 }
2595                 if (was_empty) {
2596                         queue_delayed_work(dev_priv->wq,
2597                                            &dev_priv->mm.retire_work,
2598                                            round_jiffies_up_relative(hz));
2599                         intel_mark_busy(dev_priv->dev);
2600                 }
2601         }
2602         return (0);
2603 }
2604
2605 static inline void
2606 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2607 {
2608         struct drm_i915_file_private *file_priv = request->file_priv;
2609
2610         if (!file_priv)
2611                 return;
2612
2613         DRM_LOCK_ASSERT(request->ring->dev);
2614
2615         spin_lock(&file_priv->mm.lock);
2616         if (request->file_priv != NULL) {
2617                 list_del(&request->client_list);
2618                 request->file_priv = NULL;
2619         }
2620         spin_unlock(&file_priv->mm.lock);
2621 }
2622
2623 void
2624 i915_gem_release(struct drm_device *dev, struct drm_file *file)
2625 {
2626         struct drm_i915_file_private *file_priv;
2627         struct drm_i915_gem_request *request;
2628
2629         file_priv = file->driver_priv;
2630
2631         /* Clean up our request list when the client is going away, so that
2632          * later retire_requests won't dereference our soon-to-be-gone
2633          * file_priv.
2634          */
2635         spin_lock(&file_priv->mm.lock);
2636         while (!list_empty(&file_priv->mm.request_list)) {
2637                 request = list_first_entry(&file_priv->mm.request_list,
2638                                            struct drm_i915_gem_request,
2639                                            client_list);
2640                 list_del(&request->client_list);
2641                 request->file_priv = NULL;
2642         }
2643         spin_unlock(&file_priv->mm.lock);
2644 }
2645
2646 static void
2647 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2648     struct intel_ring_buffer *ring)
2649 {
2650
2651         if (ring->dev != NULL)
2652                 DRM_LOCK_ASSERT(ring->dev);
2653
2654         while (!list_empty(&ring->request_list)) {
2655                 struct drm_i915_gem_request *request;
2656
2657                 request = list_first_entry(&ring->request_list,
2658                     struct drm_i915_gem_request, list);
2659
2660                 list_del(&request->list);
2661                 i915_gem_request_remove_from_client(request);
2662                 drm_free(request, DRM_I915_GEM);
2663         }
2664
2665         while (!list_empty(&ring->active_list)) {
2666                 struct drm_i915_gem_object *obj;
2667
2668                 obj = list_first_entry(&ring->active_list,
2669                     struct drm_i915_gem_object, ring_list);
2670
2671                 obj->base.write_domain = 0;
2672                 list_del_init(&obj->gpu_write_list);
2673                 i915_gem_object_move_to_inactive(obj);
2674         }
2675 }
2676
2677 static void
2678 i915_gem_reset_fences(struct drm_device *dev)
2679 {
2680         struct drm_i915_private *dev_priv = dev->dev_private;
2681         int i;
2682
2683         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2684                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2685                 struct drm_i915_gem_object *obj = reg->obj;
2686
2687                 if (!obj)
2688                         continue;
2689
2690                 if (obj->tiling_mode)
2691                         i915_gem_release_mmap(obj);
2692
2693                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
2694                 reg->obj->fenced_gpu_access = false;
2695                 reg->obj->last_fenced_seqno = 0;
2696                 reg->obj->last_fenced_ring = NULL;
2697                 i915_gem_clear_fence_reg(dev, reg);
2698         }
2699 }
2700
2701 void
2702 i915_gem_reset(struct drm_device *dev)
2703 {
2704         struct drm_i915_private *dev_priv = dev->dev_private;
2705         struct drm_i915_gem_object *obj;
2706         int i;
2707
2708         for (i = 0; i < I915_NUM_RINGS; i++)
2709                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
2710
2711         /* Remove anything from the flushing lists. The GPU cache is likely
2712          * to be lost on reset along with the data, so simply move the
2713          * lost bo to the inactive list.
2714          */
2715         while (!list_empty(&dev_priv->mm.flushing_list)) {
2716                 obj = list_first_entry(&dev_priv->mm.flushing_list,
2717                                       struct drm_i915_gem_object,
2718                                       mm_list);
2719
2720                 obj->base.write_domain = 0;
2721                 list_del_init(&obj->gpu_write_list);
2722                 i915_gem_object_move_to_inactive(obj);
2723         }
2724
2725         /* Move everything out of the GPU domains to ensure we do any
2726          * necessary invalidation upon reuse.
2727          */
2728         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
2729                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2730         }
2731
2732         /* The fence registers are invalidated so clear them out */
2733         i915_gem_reset_fences(dev);
2734 }
2735
2736 /**
2737  * This function clears the request list as sequence numbers are passed.
2738  */
2739 void
2740 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2741 {
2742         uint32_t seqno;
2743         int i;
2744
2745         if (list_empty(&ring->request_list))
2746                 return;
2747
2748         seqno = ring->get_seqno(ring);
2749         for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++)
2750                 if (seqno >= ring->sync_seqno[i])
2751                         ring->sync_seqno[i] = 0;
2752
2753         while (!list_empty(&ring->request_list)) {
2754                 struct drm_i915_gem_request *request;
2755
2756                 request = list_first_entry(&ring->request_list,
2757                                            struct drm_i915_gem_request,
2758                                            list);
2759
2760                 if (!i915_seqno_passed(seqno, request->seqno))
2761                         break;
2762
2763                 ring->last_retired_head = request->tail;
2764
2765                 list_del(&request->list);
2766                 i915_gem_request_remove_from_client(request);
2767                 drm_free(request, DRM_I915_GEM);
2768         }
2769
2770         /* Move any buffers on the active list that are no longer referenced
2771          * by the ringbuffer to the flushing/inactive lists as appropriate.
2772          */
2773         while (!list_empty(&ring->active_list)) {
2774                 struct drm_i915_gem_object *obj;
2775
2776                 obj = list_first_entry(&ring->active_list,
2777                                       struct drm_i915_gem_object,
2778                                       ring_list);
2779
2780                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
2781                         break;
2782
2783                 if (obj->base.write_domain != 0)
2784                         i915_gem_object_move_to_flushing(obj);
2785                 else
2786                         i915_gem_object_move_to_inactive(obj);
2787         }
2788
2789         if (ring->trace_irq_seqno &&
2790             i915_seqno_passed(seqno, ring->trace_irq_seqno)) {
2791                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2792                 ring->irq_put(ring);
2793                 lockmgr(&ring->irq_lock, LK_RELEASE);
2794                 ring->trace_irq_seqno = 0;
2795         }
2796 }
2797
2798 void
2799 i915_gem_retire_requests(struct drm_device *dev)
2800 {
2801         drm_i915_private_t *dev_priv = dev->dev_private;
2802         struct drm_i915_gem_object *obj, *next;
2803         int i;
2804
2805         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2806                 list_for_each_entry_safe(obj, next,
2807                     &dev_priv->mm.deferred_free_list, mm_list)
2808                         i915_gem_free_object_tail(obj);
2809         }
2810
2811         for (i = 0; i < I915_NUM_RINGS; i++)
2812                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
2813 }
2814
2815 static int
2816 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2817     struct intel_ring_buffer *pipelined)
2818 {
2819         struct drm_device *dev = obj->base.dev;
2820         drm_i915_private_t *dev_priv = dev->dev_private;
2821         u32 size = obj->gtt_space->size;
2822         int regnum = obj->fence_reg;
2823         uint64_t val;
2824
2825         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2826                          0xfffff000) << 32;
2827         val |= obj->gtt_offset & 0xfffff000;
2828         val |= (uint64_t)((obj->stride / 128) - 1) <<
2829                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2830
2831         if (obj->tiling_mode == I915_TILING_Y)
2832                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2833         val |= I965_FENCE_REG_VALID;
2834
2835         if (pipelined) {
2836                 int ret = intel_ring_begin(pipelined, 6);
2837                 if (ret)
2838                         return ret;
2839
2840                 intel_ring_emit(pipelined, MI_NOOP);
2841                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2842                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2843                 intel_ring_emit(pipelined, (u32)val);
2844                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2845                 intel_ring_emit(pipelined, (u32)(val >> 32));
2846                 intel_ring_advance(pipelined);
2847         } else
2848                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2849
2850         return 0;
2851 }
2852
2853 static int
2854 i965_write_fence_reg(struct drm_i915_gem_object *obj,
2855     struct intel_ring_buffer *pipelined)
2856 {
2857         struct drm_device *dev = obj->base.dev;
2858         drm_i915_private_t *dev_priv = dev->dev_private;
2859         u32 size = obj->gtt_space->size;
2860         int regnum = obj->fence_reg;
2861         uint64_t val;
2862
2863         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2864                     0xfffff000) << 32;
2865         val |= obj->gtt_offset & 0xfffff000;
2866         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2867         if (obj->tiling_mode == I915_TILING_Y)
2868                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2869         val |= I965_FENCE_REG_VALID;
2870
2871         if (pipelined) {
2872                 int ret = intel_ring_begin(pipelined, 6);
2873                 if (ret)
2874                         return ret;
2875
2876                 intel_ring_emit(pipelined, MI_NOOP);
2877                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2878                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2879                 intel_ring_emit(pipelined, (u32)val);
2880                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2881                 intel_ring_emit(pipelined, (u32)(val >> 32));
2882                 intel_ring_advance(pipelined);
2883         } else
2884                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2885
2886         return 0;
2887 }
2888
2889 static int
2890 i915_write_fence_reg(struct drm_i915_gem_object *obj,
2891     struct intel_ring_buffer *pipelined)
2892 {
2893         struct drm_device *dev = obj->base.dev;
2894         drm_i915_private_t *dev_priv = dev->dev_private;
2895         u32 size = obj->gtt_space->size;
2896         u32 fence_reg, val, pitch_val;
2897         int tile_width;
2898
2899         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2900             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
2901                 kprintf(
2902 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2903                  obj->gtt_offset, obj->map_and_fenceable, size);
2904                 return -EINVAL;
2905         }
2906
2907         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2908                 tile_width = 128;
2909         else
2910                 tile_width = 512;
2911
2912         /* Note: pitch better be a power of two tile widths */
2913         pitch_val = obj->stride / tile_width;
2914         pitch_val = ffs(pitch_val) - 1;
2915
2916         val = obj->gtt_offset;
2917         if (obj->tiling_mode == I915_TILING_Y)
2918                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2919         val |= I915_FENCE_SIZE_BITS(size);
2920         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2921         val |= I830_FENCE_REG_VALID;
2922
2923         fence_reg = obj->fence_reg;
2924         if (fence_reg < 8)
2925                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2926         else
2927                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2928
2929         if (pipelined) {
2930                 int ret = intel_ring_begin(pipelined, 4);
2931                 if (ret)
2932                         return ret;
2933
2934                 intel_ring_emit(pipelined, MI_NOOP);
2935                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2936                 intel_ring_emit(pipelined, fence_reg);
2937                 intel_ring_emit(pipelined, val);
2938                 intel_ring_advance(pipelined);
2939         } else
2940                 I915_WRITE(fence_reg, val);
2941
2942         return 0;
2943 }
2944
2945 static int
2946 i830_write_fence_reg(struct drm_i915_gem_object *obj,
2947     struct intel_ring_buffer *pipelined)
2948 {
2949         struct drm_device *dev = obj->base.dev;
2950         drm_i915_private_t *dev_priv = dev->dev_private;
2951         u32 size = obj->gtt_space->size;
2952         int regnum = obj->fence_reg;
2953         uint32_t val;
2954         uint32_t pitch_val;
2955
2956         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2957             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
2958                 kprintf(
2959 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2960                     obj->gtt_offset, size);
2961                 return -EINVAL;
2962         }
2963
2964         pitch_val = obj->stride / 128;
2965         pitch_val = ffs(pitch_val) - 1;
2966
2967         val = obj->gtt_offset;
2968         if (obj->tiling_mode == I915_TILING_Y)
2969                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2970         val |= I830_FENCE_SIZE_BITS(size);
2971         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2972         val |= I830_FENCE_REG_VALID;
2973
2974         if (pipelined) {
2975                 int ret = intel_ring_begin(pipelined, 4);
2976                 if (ret)
2977                         return ret;
2978
2979                 intel_ring_emit(pipelined, MI_NOOP);
2980                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2981                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2982                 intel_ring_emit(pipelined, val);
2983                 intel_ring_advance(pipelined);
2984         } else
2985                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2986
2987         return 0;
2988 }
2989
2990 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2991 {
2992         return i915_seqno_passed(ring->get_seqno(ring), seqno);
2993 }
2994
2995 static int
2996 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2997     struct intel_ring_buffer *pipelined)
2998 {
2999         int ret;
3000
3001         if (obj->fenced_gpu_access) {
3002                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3003                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
3004                             obj->base.write_domain);
3005                         if (ret)
3006                                 return ret;
3007                 }
3008
3009                 obj->fenced_gpu_access = false;
3010         }
3011
3012         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
3013                 if (!ring_passed_seqno(obj->last_fenced_ring,
3014                                        obj->last_fenced_seqno)) {
3015                         ret = i915_wait_request(obj->last_fenced_ring,
3016                                                 obj->last_fenced_seqno,
3017                                                 true);
3018                         if (ret)
3019                                 return ret;
3020                 }
3021
3022                 obj->last_fenced_seqno = 0;
3023                 obj->last_fenced_ring = NULL;
3024         }
3025
3026         /* Ensure that all CPU reads are completed before installing a fence
3027          * and all writes before removing the fence.
3028          */
3029         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
3030                 cpu_mfence();
3031
3032         return 0;
3033 }
3034
3035 int
3036 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3037 {
3038         int ret;
3039
3040         if (obj->tiling_mode)
3041                 i915_gem_release_mmap(obj);
3042
3043         ret = i915_gem_object_flush_fence(obj, NULL);
3044         if (ret)
3045                 return ret;
3046
3047         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3048                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3049
3050                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
3051                         kprintf("%s: pin_count %d\n", __func__,
3052                             dev_priv->fence_regs[obj->fence_reg].pin_count);
3053                 i915_gem_clear_fence_reg(obj->base.dev,
3054                                          &dev_priv->fence_regs[obj->fence_reg]);
3055
3056                 obj->fence_reg = I915_FENCE_REG_NONE;
3057         }
3058
3059         return 0;
3060 }
3061
3062 static struct drm_i915_fence_reg *
3063 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
3064 {
3065         struct drm_i915_private *dev_priv = dev->dev_private;
3066         struct drm_i915_fence_reg *reg, *first, *avail;
3067         int i;
3068
3069         /* First try to find a free reg */
3070         avail = NULL;
3071         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3072                 reg = &dev_priv->fence_regs[i];
3073                 if (!reg->obj)
3074                         return reg;
3075
3076                 if (!reg->pin_count)
3077                         avail = reg;
3078         }
3079
3080         if (avail == NULL)
3081                 return NULL;
3082
3083         /* None available, try to steal one or wait for a user to finish */
3084         avail = first = NULL;
3085         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3086                 if (reg->pin_count)
3087                         continue;
3088
3089                 if (first == NULL)
3090                         first = reg;
3091
3092                 if (!pipelined ||
3093                     !reg->obj->last_fenced_ring ||
3094                     reg->obj->last_fenced_ring == pipelined) {
3095                         avail = reg;
3096                         break;
3097                 }
3098         }
3099
3100         if (avail == NULL)
3101                 avail = first;
3102
3103         return avail;
3104 }
3105
3106 int
3107 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
3108     struct intel_ring_buffer *pipelined)
3109 {
3110         struct drm_device *dev = obj->base.dev;
3111         struct drm_i915_private *dev_priv = dev->dev_private;
3112         struct drm_i915_fence_reg *reg;
3113         int ret;
3114
3115         pipelined = NULL;
3116         ret = 0;
3117
3118         if (obj->fence_reg != I915_FENCE_REG_NONE) {
3119                 reg = &dev_priv->fence_regs[obj->fence_reg];
3120                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
3121
3122                 if (obj->tiling_changed) {
3123                         ret = i915_gem_object_flush_fence(obj, pipelined);
3124                         if (ret)
3125                                 return ret;
3126
3127                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3128                                 pipelined = NULL;
3129
3130                         if (pipelined) {
3131                                 reg->setup_seqno =
3132                                         i915_gem_next_request_seqno(pipelined);
3133                                 obj->last_fenced_seqno = reg->setup_seqno;
3134                                 obj->last_fenced_ring = pipelined;
3135                         }
3136
3137                         goto update;
3138                 }
3139
3140                 if (!pipelined) {
3141                         if (reg->setup_seqno) {
3142                                 if (!ring_passed_seqno(obj->last_fenced_ring,
3143                                     reg->setup_seqno)) {
3144                                         ret = i915_wait_request(
3145                                             obj->last_fenced_ring,
3146                                             reg->setup_seqno,
3147                                             true);
3148                                         if (ret)
3149                                                 return ret;
3150                                 }
3151
3152                                 reg->setup_seqno = 0;
3153                         }
3154                 } else if (obj->last_fenced_ring &&
3155                            obj->last_fenced_ring != pipelined) {
3156                         ret = i915_gem_object_flush_fence(obj, pipelined);
3157                         if (ret)
3158                                 return ret;
3159                 }
3160
3161                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3162                         pipelined = NULL;
3163                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
3164
3165                 if (obj->tiling_changed) {
3166                         if (pipelined) {
3167                                 reg->setup_seqno =
3168                                         i915_gem_next_request_seqno(pipelined);
3169                                 obj->last_fenced_seqno = reg->setup_seqno;
3170                                 obj->last_fenced_ring = pipelined;
3171                         }
3172                         goto update;
3173                 }
3174
3175                 return 0;
3176         }
3177
3178         reg = i915_find_fence_reg(dev, pipelined);
3179         if (reg == NULL)
3180                 return -EDEADLK;
3181
3182         ret = i915_gem_object_flush_fence(obj, pipelined);
3183         if (ret)
3184                 return ret;
3185
3186         if (reg->obj) {
3187                 struct drm_i915_gem_object *old = reg->obj;
3188
3189                 drm_gem_object_reference(&old->base);
3190
3191                 if (old->tiling_mode)
3192                         i915_gem_release_mmap(old);
3193
3194                 ret = i915_gem_object_flush_fence(old, pipelined);
3195                 if (ret) {
3196                         drm_gem_object_unreference(&old->base);
3197                         return ret;
3198                 }
3199
3200                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
3201                         pipelined = NULL;
3202
3203                 old->fence_reg = I915_FENCE_REG_NONE;
3204                 old->last_fenced_ring = pipelined;
3205                 old->last_fenced_seqno =
3206                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3207
3208                 drm_gem_object_unreference(&old->base);
3209         } else if (obj->last_fenced_seqno == 0)
3210                 pipelined = NULL;
3211
3212         reg->obj = obj;
3213         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
3214         obj->fence_reg = reg - dev_priv->fence_regs;
3215         obj->last_fenced_ring = pipelined;
3216
3217         reg->setup_seqno =
3218                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3219         obj->last_fenced_seqno = reg->setup_seqno;
3220
3221 update:
3222         obj->tiling_changed = false;
3223         switch (INTEL_INFO(dev)->gen) {
3224         case 7:
3225         case 6:
3226                 ret = sandybridge_write_fence_reg(obj, pipelined);
3227                 break;
3228         case 5:
3229         case 4:
3230                 ret = i965_write_fence_reg(obj, pipelined);
3231                 break;
3232         case 3:
3233                 ret = i915_write_fence_reg(obj, pipelined);
3234                 break;
3235         case 2:
3236                 ret = i830_write_fence_reg(obj, pipelined);
3237                 break;
3238         }
3239
3240         return ret;
3241 }
3242
3243 static void
3244 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3245 {
3246         drm_i915_private_t *dev_priv = dev->dev_private;
3247         uint32_t fence_reg = reg - dev_priv->fence_regs;
3248
3249         switch (INTEL_INFO(dev)->gen) {
3250         case 7:
3251         case 6:
3252                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3253                 break;
3254         case 5:
3255         case 4:
3256                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3257                 break;
3258         case 3:
3259                 if (fence_reg >= 8)
3260                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3261                 else
3262         case 2:
3263                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3264
3265                 I915_WRITE(fence_reg, 0);
3266                 break;
3267         }
3268
3269         list_del_init(&reg->lru_list);
3270         reg->obj = NULL;
3271         reg->setup_seqno = 0;
3272         reg->pin_count = 0;
3273 }
3274
3275 int
3276 i915_gem_init_object(struct drm_gem_object *obj)
3277 {
3278
3279         kprintf("i915_gem_init_object called\n");
3280         return (0);
3281 }
3282
3283 static bool
3284 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
3285 {
3286
3287         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
3288 }
3289
3290 static void
3291 i915_gem_retire_work_handler(struct work_struct *work)
3292 {
3293         drm_i915_private_t *dev_priv;
3294         struct drm_device *dev;
3295         struct intel_ring_buffer *ring;
3296         bool idle;
3297         int i;
3298
3299         dev_priv = container_of(work, drm_i915_private_t,
3300                                 mm.retire_work.work);
3301         dev = dev_priv->dev;
3302
3303         /* Come back later if the device is busy... */
3304         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
3305                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3306                                    round_jiffies_up_relative(hz));
3307                 return;
3308         }
3309
3310         i915_gem_retire_requests(dev);
3311
3312         /* Send a periodic flush down the ring so we don't hold onto GEM
3313          * objects indefinitely.
3314          */
3315         idle = true;
3316         for (i = 0; i < I915_NUM_RINGS; i++) {
3317                 ring = &dev_priv->ring[i];
3318
3319                 if (!list_empty(&ring->gpu_write_list)) {
3320                         struct drm_i915_gem_request *request;
3321                         int ret;
3322
3323                         ret = i915_gem_flush_ring(ring,
3324                                                   0, I915_GEM_GPU_DOMAINS);
3325                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
3326                             M_WAITOK | M_ZERO);
3327                         if (ret || request == NULL ||
3328                             i915_add_request(ring, NULL, request))
3329                                 drm_free(request, DRM_I915_GEM);
3330                 }
3331
3332                 idle &= list_empty(&ring->request_list);
3333         }
3334
3335         if (!dev_priv->mm.suspended && !idle)
3336                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3337                                    round_jiffies_up_relative(hz));
3338
3339         DRM_UNLOCK(dev);
3340 }
3341
3342 void
3343 i915_gem_lastclose(struct drm_device *dev)
3344 {
3345         int ret;
3346
3347         if (drm_core_check_feature(dev, DRIVER_MODESET))
3348                 return;
3349
3350         ret = i915_gem_idle(dev);
3351         if (ret != 0)
3352                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3353 }
3354
3355 static int
3356 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
3357 {
3358         drm_i915_private_t *dev_priv;
3359         struct drm_i915_gem_phys_object *phys_obj;
3360         int ret;
3361
3362         dev_priv = dev->dev_private;
3363         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
3364                 return (0);
3365
3366         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3367             M_WAITOK | M_ZERO);
3368
3369         phys_obj->id = id;
3370
3371         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3372         if (phys_obj->handle == NULL) {
3373                 ret = -ENOMEM;
3374                 goto free_obj;
3375         }
3376         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3377             size / PAGE_SIZE, PAT_WRITE_COMBINING);
3378
3379         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3380
3381         return (0);
3382
3383 free_obj:
3384         drm_free(phys_obj, DRM_I915_GEM);
3385         return (ret);
3386 }
3387
3388 static void
3389 i915_gem_free_phys_object(struct drm_device *dev, int id)
3390 {
3391         drm_i915_private_t *dev_priv;
3392         struct drm_i915_gem_phys_object *phys_obj;
3393
3394         dev_priv = dev->dev_private;
3395         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3396                 return;
3397
3398         phys_obj = dev_priv->mm.phys_objs[id - 1];
3399         if (phys_obj->cur_obj != NULL)
3400                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3401
3402         drm_pci_free(dev, phys_obj->handle);
3403         drm_free(phys_obj, DRM_I915_GEM);
3404         dev_priv->mm.phys_objs[id - 1] = NULL;
3405 }
3406
3407 void
3408 i915_gem_free_all_phys_object(struct drm_device *dev)
3409 {
3410         int i;
3411
3412         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3413                 i915_gem_free_phys_object(dev, i);
3414 }
3415
3416 void
3417 i915_gem_detach_phys_object(struct drm_device *dev,
3418     struct drm_i915_gem_object *obj)
3419 {
3420         vm_page_t m;
3421         struct sf_buf *sf;
3422         char *vaddr, *dst;
3423         int i, page_count;
3424
3425         if (obj->phys_obj == NULL)
3426                 return;
3427         vaddr = obj->phys_obj->handle->vaddr;
3428
3429         page_count = obj->base.size / PAGE_SIZE;
3430         VM_OBJECT_LOCK(obj->base.vm_obj);
3431         for (i = 0; i < page_count; i++) {
3432                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3433                 if (m == NULL)
3434                         continue; /* XXX */
3435
3436                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3437                 sf = sf_buf_alloc(m);
3438                 if (sf != NULL) {
3439                         dst = (char *)sf_buf_kva(sf);
3440                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3441                         sf_buf_free(sf);
3442                 }
3443                 drm_clflush_pages(&m, 1);
3444
3445                 VM_OBJECT_LOCK(obj->base.vm_obj);
3446                 vm_page_reference(m);
3447                 vm_page_dirty(m);
3448                 vm_page_busy_wait(m, FALSE, "i915gem");
3449                 vm_page_unwire(m, 0);
3450                 vm_page_wakeup(m);
3451                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3452         }
3453         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3454         intel_gtt_chipset_flush();
3455
3456         obj->phys_obj->cur_obj = NULL;
3457         obj->phys_obj = NULL;
3458 }
3459
3460 static void
3461 init_ring_lists(struct intel_ring_buffer *ring)
3462 {
3463
3464         INIT_LIST_HEAD(&ring->active_list);
3465         INIT_LIST_HEAD(&ring->request_list);
3466         INIT_LIST_HEAD(&ring->gpu_write_list);
3467 }
3468
3469 void
3470 i915_gem_load(struct drm_device *dev)
3471 {
3472         int i;
3473         drm_i915_private_t *dev_priv = dev->dev_private;
3474
3475         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3476         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3477         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3478         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3479         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3480         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3481         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3482         for (i = 0; i < I915_NUM_RINGS; i++)
3483                 init_ring_lists(&dev_priv->ring[i]);
3484         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3485                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3486         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3487                           i915_gem_retire_work_handler);
3488         dev_priv->error_completion = 0;
3489
3490         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3491         if (IS_GEN3(dev)) {
3492                 I915_WRITE(MI_ARB_STATE,
3493                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3494         }
3495
3496         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3497
3498         /* Old X drivers will take 0-2 for front, back, depth buffers */
3499         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3500                 dev_priv->fence_reg_start = 3;
3501
3502         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3503                 dev_priv->num_fence_regs = 16;
3504         else
3505                 dev_priv->num_fence_regs = 8;
3506
3507         /* Initialize fence registers to zero */
3508         i915_gem_reset_fences(dev);
3509
3510         i915_gem_detect_bit_6_swizzle(dev);
3511
3512         dev_priv->mm.interruptible = true;
3513
3514         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
3515             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
3516 }
3517
3518 int
3519 i915_gem_attach_phys_object(struct drm_device *dev,
3520     struct drm_i915_gem_object *obj, int id, int align)
3521 {
3522         drm_i915_private_t *dev_priv;
3523         vm_page_t m;
3524         struct sf_buf *sf;
3525         char *dst, *src;
3526         int i, page_count, ret;
3527
3528         if (id > I915_MAX_PHYS_OBJECT)
3529                 return (-EINVAL);
3530
3531         if (obj->phys_obj != NULL) {
3532                 if (obj->phys_obj->id == id)
3533                         return (0);
3534                 i915_gem_detach_phys_object(dev, obj);
3535         }
3536
3537         dev_priv = dev->dev_private;
3538         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3539                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3540                 if (ret != 0) {
3541                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3542                                   id, obj->base.size);
3543                         return (ret);
3544                 }
3545         }
3546
3547         /* bind to the object */
3548         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3549         obj->phys_obj->cur_obj = obj;
3550
3551         page_count = obj->base.size / PAGE_SIZE;
3552
3553         VM_OBJECT_LOCK(obj->base.vm_obj);
3554         ret = 0;
3555         for (i = 0; i < page_count; i++) {
3556                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3557                 if (m == NULL) {
3558                         ret = -EIO;
3559                         break;
3560                 }
3561                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3562                 sf = sf_buf_alloc(m);
3563                 src = (char *)sf_buf_kva(sf);
3564                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3565                 memcpy(dst, src, PAGE_SIZE);
3566                 sf_buf_free(sf);
3567
3568                 VM_OBJECT_LOCK(obj->base.vm_obj);
3569
3570                 vm_page_reference(m);
3571                 vm_page_busy_wait(m, FALSE, "i915gem");
3572                 vm_page_unwire(m, 0);
3573                 vm_page_wakeup(m);
3574                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3575         }
3576         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3577
3578         return (0);
3579 }
3580
3581 static int
3582 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3583     uint64_t data_ptr, uint64_t offset, uint64_t size,
3584     struct drm_file *file_priv)
3585 {
3586         char *user_data, *vaddr;
3587         int ret;
3588
3589         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3590         user_data = (char *)(uintptr_t)data_ptr;
3591
3592         if (copyin_nofault(user_data, vaddr, size) != 0) {
3593                 /* The physical object once assigned is fixed for the lifetime
3594                  * of the obj, so we can safely drop the lock and continue
3595                  * to access vaddr.
3596                  */
3597                 DRM_UNLOCK(dev);
3598                 ret = -copyin(user_data, vaddr, size);
3599                 DRM_LOCK(dev);
3600                 if (ret != 0)
3601                         return (ret);
3602         }
3603
3604         intel_gtt_chipset_flush();
3605         return (0);
3606 }
3607
3608 static int
3609 i915_gpu_is_active(struct drm_device *dev)
3610 {
3611         drm_i915_private_t *dev_priv;
3612
3613         dev_priv = dev->dev_private;
3614         return (!list_empty(&dev_priv->mm.flushing_list) ||
3615             !list_empty(&dev_priv->mm.active_list));
3616 }
3617
3618 static void
3619 i915_gem_lowmem(void *arg)
3620 {
3621         struct drm_device *dev;
3622         struct drm_i915_private *dev_priv;
3623         struct drm_i915_gem_object *obj, *next;
3624         int cnt, cnt_fail, cnt_total;
3625
3626         dev = arg;
3627         dev_priv = dev->dev_private;
3628
3629         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3630                 return;
3631
3632 rescan:
3633         /* first scan for clean buffers */
3634         i915_gem_retire_requests(dev);
3635
3636         cnt_total = cnt_fail = cnt = 0;
3637
3638         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3639             mm_list) {
3640                 if (i915_gem_object_is_purgeable(obj)) {
3641                         if (i915_gem_object_unbind(obj) != 0)
3642                                 cnt_total++;
3643                 } else
3644                         cnt_total++;
3645         }
3646
3647         /* second pass, evict/count anything still on the inactive list */
3648         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3649             mm_list) {
3650                 if (i915_gem_object_unbind(obj) == 0)
3651                         cnt++;
3652                 else
3653                         cnt_fail++;
3654         }
3655
3656         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3657                 /*
3658                  * We are desperate for pages, so as a last resort, wait
3659                  * for the GPU to finish and discard whatever we can.
3660                  * This has a dramatic impact to reduce the number of
3661                  * OOM-killer events whilst running the GPU aggressively.
3662                  */
3663                 if (i915_gpu_idle(dev, true) == 0)
3664                         goto rescan;
3665         }
3666         DRM_UNLOCK(dev);
3667 }
3668
3669 void
3670 i915_gem_unload(struct drm_device *dev)
3671 {
3672         struct drm_i915_private *dev_priv;
3673
3674         dev_priv = dev->dev_private;
3675         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3676 }