sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64 #include <linux/completion.h>
  65 #include <linux/jiffies.h>
  66 #include <linux/time.h>
  67
  68 static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  69 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  70 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  71 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  72     unsigned alignment, bool map_and_fenceable);
  73
  74 static int i915_gem_phys_pwrite(struct drm_device *dev,
  75     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  76     uint64_t size, struct drm_file *file_priv);
  77
  78 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  79     int tiling_mode);
  80 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  81     uint32_t size, int tiling_mode);
  82 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  83     int flags);
  84 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  85 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  86 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  87 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  88 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  89 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  90 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  91     uint32_t flush_domains);
  92 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  93     struct drm_i915_fence_reg *reg);
  94 static void i915_gem_reset_fences(struct drm_device *dev);
  95 static void i915_gem_lowmem(void *arg);
  96
  97 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
  98     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file);
  99
 100 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
 101 long i915_gem_wired_pages_cnt;
 102
 103 /* some bookkeeping */
 104 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 105                                   size_t size)
 106 {
 107
 108         dev_priv->mm.object_count++;
 109         dev_priv->mm.object_memory += size;
 110 }
 111
 112 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 113                                      size_t size)
 114 {
 115
 116         dev_priv->mm.object_count--;
 117         dev_priv->mm.object_memory -= size;
 118 }
 119
 120 static int
 121 i915_gem_wait_for_error(struct drm_device *dev)
 122 {
 123         struct drm_i915_private *dev_priv = dev->dev_private;
 124         struct completion *x = &dev_priv->error_completion;
 125         int ret;
 126
 127         if (!atomic_read(&dev_priv->mm.wedged))
 128                 return 0;
 129
 130         /*
 131          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 132          * userspace. If it takes that long something really bad is going on and
 133          * we should simply try to bail out and fail as gracefully as possible.
 134          */
 135         ret = wait_for_completion_interruptible_timeout(x, 10*hz);
 136         if (ret == 0) {
 137                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 138                 return -EIO;
 139         } else if (ret < 0) {
 140                 return ret;
 141         }
 142
 143         if (atomic_read(&dev_priv->mm.wedged)) {
 144                 /* GPU is hung, bump the completion count to account for
 145                  * the token we just consumed so that we never hit zero and
 146                  * end up waiting upon a subsequent completion event that
 147                  * will never happen.
 148                  */
 149                 spin_lock(&x->wait.lock);
 150                 x->done++;
 151                 spin_unlock(&x->wait.lock);
 152         }
 153         return 0;
 154 }
 155
 156 int i915_mutex_lock_interruptible(struct drm_device *dev)
 157 {
 158         int ret;
 159
 160         ret = i915_gem_wait_for_error(dev);
 161         if (ret != 0)
 162                 return (ret);
 163
 164         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 165         if (ret)
 166                 return -EINTR;
 167
 168         WARN_ON(i915_verify_lists(dev));
 169         return 0;
 170 }
 171
 172 static inline bool
 173 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 174 {
 175         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
 176 }
 177
 178 int
 179 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 180     struct drm_file *file)
 181 {
 182         struct drm_i915_gem_init *args;
 183         drm_i915_private_t *dev_priv;
 184
 185         dev_priv = dev->dev_private;
 186         args = data;
 187
 188         if (args->gtt_start >= args->gtt_end ||
 189             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 190                 return (-EINVAL);
 191
 192         /*
 193          * XXXKIB. The second-time initialization should be guarded
 194          * against.
 195          */
 196         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 197         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 198         lockmgr(&dev->dev_lock, LK_RELEASE);
 199
 200         return 0;
 201 }
 202
 203 int
 204 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 205                             struct drm_file *file)
 206 {
 207         struct drm_i915_private *dev_priv;
 208         struct drm_i915_gem_get_aperture *args;
 209         struct drm_i915_gem_object *obj;
 210         size_t pinned;
 211
 212         dev_priv = dev->dev_private;
 213         args = data;
 214
 215         pinned = 0;
 216         DRM_LOCK(dev);
 217         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 218                 pinned += obj->gtt_space->size;
 219         DRM_UNLOCK(dev);
 220
 221         args->aper_size = dev_priv->mm.gtt_total;
 222         args->aper_available_size = args->aper_size - pinned;
 223
 224         return (0);
 225 }
 226
 227 static int
 228 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 229     uint32_t *handle_p)
 230 {
 231         struct drm_i915_gem_object *obj;
 232         uint32_t handle;
 233         int ret;
 234
 235         size = roundup(size, PAGE_SIZE);
 236         if (size == 0)
 237                 return (-EINVAL);
 238
 239         obj = i915_gem_alloc_object(dev, size);
 240         if (obj == NULL)
 241                 return (-ENOMEM);
 242
 243         handle = 0;
 244         ret = drm_gem_handle_create(file, &obj->base, &handle);
 245         if (ret != 0) {
 246                 drm_gem_object_release(&obj->base);
 247                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 248                 drm_free(obj, DRM_I915_GEM);
 249                 return (-ret);
 250         }
 251
 252         /* drop reference from allocate - handle holds it now */
 253         drm_gem_object_unreference(&obj->base);
 254         *handle_p = handle;
 255         return (0);
 256 }
 257
 258 int
 259 i915_gem_dumb_create(struct drm_file *file,
 260                      struct drm_device *dev,
 261                      struct drm_mode_create_dumb *args)
 262 {
 263
 264         /* have to work out size/pitch and return them */
 265         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 266         args->size = args->pitch * args->height;
 267         return (i915_gem_create(file, dev, args->size, &args->handle));
 268 }
 269
 270 int i915_gem_dumb_destroy(struct drm_file *file,
 271                           struct drm_device *dev,
 272                           uint32_t handle)
 273 {
 274
 275         return (drm_gem_handle_delete(file, handle));
 276 }
 277
 278 /**
 279  * Creates a new mm object and returns a handle to it.
 280  */
 281 int
 282 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 283                       struct drm_file *file)
 284 {
 285         struct drm_i915_gem_create *args = data;
 286
 287         return (i915_gem_create(file, dev, args->size, &args->handle));
 288 }
 289
 290 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 291 {
 292         drm_i915_private_t *dev_priv;
 293
 294         dev_priv = obj->base.dev->dev_private;
 295         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 296             obj->tiling_mode != I915_TILING_NONE);
 297 }
 298
 299 /**
 300  * Reads data from the object referenced by handle.
 301  *
 302  * On error, the contents of *data are undefined.
 303  */
 304 int
 305 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 306                      struct drm_file *file)
 307 {
 308         struct drm_i915_gem_pread *args;
 309
 310         args = data;
 311         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 312             args->offset, UIO_READ, file));
 313 }
 314
 315 /**
 316  * Writes data to the object referenced by handle.
 317  *
 318  * On error, the contents of the buffer that were to be modified are undefined.
 319  */
 320 int
 321 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 322                       struct drm_file *file)
 323 {
 324         struct drm_i915_gem_pwrite *args;
 325
 326         args = data;
 327         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 328             args->offset, UIO_WRITE, file));
 329 }
 330
 331 int
 332 i915_gem_check_wedge(struct drm_i915_private *dev_priv,
 333                      bool interruptible)
 334 {
 335         if (atomic_read(&dev_priv->mm.wedged)) {
 336                 struct completion *x = &dev_priv->error_completion;
 337                 bool recovery_complete;
 338
 339                 /* Give the error handler a chance to run. */
 340                 spin_lock(&x->wait.lock);
 341                 recovery_complete = x->done > 0;
 342                 spin_unlock(&x->wait.lock);
 343
 344                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 345                  * -EIO unconditionally for these. */
 346                 if (!interruptible)
 347                         return -EIO;
 348
 349                 /* Recovery complete, but still wedged means reset failure. */
 350                 if (recovery_complete)
 351                         return -EIO;
 352
 353                 return -EAGAIN;
 354         }
 355
 356         return 0;
 357 }
 358
 359 /*
 360  * Compare seqno against outstanding lazy request. Emit a request if they are
 361  * equal.
 362  */
 363 static int
 364 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
 365 {
 366         int ret;
 367
 368         DRM_LOCK_ASSERT(ring->dev);
 369
 370         ret = 0;
 371         if (seqno == ring->outstanding_lazy_request)
 372                 ret = i915_add_request(ring, NULL, NULL);
 373
 374         return ret;
 375 }
 376
 377 /**
 378  * __wait_seqno - wait until execution of seqno has finished
 379  * @ring: the ring expected to report seqno
 380  * @seqno: duh!
 381  * @interruptible: do an interruptible wait (normally yes)
 382  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 383  *
 384  * Returns 0 if the seqno was found within the alloted time. Else returns the
 385  * errno with remaining time filled in timeout argument.
 386  */
 387 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
 388                         bool interruptible, struct timespec *timeout)
 389 {
 390         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 391         struct timespec before, now, wait_time={1,0};
 392         unsigned long timeout_jiffies;
 393         long end;
 394         bool wait_forever = true;
 395         int ret;
 396
 397         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
 398                 return 0;
 399
 400         if (timeout != NULL) {
 401                 wait_time = *timeout;
 402                 wait_forever = false;
 403         }
 404
 405         timeout_jiffies = timespec_to_jiffies(&wait_time);
 406
 407         if (WARN_ON(!ring->irq_get(ring)))
 408                 return -ENODEV;
 409
 410         /* Record current time in case interrupted by signal, or wedged * */
 411         getrawmonotonic(&before);
 412
 413 #define EXIT_COND \
 414         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
 415         atomic_read(&dev_priv->mm.wedged))
 416         do {
 417                 if (interruptible)
 418                         end = wait_event_interruptible_timeout(ring->irq_queue,
 419                                                                EXIT_COND,
 420                                                                timeout_jiffies);
 421                 else
 422                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
 423                                                  timeout_jiffies);
 424
 425                 ret = i915_gem_check_wedge(dev_priv, interruptible);
 426                 if (ret)
 427                         end = ret;
 428         } while (end == 0 && wait_forever);
 429
 430         getrawmonotonic(&now);
 431
 432         ring->irq_put(ring);
 433 #undef EXIT_COND
 434
 435         if (timeout) {
 436                 struct timespec sleep_time = timespec_sub(now, before);
 437                 *timeout = timespec_sub(*timeout, sleep_time);
 438         }
 439
 440         switch (end) {
 441         case -EIO:
 442         case -EAGAIN: /* Wedged */
 443         case -ERESTARTSYS: /* Signal */
 444                 return (int)end;
 445         case 0: /* Timeout */
 446                 if (timeout)
 447                         set_normalized_timespec(timeout, 0, 0);
 448                 return -ETIMEDOUT;      /* -ETIME on Linux */
 449         default: /* Completed */
 450                 WARN_ON(end < 0); /* We're not aware of other errors */
 451                 return 0;
 452         }
 453 }
 454
 455 /**
 456  * Waits for a sequence number to be signaled, and cleans up the
 457  * request and object lists appropriately for that event.
 458  */
 459 int
 460 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
 461 {
 462         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 463         int ret = 0;
 464
 465         BUG_ON(seqno == 0);
 466
 467         ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
 468         if (ret)
 469                 return ret;
 470
 471         ret = i915_gem_check_olr(ring, seqno);
 472         if (ret)
 473                 return ret;
 474
 475         ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible, NULL);
 476
 477         return ret;
 478 }
 479
 480 /**
 481  * Ensures that all rendering to the object has completed and the object is
 482  * safe to unbind from the GTT or access from the CPU.
 483  */
 484 static __must_check int
 485 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
 486                                bool readonly)
 487 {
 488         u32 seqno;
 489         int ret;
 490
 491         /* This function only exists to support waiting for existing rendering,
 492          * not for emitting required flushes.
 493          */
 494         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
 495
 496         /* If there is rendering queued on the buffer being evicted, wait for
 497          * it.
 498          */
 499         if (readonly)
 500                 seqno = obj->last_write_seqno;
 501         else
 502                 seqno = obj->last_read_seqno;
 503         if (seqno == 0)
 504                 return 0;
 505
 506         ret = i915_wait_seqno(obj->ring, seqno);
 507         if (ret)
 508                 return ret;
 509
 510         /* Manually manage the write flush as we may have not yet retired
 511          * the buffer.
 512          */
 513         if (obj->last_write_seqno &&
 514             i915_seqno_passed(seqno, obj->last_write_seqno)) {
 515                 obj->last_write_seqno = 0;
 516                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
 517         }
 518
 519         i915_gem_retire_requests_ring(obj->ring);
 520         return 0;
 521 }
 522
 523 /**
 524  * Ensures that an object will eventually get non-busy by flushing any required
 525  * write domains, emitting any outstanding lazy request and retiring and
 526  * completed requests.
 527  */
 528 static int
 529 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
 530 {
 531         int ret;
 532
 533         if (obj->active) {
 534                 ret = i915_gem_object_flush_gpu_write_domain(obj);
 535                 if (ret)
 536                         return ret;
 537
 538                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
 539                 if (ret)
 540                         return ret;
 541
 542                 i915_gem_retire_requests_ring(obj->ring);
 543         }
 544
 545         return 0;
 546 }
 547
 548 /**
 549  * Called when user space prepares to use an object with the CPU, either
 550  * through the mmap ioctl's mapping or a GTT mapping.
 551  */
 552 int
 553 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 554                           struct drm_file *file)
 555 {
 556         struct drm_i915_gem_set_domain *args;
 557         struct drm_i915_gem_object *obj;
 558         uint32_t read_domains;
 559         uint32_t write_domain;
 560         int ret;
 561
 562         args = data;
 563         read_domains = args->read_domains;
 564         write_domain = args->write_domain;
 565
 566         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
 567             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
 568             (write_domain != 0 && read_domains != write_domain))
 569                 return (-EINVAL);
 570
 571         ret = i915_mutex_lock_interruptible(dev);
 572         if (ret != 0)
 573                 return (ret);
 574
 575         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 576         if (&obj->base == NULL) {
 577                 ret = -ENOENT;
 578                 goto unlock;
 579         }
 580
 581         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
 582                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 583                 if (ret == -EINVAL)
 584                         ret = 0;
 585         } else
 586                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 587
 588         drm_gem_object_unreference(&obj->base);
 589 unlock:
 590         DRM_UNLOCK(dev);
 591         return (ret);
 592 }
 593
 594 /**
 595  * Called when user space has done writes to this buffer
 596  */
 597 int
 598 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 599                          struct drm_file *file)
 600 {
 601         struct drm_i915_gem_sw_finish *args = data;
 602         struct drm_i915_gem_object *obj;
 603         int ret = 0;
 604
 605         ret = i915_mutex_lock_interruptible(dev);
 606         if (ret != 0)
 607                 return (ret);
 608         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 609         if (&obj->base == NULL) {
 610                 ret = -ENOENT;
 611                 goto unlock;
 612         }
 613         if (obj->pin_count != 0)
 614                 i915_gem_object_flush_cpu_write_domain(obj);
 615         drm_gem_object_unreference(&obj->base);
 616 unlock:
 617         DRM_UNLOCK(dev);
 618         return (ret);
 619 }
 620
 621 /**
 622  * Maps the contents of an object, returning the address it is mapped
 623  * into.
 624  *
 625  * While the mapping holds a reference on the contents of the object, it doesn't
 626  * imply a ref on the object itself.
 627  */
 628 int
 629 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 630                     struct drm_file *file)
 631 {
 632         struct drm_i915_gem_mmap *args;
 633         struct drm_gem_object *obj;
 634         struct proc *p;
 635         vm_map_t map;
 636         vm_offset_t addr;
 637         vm_size_t size;
 638         int error, rv;
 639
 640         args = data;
 641
 642         obj = drm_gem_object_lookup(dev, file, args->handle);
 643         if (obj == NULL)
 644                 return (-ENOENT);
 645         error = 0;
 646         if (args->size == 0)
 647                 goto out;
 648         p = curproc;
 649         map = &p->p_vmspace->vm_map;
 650         size = round_page(args->size);
 651         PROC_LOCK(p);
 652         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
 653                 PROC_UNLOCK(p);
 654                 error = ENOMEM;
 655                 goto out;
 656         }
 657         PROC_UNLOCK(p);
 658
 659         addr = 0;
 660         vm_object_hold(obj->vm_obj);
 661         vm_object_reference_locked(obj->vm_obj);
 662         vm_object_drop(obj->vm_obj);
 663         DRM_UNLOCK(dev);
 664         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
 665             PAGE_SIZE, /* align */
 666             TRUE, /* fitit */
 667             VM_MAPTYPE_NORMAL, /* maptype */
 668             VM_PROT_READ | VM_PROT_WRITE, /* prot */
 669             VM_PROT_READ | VM_PROT_WRITE, /* max */
 670             MAP_SHARED /* cow */);
 671         if (rv != KERN_SUCCESS) {
 672                 vm_object_deallocate(obj->vm_obj);
 673                 error = -vm_mmap_to_errno(rv);
 674         } else {
 675                 args->addr_ptr = (uint64_t)addr;
 676         }
 677         DRM_LOCK(dev);
 678 out:
 679         drm_gem_object_unreference(obj);
 680         return (error);
 681 }
 682
 683 /**
 684  * i915_gem_release_mmap - remove physical page mappings
 685  * @obj: obj in question
 686  *
 687  * Preserve the reservation of the mmapping with the DRM core code, but
 688  * relinquish ownership of the pages back to the system.
 689  *
 690  * It is vital that we remove the page mapping if we have mapped a tiled
 691  * object through the GTT and then lose the fence register due to
 692  * resource pressure. Similarly if the object has been moved out of the
 693  * aperture, than pages mapped into userspace must be revoked. Removing the
 694  * mapping will then trigger a page fault on the next user access, allowing
 695  * fixup by i915_gem_fault().
 696  */
 697 void
 698 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 699 {
 700         vm_object_t devobj;
 701         vm_page_t m;
 702         int i, page_count;
 703
 704         if (!obj->fault_mappable)
 705                 return;
 706
 707         devobj = cdev_pager_lookup(obj);
 708         if (devobj != NULL) {
 709                 page_count = OFF_TO_IDX(obj->base.size);
 710
 711                 VM_OBJECT_LOCK(devobj);
 712                 for (i = 0; i < page_count; i++) {
 713                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
 714                         if (m == NULL)
 715                                 continue;
 716                         cdev_pager_free_page(devobj, m);
 717                 }
 718                 VM_OBJECT_UNLOCK(devobj);
 719                 vm_object_deallocate(devobj);
 720         }
 721
 722         obj->fault_mappable = false;
 723 }
 724
 725 static uint32_t
 726 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 727 {
 728         uint32_t gtt_size;
 729
 730         if (INTEL_INFO(dev)->gen >= 4 ||
 731             tiling_mode == I915_TILING_NONE)
 732                 return (size);
 733
 734         /* Previous chips need a power-of-two fence region when tiling */
 735         if (INTEL_INFO(dev)->gen == 3)
 736                 gtt_size = 1024*1024;
 737         else
 738                 gtt_size = 512*1024;
 739
 740         while (gtt_size < size)
 741                 gtt_size <<= 1;
 742
 743         return (gtt_size);
 744 }
 745
 746 /**
 747  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 748  * @obj: object to check
 749  *
 750  * Return the required GTT alignment for an object, taking into account
 751  * potential fence register mapping.
 752  */
 753 static uint32_t
 754 i915_gem_get_gtt_alignment(struct drm_device *dev,
 755                            uint32_t size,
 756                            int tiling_mode)
 757 {
 758
 759         /*
 760          * Minimum alignment is 4k (GTT page size), but might be greater
 761          * if a fence register is needed for the object.
 762          */
 763         if (INTEL_INFO(dev)->gen >= 4 ||
 764             tiling_mode == I915_TILING_NONE)
 765                 return (4096);
 766
 767         /*
 768          * Previous chips need to be aligned to the size of the smallest
 769          * fence register that can contain the object.
 770          */
 771         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 772 }
 773
 774 /**
 775  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
 776  *                                       unfenced object
 777  * @dev: the device
 778  * @size: size of the object
 779  * @tiling_mode: tiling mode of the object
 780  *
 781  * Return the required GTT alignment for an object, only taking into account
 782  * unfenced tiled surface requirements.
 783  */
 784 uint32_t
 785 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 786                                     uint32_t size,
 787                                     int tiling_mode)
 788 {
 789
 790         if (tiling_mode == I915_TILING_NONE)
 791                 return (4096);
 792
 793         /*
 794          * Minimum alignment is 4k (GTT page size) for sane hw.
 795          */
 796         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
 797                 return (4096);
 798
 799         /*
 800          * Previous hardware however needs to be aligned to a power-of-two
 801          * tile height. The simplest method for determining this is to reuse
 802          * the power-of-tile object size.
 803          */
 804         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 805 }
 806
 807 int
 808 i915_gem_mmap_gtt(struct drm_file *file,
 809                   struct drm_device *dev,
 810                   uint32_t handle,
 811                   uint64_t *offset)
 812 {
 813         struct drm_i915_private *dev_priv;
 814         struct drm_i915_gem_object *obj;
 815         int ret;
 816
 817         dev_priv = dev->dev_private;
 818
 819         ret = i915_mutex_lock_interruptible(dev);
 820         if (ret != 0)
 821                 return (ret);
 822
 823         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 824         if (&obj->base == NULL) {
 825                 ret = -ENOENT;
 826                 goto unlock;
 827         }
 828
 829         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
 830                 ret = -E2BIG;
 831                 goto out;
 832         }
 833
 834         if (obj->madv != I915_MADV_WILLNEED) {
 835                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
 836                 ret = -EINVAL;
 837                 goto out;
 838         }
 839
 840         ret = drm_gem_create_mmap_offset(&obj->base);
 841         if (ret != 0)
 842                 goto out;
 843
 844         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
 845             DRM_GEM_MAPPING_KEY;
 846 out:
 847         drm_gem_object_unreference(&obj->base);
 848 unlock:
 849         DRM_UNLOCK(dev);
 850         return (ret);
 851 }
 852
 853 /**
 854  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 855  * @dev: DRM device
 856  * @data: GTT mapping ioctl data
 857  * @file: GEM object info
 858  *
 859  * Simply returns the fake offset to userspace so it can mmap it.
 860  * The mmap call will end up in drm_gem_mmap(), which will set things
 861  * up so we can get faults in the handler above.
 862  *
 863  * The fault handler will take care of binding the object into the GTT
 864  * (since it may have been evicted to make room for something), allocating
 865  * a fence register, and mapping the appropriate aperture address into
 866  * userspace.
 867  */
 868 int
 869 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 870                         struct drm_file *file)
 871 {
 872         struct drm_i915_private *dev_priv;
 873         struct drm_i915_gem_mmap_gtt *args = data;
 874
 875         dev_priv = dev->dev_private;
 876
 877         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
 878 }
 879
 880 /* Immediately discard the backing storage */
 881 static void
 882 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 883 {
 884         vm_object_t vm_obj;
 885
 886         vm_obj = obj->base.vm_obj;
 887         VM_OBJECT_LOCK(vm_obj);
 888         vm_object_page_remove(vm_obj, 0, 0, false);
 889         VM_OBJECT_UNLOCK(vm_obj);
 890         obj->madv = __I915_MADV_PURGED;
 891 }
 892
 893 static inline int
 894 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 895 {
 896         return obj->madv == I915_MADV_DONTNEED;
 897 }
 898
 899 static inline void vm_page_reference(vm_page_t m)
 900 {
 901         vm_page_flag_set(m, PG_REFERENCED);
 902 }
 903
 904 static void
 905 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 906 {
 907         vm_page_t m;
 908         int page_count, i;
 909
 910         BUG_ON(obj->madv == __I915_MADV_PURGED);
 911
 912         if (obj->tiling_mode != I915_TILING_NONE)
 913                 i915_gem_object_save_bit_17_swizzle(obj);
 914         if (obj->madv == I915_MADV_DONTNEED)
 915                 obj->dirty = 0;
 916         page_count = obj->base.size / PAGE_SIZE;
 917         VM_OBJECT_LOCK(obj->base.vm_obj);
 918 #if GEM_PARANOID_CHECK_GTT
 919         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
 920 #endif
 921         for (i = 0; i < page_count; i++) {
 922                 m = obj->pages[i];
 923                 if (obj->dirty)
 924                         vm_page_dirty(m);
 925                 if (obj->madv == I915_MADV_WILLNEED)
 926                         vm_page_reference(m);
 927                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
 928                 vm_page_unwire(obj->pages[i], 1);
 929                 vm_page_wakeup(obj->pages[i]);
 930                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 931         }
 932         VM_OBJECT_UNLOCK(obj->base.vm_obj);
 933         obj->dirty = 0;
 934         drm_free(obj->pages, DRM_I915_GEM);
 935         obj->pages = NULL;
 936 }
 937
 938 static int
 939 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 940     int flags)
 941 {
 942         struct drm_device *dev;
 943         vm_object_t vm_obj;
 944         vm_page_t m;
 945         int page_count, i, j;
 946
 947         dev = obj->base.dev;
 948         KASSERT(obj->pages == NULL, ("Obj already has pages"));
 949         page_count = obj->base.size / PAGE_SIZE;
 950         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 951             M_WAITOK);
 952         vm_obj = obj->base.vm_obj;
 953         VM_OBJECT_LOCK(vm_obj);
 954         for (i = 0; i < page_count; i++) {
 955                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
 956                         goto failed;
 957         }
 958         VM_OBJECT_UNLOCK(vm_obj);
 959         if (i915_gem_object_needs_bit17_swizzle(obj))
 960                 i915_gem_object_do_bit_17_swizzle(obj);
 961         return (0);
 962
 963 failed:
 964         for (j = 0; j < i; j++) {
 965                 m = obj->pages[j];
 966                 vm_page_busy_wait(m, FALSE, "i915gem");
 967                 vm_page_unwire(m, 0);
 968                 vm_page_wakeup(m);
 969                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 970         }
 971         VM_OBJECT_UNLOCK(vm_obj);
 972         drm_free(obj->pages, DRM_I915_GEM);
 973         obj->pages = NULL;
 974         return (-EIO);
 975 }
 976
 977 void
 978 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 979                                struct intel_ring_buffer *ring,
 980                                u32 seqno)
 981 {
 982         struct drm_device *dev = obj->base.dev;
 983         struct drm_i915_private *dev_priv = dev->dev_private;
 984
 985         BUG_ON(ring == NULL);
 986         obj->ring = ring;
 987
 988         /* Add a reference if we're newly entering the active list. */
 989         if (!obj->active) {
 990                 drm_gem_object_reference(&obj->base);
 991                 obj->active = 1;
 992         }
 993
 994         /* Move from whatever list we were on to the tail of execution. */
 995         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 996         list_move_tail(&obj->ring_list, &ring->active_list);
 997
 998         obj->last_read_seqno = seqno;
 999
1000         if (obj->fenced_gpu_access) {
1001                 obj->last_fenced_seqno = seqno;
1002
1003                 /* Bump MRU to take account of the delayed flush */
1004                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1005                         struct drm_i915_fence_reg *reg;
1006
1007                         reg = &dev_priv->fence_regs[obj->fence_reg];
1008                         list_move_tail(&reg->lru_list,
1009                                        &dev_priv->mm.fence_list);
1010                 }
1011         }
1012 }
1013
1014 static void
1015 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1016 {
1017         list_del_init(&obj->ring_list);
1018         obj->last_read_seqno = 0;
1019         obj->last_write_seqno = 0;
1020         obj->last_fenced_seqno = 0;
1021 }
1022
1023 static void
1024 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1025 {
1026         struct drm_device *dev = obj->base.dev;
1027         struct drm_i915_private *dev_priv = dev->dev_private;
1028
1029         list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1030
1031         BUG_ON(!list_empty(&obj->gpu_write_list));
1032         BUG_ON(!obj->active);
1033         obj->ring = NULL;
1034
1035         i915_gem_object_move_off_active(obj);
1036         obj->fenced_gpu_access = false;
1037
1038         obj->active = 0;
1039         drm_gem_object_unreference(&obj->base);
1040
1041         WARN_ON(i915_verify_lists(dev));
1042 }
1043
1044 static u32
1045 i915_gem_get_seqno(struct drm_device *dev)
1046 {
1047         drm_i915_private_t *dev_priv = dev->dev_private;
1048         u32 seqno = dev_priv->next_seqno;
1049
1050         /* reserve 0 for non-seqno */
1051         if (++dev_priv->next_seqno == 0)
1052                 dev_priv->next_seqno = 1;
1053
1054         return seqno;
1055 }
1056
1057 int
1058 i915_add_request(struct intel_ring_buffer *ring,
1059                  struct drm_file *file,
1060                  struct drm_i915_gem_request *request)
1061 {
1062         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1063         uint32_t seqno;
1064         u32 request_ring_position;
1065         int was_empty;
1066         int ret;
1067
1068         /*
1069          * Emit any outstanding flushes - execbuf can fail to emit the flush
1070          * after having emitted the batchbuffer command. Hence we need to fix
1071          * things up similar to emitting the lazy request. The difference here
1072          * is that the flush _must_ happen before the next request, no matter
1073          * what.
1074          */
1075         if (ring->gpu_caches_dirty) {
1076                 ret = i915_gem_flush_ring(ring, 0, I915_GEM_GPU_DOMAINS);
1077                 if (ret)
1078                         return ret;
1079
1080                 ring->gpu_caches_dirty = false;
1081         }
1082
1083         if (request == NULL) {
1084                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
1085                     M_WAITOK | M_ZERO);
1086                 if (request == NULL)
1087                         return -ENOMEM;
1088         }
1089
1090         seqno = i915_gem_next_request_seqno(ring);
1091
1092         /* Record the position of the start of the request so that
1093          * should we detect the updated seqno part-way through the
1094          * GPU processing the request, we never over-estimate the
1095          * position of the head.
1096          */
1097         request_ring_position = intel_ring_get_tail(ring);
1098
1099         ret = ring->add_request(ring, &seqno);
1100         if (ret) {
1101                 kfree(request, DRM_I915_GEM);
1102                 return ret;
1103         }
1104
1105         request->seqno = seqno;
1106         request->ring = ring;
1107         request->tail = request_ring_position;
1108         request->emitted_jiffies = jiffies;
1109         was_empty = list_empty(&ring->request_list);
1110         list_add_tail(&request->list, &ring->request_list);
1111         request->file_priv = NULL;
1112
1113         if (file) {
1114                 struct drm_i915_file_private *file_priv = file->driver_priv;
1115
1116                 spin_lock(&file_priv->mm.lock);
1117                 request->file_priv = file_priv;
1118                 list_add_tail(&request->client_list,
1119                               &file_priv->mm.request_list);
1120                 spin_unlock(&file_priv->mm.lock);
1121         }
1122
1123         ring->outstanding_lazy_request = 0;
1124
1125         if (!dev_priv->mm.suspended) {
1126                 if (i915_enable_hangcheck) {
1127                         mod_timer(&dev_priv->hangcheck_timer,
1128                                   jiffies +
1129                                   msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1130                 }
1131                 if (was_empty)
1132                         queue_delayed_work(dev_priv->wq,
1133                                            &dev_priv->mm.retire_work, hz);
1134         }
1135
1136         WARN_ON(!list_empty(&ring->gpu_write_list));
1137
1138         return 0;
1139 }
1140
1141 static inline void
1142 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1143 {
1144         struct drm_i915_file_private *file_priv = request->file_priv;
1145
1146         if (!file_priv)
1147                 return;
1148
1149         DRM_LOCK_ASSERT(request->ring->dev);
1150
1151         spin_lock(&file_priv->mm.lock);
1152         if (request->file_priv != NULL) {
1153                 list_del(&request->client_list);
1154                 request->file_priv = NULL;
1155         }
1156         spin_unlock(&file_priv->mm.lock);
1157 }
1158
1159 static void
1160 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1161     struct intel_ring_buffer *ring)
1162 {
1163
1164         if (ring->dev != NULL)
1165                 DRM_LOCK_ASSERT(ring->dev);
1166
1167         while (!list_empty(&ring->request_list)) {
1168                 struct drm_i915_gem_request *request;
1169
1170                 request = list_first_entry(&ring->request_list,
1171                     struct drm_i915_gem_request, list);
1172
1173                 list_del(&request->list);
1174                 i915_gem_request_remove_from_client(request);
1175                 drm_free(request, DRM_I915_GEM);
1176         }
1177
1178         while (!list_empty(&ring->active_list)) {
1179                 struct drm_i915_gem_object *obj;
1180
1181                 obj = list_first_entry(&ring->active_list,
1182                     struct drm_i915_gem_object, ring_list);
1183
1184                 obj->base.write_domain = 0;
1185                 list_del_init(&obj->gpu_write_list);
1186                 i915_gem_object_move_to_inactive(obj);
1187         }
1188 }
1189
1190 static void
1191 i915_gem_reset_fences(struct drm_device *dev)
1192 {
1193         struct drm_i915_private *dev_priv = dev->dev_private;
1194         int i;
1195
1196         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1197                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1198                 struct drm_i915_gem_object *obj = reg->obj;
1199
1200                 if (!obj)
1201                         continue;
1202
1203                 if (obj->tiling_mode)
1204                         i915_gem_release_mmap(obj);
1205
1206                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1207                 reg->obj->fenced_gpu_access = false;
1208                 reg->obj->last_fenced_seqno = 0;
1209                 reg->obj->last_fenced_ring = NULL;
1210                 i915_gem_clear_fence_reg(dev, reg);
1211         }
1212 }
1213
1214 void i915_gem_reset(struct drm_device *dev)
1215 {
1216         struct drm_i915_private *dev_priv = dev->dev_private;
1217         struct drm_i915_gem_object *obj;
1218         int i;
1219
1220         for (i = 0; i < I915_NUM_RINGS; i++)
1221                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1222
1223         /* Remove anything from the flushing lists. The GPU cache is likely
1224          * to be lost on reset along with the data, so simply move the
1225          * lost bo to the inactive list.
1226          */
1227         while (!list_empty(&dev_priv->mm.flushing_list)) {
1228                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1229                                       struct drm_i915_gem_object,
1230                                       mm_list);
1231
1232                 obj->base.write_domain = 0;
1233                 list_del_init(&obj->gpu_write_list);
1234                 i915_gem_object_move_to_inactive(obj);
1235         }
1236
1237         /* Move everything out of the GPU domains to ensure we do any
1238          * necessary invalidation upon reuse.
1239          */
1240         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
1241                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1242         }
1243
1244         /* The fence registers are invalidated so clear them out */
1245         i915_gem_reset_fences(dev);
1246 }
1247
1248 static void
1249 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1250 {
1251         struct drm_device *dev = obj->base.dev;
1252         drm_i915_private_t *dev_priv = dev->dev_private;
1253
1254         KASSERT(obj->active, ("Object not active"));
1255         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1256
1257         i915_gem_object_move_off_active(obj);
1258 }
1259
1260 /**
1261  * This function clears the request list as sequence numbers are passed.
1262  */
1263 void
1264 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1265 {
1266         uint32_t seqno;
1267         int i;
1268
1269         if (list_empty(&ring->request_list))
1270                 return;
1271
1272         WARN_ON(i915_verify_lists(ring->dev));
1273
1274         seqno = ring->get_seqno(ring, true);
1275
1276         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1277                 if (seqno >= ring->sync_seqno[i])
1278                         ring->sync_seqno[i] = 0;
1279
1280         while (!list_empty(&ring->request_list)) {
1281                 struct drm_i915_gem_request *request;
1282
1283                 request = list_first_entry(&ring->request_list,
1284                                            struct drm_i915_gem_request,
1285                                            list);
1286
1287                 if (!i915_seqno_passed(seqno, request->seqno))
1288                         break;
1289
1290                 /* We know the GPU must have read the request to have
1291                  * sent us the seqno + interrupt, so use the position
1292                  * of tail of the request to update the last known position
1293                  * of the GPU head.
1294                  */
1295                 ring->last_retired_head = request->tail;
1296
1297                 list_del(&request->list);
1298                 i915_gem_request_remove_from_client(request);
1299                 kfree(request, DRM_I915_GEM);
1300         }
1301
1302         /* Move any buffers on the active list that are no longer referenced
1303          * by the ringbuffer to the flushing/inactive lists as appropriate.
1304          */
1305         while (!list_empty(&ring->active_list)) {
1306                 struct drm_i915_gem_object *obj;
1307
1308                 obj = list_first_entry(&ring->active_list,
1309                                       struct drm_i915_gem_object,
1310                                       ring_list);
1311
1312                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
1313                         break;
1314
1315                 if (obj->base.write_domain != 0)
1316                         i915_gem_object_move_to_flushing(obj);
1317                 else
1318                         i915_gem_object_move_to_inactive(obj);
1319         }
1320
1321         if (unlikely(ring->trace_irq_seqno &&
1322                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1323                 ring->irq_put(ring);
1324                 ring->trace_irq_seqno = 0;
1325         }
1326
1327 }
1328
1329 static void
1330 i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
1331
1332 void
1333 i915_gem_retire_requests(struct drm_device *dev)
1334 {
1335         drm_i915_private_t *dev_priv = dev->dev_private;
1336         struct drm_i915_gem_object *obj, *next;
1337         int i;
1338
1339         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1340                 list_for_each_entry_safe(obj, next,
1341                     &dev_priv->mm.deferred_free_list, mm_list)
1342                         i915_gem_free_object_tail(obj);
1343         }
1344
1345         for (i = 0; i < I915_NUM_RINGS; i++)
1346                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1347 }
1348
1349 static void
1350 i915_gem_retire_work_handler(struct work_struct *work)
1351 {
1352         drm_i915_private_t *dev_priv;
1353         struct drm_device *dev;
1354         struct intel_ring_buffer *ring;
1355         bool idle;
1356         int i;
1357
1358         dev_priv = container_of(work, drm_i915_private_t,
1359                                 mm.retire_work.work);
1360         dev = dev_priv->dev;
1361
1362         /* Come back later if the device is busy... */
1363         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
1364                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1365                                    round_jiffies_up_relative(hz));
1366                 return;
1367         }
1368
1369         i915_gem_retire_requests(dev);
1370
1371         /* Send a periodic flush down the ring so we don't hold onto GEM
1372          * objects indefinitely.
1373          */
1374         idle = true;
1375         for_each_ring(ring, dev_priv, i) {
1376                 if (ring->gpu_caches_dirty)
1377                         i915_add_request(ring, NULL, NULL);
1378
1379                 idle &= list_empty(&ring->request_list);
1380         }
1381
1382         if (!dev_priv->mm.suspended && !idle)
1383                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1384                                    round_jiffies_up_relative(hz));
1385         if (idle)
1386                 intel_mark_idle(dev);
1387
1388         DRM_UNLOCK(dev);
1389 }
1390
1391 /**
1392  * i915_gem_object_sync - sync an object to a ring.
1393  *
1394  * @obj: object which may be in use on another ring.
1395  * @to: ring we wish to use the object on. May be NULL.
1396  *
1397  * This code is meant to abstract object synchronization with the GPU.
1398  * Calling with NULL implies synchronizing the object with the CPU
1399  * rather than a particular GPU ring.
1400  *
1401  * Returns 0 if successful, else propagates up the lower layer error.
1402  */
1403 int
1404 i915_gem_object_sync(struct drm_i915_gem_object *obj,
1405                      struct intel_ring_buffer *to)
1406 {
1407         struct intel_ring_buffer *from = obj->ring;
1408         u32 seqno;
1409         int ret, idx;
1410
1411         if (from == NULL || to == from)
1412                 return 0;
1413
1414         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
1415                 return i915_gem_object_wait_rendering(obj, false);
1416
1417         idx = intel_ring_sync_index(from, to);
1418
1419         seqno = obj->last_read_seqno;
1420         if (seqno <= from->sync_seqno[idx])
1421                 return 0;
1422
1423         ret = i915_gem_check_olr(obj->ring, seqno);
1424         if (ret)
1425                 return ret;
1426
1427         ret = to->sync_to(to, from, seqno);
1428         if (!ret)
1429                 from->sync_seqno[idx] = seqno;
1430
1431         return ret;
1432 }
1433
1434 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1435 {
1436         u32 old_write_domain, old_read_domains;
1437
1438         /* Act a barrier for all accesses through the GTT */
1439         cpu_mfence();
1440
1441         /* Force a pagefault for domain tracking on next user access */
1442         i915_gem_release_mmap(obj);
1443
1444         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1445                 return;
1446
1447         old_read_domains = obj->base.read_domains;
1448         old_write_domain = obj->base.write_domain;
1449
1450         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1451         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1452
1453 }
1454
1455 int
1456 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1457 {
1458         drm_i915_private_t *dev_priv;
1459         int ret;
1460
1461         dev_priv = obj->base.dev->dev_private;
1462         ret = 0;
1463         if (obj->gtt_space == NULL)
1464                 return (0);
1465         if (obj->pin_count != 0) {
1466                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1467                 return (-EINVAL);
1468         }
1469
1470         ret = i915_gem_object_finish_gpu(obj);
1471         if (ret == -ERESTART || ret == -EINTR)
1472                 return (ret);
1473
1474         i915_gem_object_finish_gtt(obj);
1475
1476         if (ret == 0)
1477                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1478         if (ret == -ERESTART || ret == -EINTR)
1479                 return (ret);
1480         if (ret != 0) {
1481                 i915_gem_clflush_object(obj);
1482                 obj->base.read_domains = obj->base.write_domain =
1483                     I915_GEM_DOMAIN_CPU;
1484         }
1485
1486         ret = i915_gem_object_put_fence(obj);
1487         if (ret == -ERESTART)
1488                 return (ret);
1489
1490         i915_gem_gtt_unbind_object(obj);
1491         if (obj->has_aliasing_ppgtt_mapping) {
1492                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
1493                 obj->has_aliasing_ppgtt_mapping = 0;
1494         }
1495         i915_gem_object_put_pages_gtt(obj);
1496
1497         list_del_init(&obj->gtt_list);
1498         list_del_init(&obj->mm_list);
1499         obj->map_and_fenceable = true;
1500
1501         drm_mm_put_block(obj->gtt_space);
1502         obj->gtt_space = NULL;
1503         obj->gtt_offset = 0;
1504
1505         if (i915_gem_object_is_purgeable(obj))
1506                 i915_gem_object_truncate(obj);
1507
1508         return (ret);
1509 }
1510
1511 int i915_gpu_idle(struct drm_device *dev)
1512 {
1513         drm_i915_private_t *dev_priv = dev->dev_private;
1514         struct intel_ring_buffer *ring;
1515         int ret, i;
1516
1517         /* Flush everything onto the inactive list. */
1518         for_each_ring(ring, dev_priv, i) {
1519                 ret = intel_ring_idle(ring);
1520                 if (ret)
1521                         return ret;
1522         }
1523
1524         return 0;
1525 }
1526
1527 static int
1528 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
1529     struct intel_ring_buffer *pipelined)
1530 {
1531         struct drm_device *dev = obj->base.dev;
1532         drm_i915_private_t *dev_priv = dev->dev_private;
1533         u32 size = obj->gtt_space->size;
1534         int regnum = obj->fence_reg;
1535         uint64_t val;
1536
1537         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1538                          0xfffff000) << 32;
1539         val |= obj->gtt_offset & 0xfffff000;
1540         val |= (uint64_t)((obj->stride / 128) - 1) <<
1541                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
1542
1543         if (obj->tiling_mode == I915_TILING_Y)
1544                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1545         val |= I965_FENCE_REG_VALID;
1546
1547         if (pipelined) {
1548                 int ret = intel_ring_begin(pipelined, 6);
1549                 if (ret)
1550                         return ret;
1551
1552                 intel_ring_emit(pipelined, MI_NOOP);
1553                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1554                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
1555                 intel_ring_emit(pipelined, (u32)val);
1556                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
1557                 intel_ring_emit(pipelined, (u32)(val >> 32));
1558                 intel_ring_advance(pipelined);
1559         } else
1560                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
1561
1562         return 0;
1563 }
1564
1565 static int
1566 i965_write_fence_reg(struct drm_i915_gem_object *obj,
1567     struct intel_ring_buffer *pipelined)
1568 {
1569         struct drm_device *dev = obj->base.dev;
1570         drm_i915_private_t *dev_priv = dev->dev_private;
1571         u32 size = obj->gtt_space->size;
1572         int regnum = obj->fence_reg;
1573         uint64_t val;
1574
1575         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1576                     0xfffff000) << 32;
1577         val |= obj->gtt_offset & 0xfffff000;
1578         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1579         if (obj->tiling_mode == I915_TILING_Y)
1580                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1581         val |= I965_FENCE_REG_VALID;
1582
1583         if (pipelined) {
1584                 int ret = intel_ring_begin(pipelined, 6);
1585                 if (ret)
1586                         return ret;
1587
1588                 intel_ring_emit(pipelined, MI_NOOP);
1589                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1590                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
1591                 intel_ring_emit(pipelined, (u32)val);
1592                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
1593                 intel_ring_emit(pipelined, (u32)(val >> 32));
1594                 intel_ring_advance(pipelined);
1595         } else
1596                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
1597
1598         return 0;
1599 }
1600
1601 static int
1602 i915_write_fence_reg(struct drm_i915_gem_object *obj,
1603     struct intel_ring_buffer *pipelined)
1604 {
1605         struct drm_device *dev = obj->base.dev;
1606         drm_i915_private_t *dev_priv = dev->dev_private;
1607         u32 size = obj->gtt_space->size;
1608         u32 fence_reg, val, pitch_val;
1609         int tile_width;
1610
1611         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
1612             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1613                 kprintf(
1614 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1615                  obj->gtt_offset, obj->map_and_fenceable, size);
1616                 return -EINVAL;
1617         }
1618
1619         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1620                 tile_width = 128;
1621         else
1622                 tile_width = 512;
1623
1624         /* Note: pitch better be a power of two tile widths */
1625         pitch_val = obj->stride / tile_width;
1626         pitch_val = ffs(pitch_val) - 1;
1627
1628         val = obj->gtt_offset;
1629         if (obj->tiling_mode == I915_TILING_Y)
1630                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1631         val |= I915_FENCE_SIZE_BITS(size);
1632         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1633         val |= I830_FENCE_REG_VALID;
1634
1635         fence_reg = obj->fence_reg;
1636         if (fence_reg < 8)
1637                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
1638         else
1639                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
1640
1641         if (pipelined) {
1642                 int ret = intel_ring_begin(pipelined, 4);
1643                 if (ret)
1644                         return ret;
1645
1646                 intel_ring_emit(pipelined, MI_NOOP);
1647                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1648                 intel_ring_emit(pipelined, fence_reg);
1649                 intel_ring_emit(pipelined, val);
1650                 intel_ring_advance(pipelined);
1651         } else
1652                 I915_WRITE(fence_reg, val);
1653
1654         return 0;
1655 }
1656
1657 static int
1658 i830_write_fence_reg(struct drm_i915_gem_object *obj,
1659     struct intel_ring_buffer *pipelined)
1660 {
1661         struct drm_device *dev = obj->base.dev;
1662         drm_i915_private_t *dev_priv = dev->dev_private;
1663         u32 size = obj->gtt_space->size;
1664         int regnum = obj->fence_reg;
1665         uint32_t val;
1666         uint32_t pitch_val;
1667
1668         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
1669             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1670                 kprintf(
1671 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1672                     obj->gtt_offset, size);
1673                 return -EINVAL;
1674         }
1675
1676         pitch_val = obj->stride / 128;
1677         pitch_val = ffs(pitch_val) - 1;
1678
1679         val = obj->gtt_offset;
1680         if (obj->tiling_mode == I915_TILING_Y)
1681                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1682         val |= I830_FENCE_SIZE_BITS(size);
1683         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1684         val |= I830_FENCE_REG_VALID;
1685
1686         if (pipelined) {
1687                 int ret = intel_ring_begin(pipelined, 4);
1688                 if (ret)
1689                         return ret;
1690
1691                 intel_ring_emit(pipelined, MI_NOOP);
1692                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1693                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
1694                 intel_ring_emit(pipelined, val);
1695                 intel_ring_advance(pipelined);
1696         } else
1697                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
1698
1699         return 0;
1700 }
1701
1702 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
1703 {
1704         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
1705 }
1706
1707 static int
1708 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
1709     struct intel_ring_buffer *pipelined)
1710 {
1711         int ret;
1712
1713         if (obj->fenced_gpu_access) {
1714                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1715                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
1716                             obj->base.write_domain);
1717                         if (ret)
1718                                 return ret;
1719                 }
1720
1721                 obj->fenced_gpu_access = false;
1722         }
1723
1724         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
1725                 if (!ring_passed_seqno(obj->last_fenced_ring,
1726                                        obj->last_fenced_seqno)) {
1727                         ret = i915_wait_seqno(obj->last_fenced_ring,
1728                                                 obj->last_fenced_seqno);
1729                         if (ret)
1730                                 return ret;
1731                 }
1732
1733                 obj->last_fenced_seqno = 0;
1734                 obj->last_fenced_ring = NULL;
1735         }
1736
1737         /* Ensure that all CPU reads are completed before installing a fence
1738          * and all writes before removing the fence.
1739          */
1740         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
1741                 cpu_mfence();
1742
1743         return 0;
1744 }
1745
1746 int
1747 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
1748 {
1749         int ret;
1750
1751         if (obj->tiling_mode)
1752                 i915_gem_release_mmap(obj);
1753
1754         ret = i915_gem_object_flush_fence(obj, NULL);
1755         if (ret)
1756                 return ret;
1757
1758         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1759                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1760
1761                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
1762                         kprintf("%s: pin_count %d\n", __func__,
1763                             dev_priv->fence_regs[obj->fence_reg].pin_count);
1764                 i915_gem_clear_fence_reg(obj->base.dev,
1765                                          &dev_priv->fence_regs[obj->fence_reg]);
1766
1767                 obj->fence_reg = I915_FENCE_REG_NONE;
1768         }
1769
1770         return 0;
1771 }
1772
1773 static struct drm_i915_fence_reg *
1774 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
1775 {
1776         struct drm_i915_private *dev_priv = dev->dev_private;
1777         struct drm_i915_fence_reg *reg, *first, *avail;
1778         int i;
1779
1780         /* First try to find a free reg */
1781         avail = NULL;
1782         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
1783                 reg = &dev_priv->fence_regs[i];
1784                 if (!reg->obj)
1785                         return reg;
1786
1787                 if (!reg->pin_count)
1788                         avail = reg;
1789         }
1790
1791         if (avail == NULL)
1792                 return NULL;
1793
1794         /* None available, try to steal one or wait for a user to finish */
1795         avail = first = NULL;
1796         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1797                 if (reg->pin_count)
1798                         continue;
1799
1800                 if (first == NULL)
1801                         first = reg;
1802
1803                 if (!pipelined ||
1804                     !reg->obj->last_fenced_ring ||
1805                     reg->obj->last_fenced_ring == pipelined) {
1806                         avail = reg;
1807                         break;
1808                 }
1809         }
1810
1811         if (avail == NULL)
1812                 avail = first;
1813
1814         return avail;
1815 }
1816
1817 int
1818 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1819     struct intel_ring_buffer *pipelined)
1820 {
1821         struct drm_device *dev = obj->base.dev;
1822         struct drm_i915_private *dev_priv = dev->dev_private;
1823         struct drm_i915_fence_reg *reg;
1824         int ret;
1825
1826         pipelined = NULL;
1827         ret = 0;
1828
1829         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1830                 reg = &dev_priv->fence_regs[obj->fence_reg];
1831                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1832
1833                 if (obj->tiling_changed) {
1834                         ret = i915_gem_object_flush_fence(obj, pipelined);
1835                         if (ret)
1836                                 return ret;
1837
1838                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1839                                 pipelined = NULL;
1840
1841                         if (pipelined) {
1842                                 reg->setup_seqno =
1843                                         i915_gem_next_request_seqno(pipelined);
1844                                 obj->last_fenced_seqno = reg->setup_seqno;
1845                                 obj->last_fenced_ring = pipelined;
1846                         }
1847
1848                         goto update;
1849                 }
1850
1851                 if (!pipelined) {
1852                         if (reg->setup_seqno) {
1853                                 if (!ring_passed_seqno(obj->last_fenced_ring,
1854                                     reg->setup_seqno)) {
1855                                         ret = i915_wait_seqno(
1856                                             obj->last_fenced_ring,
1857                                             reg->setup_seqno);
1858                                         if (ret)
1859                                                 return ret;
1860                                 }
1861
1862                                 reg->setup_seqno = 0;
1863                         }
1864                 } else if (obj->last_fenced_ring &&
1865                            obj->last_fenced_ring != pipelined) {
1866                         ret = i915_gem_object_flush_fence(obj, pipelined);
1867                         if (ret)
1868                                 return ret;
1869                 }
1870
1871                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1872                         pipelined = NULL;
1873                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
1874
1875                 if (obj->tiling_changed) {
1876                         if (pipelined) {
1877                                 reg->setup_seqno =
1878                                         i915_gem_next_request_seqno(pipelined);
1879                                 obj->last_fenced_seqno = reg->setup_seqno;
1880                                 obj->last_fenced_ring = pipelined;
1881                         }
1882                         goto update;
1883                 }
1884
1885                 return 0;
1886         }
1887
1888         reg = i915_find_fence_reg(dev, pipelined);
1889         if (reg == NULL)
1890                 return -EDEADLK;
1891
1892         ret = i915_gem_object_flush_fence(obj, pipelined);
1893         if (ret)
1894                 return ret;
1895
1896         if (reg->obj) {
1897                 struct drm_i915_gem_object *old = reg->obj;
1898
1899                 drm_gem_object_reference(&old->base);
1900
1901                 if (old->tiling_mode)
1902                         i915_gem_release_mmap(old);
1903
1904                 ret = i915_gem_object_flush_fence(old, pipelined);
1905                 if (ret) {
1906                         drm_gem_object_unreference(&old->base);
1907                         return ret;
1908                 }
1909
1910                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
1911                         pipelined = NULL;
1912
1913                 old->fence_reg = I915_FENCE_REG_NONE;
1914                 old->last_fenced_ring = pipelined;
1915                 old->last_fenced_seqno =
1916                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1917
1918                 drm_gem_object_unreference(&old->base);
1919         } else if (obj->last_fenced_seqno == 0)
1920                 pipelined = NULL;
1921
1922         reg->obj = obj;
1923         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1924         obj->fence_reg = reg - dev_priv->fence_regs;
1925         obj->last_fenced_ring = pipelined;
1926
1927         reg->setup_seqno =
1928                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1929         obj->last_fenced_seqno = reg->setup_seqno;
1930
1931 update:
1932         obj->tiling_changed = false;
1933         switch (INTEL_INFO(dev)->gen) {
1934         case 7:
1935         case 6:
1936                 ret = sandybridge_write_fence_reg(obj, pipelined);
1937                 break;
1938         case 5:
1939         case 4:
1940                 ret = i965_write_fence_reg(obj, pipelined);
1941                 break;
1942         case 3:
1943                 ret = i915_write_fence_reg(obj, pipelined);
1944                 break;
1945         case 2:
1946                 ret = i830_write_fence_reg(obj, pipelined);
1947                 break;
1948         }
1949
1950         return ret;
1951 }
1952
1953 static int
1954 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1955     unsigned alignment, bool map_and_fenceable)
1956 {
1957         struct drm_device *dev;
1958         struct drm_i915_private *dev_priv;
1959         struct drm_mm_node *free_space;
1960         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1961         bool mappable, fenceable;
1962         int ret;
1963
1964         dev = obj->base.dev;
1965         dev_priv = dev->dev_private;
1966
1967         if (obj->madv != I915_MADV_WILLNEED) {
1968                 DRM_ERROR("Attempting to bind a purgeable object\n");
1969                 return (-EINVAL);
1970         }
1971
1972         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1973             obj->tiling_mode);
1974         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1975             obj->tiling_mode);
1976         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1977             obj->base.size, obj->tiling_mode);
1978         if (alignment == 0)
1979                 alignment = map_and_fenceable ? fence_alignment :
1980                     unfenced_alignment;
1981         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1982                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1983                 return (-EINVAL);
1984         }
1985
1986         size = map_and_fenceable ? fence_size : obj->base.size;
1987
1988         /* If the object is bigger than the entire aperture, reject it early
1989          * before evicting everything in a vain attempt to find space.
1990          */
1991         if (obj->base.size > (map_and_fenceable ?
1992             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1993                 DRM_ERROR(
1994 "Attempting to bind an object larger than the aperture\n");
1995                 return (-E2BIG);
1996         }
1997
1998  search_free:
1999         if (map_and_fenceable)
2000                 free_space = drm_mm_search_free_in_range(
2001                     &dev_priv->mm.gtt_space, size, alignment, 0,
2002                     dev_priv->mm.gtt_mappable_end, 0);
2003         else
2004                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2005                     size, alignment, 0);
2006         if (free_space != NULL) {
2007                 int color = 0;
2008                 if (map_and_fenceable)
2009                         obj->gtt_space = drm_mm_get_block_range_generic(
2010                             free_space, size, alignment, color, 0,
2011                             dev_priv->mm.gtt_mappable_end, 1);
2012                 else
2013                         obj->gtt_space = drm_mm_get_block_generic(free_space,
2014                             size, alignment, color, 1);
2015         }
2016         if (obj->gtt_space == NULL) {
2017                 ret = i915_gem_evict_something(dev, size, alignment,
2018                     map_and_fenceable);
2019                 if (ret != 0)
2020                         return (ret);
2021                 goto search_free;
2022         }
2023
2024         /*
2025          * NOTE: i915_gem_object_get_pages_gtt() cannot
2026          *       return ENOMEM, since we used VM_ALLOC_RETRY.
2027          */
2028         ret = i915_gem_object_get_pages_gtt(obj, 0);
2029         if (ret != 0) {
2030                 drm_mm_put_block(obj->gtt_space);
2031                 obj->gtt_space = NULL;
2032                 return (ret);
2033         }
2034
2035         i915_gem_gtt_bind_object(obj, obj->cache_level);
2036         if (ret != 0) {
2037                 i915_gem_object_put_pages_gtt(obj);
2038                 drm_mm_put_block(obj->gtt_space);
2039                 obj->gtt_space = NULL;
2040                 if (i915_gem_evict_everything(dev))
2041                         return (ret);
2042                 goto search_free;
2043         }
2044
2045         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2046         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2047
2048         obj->gtt_offset = obj->gtt_space->start;
2049
2050         fenceable =
2051                 obj->gtt_space->size == fence_size &&
2052                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2053
2054         mappable =
2055                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2056         obj->map_and_fenceable = mappable && fenceable;
2057
2058         return (0);
2059 }
2060
2061 void
2062 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2063 {
2064
2065         /* If we don't have a page list set up, then we're not pinned
2066          * to GPU, and we can ignore the cache flush because it'll happen
2067          * again at bind time.
2068          */
2069         if (obj->pages == NULL)
2070                 return;
2071
2072         /* If the GPU is snooping the contents of the CPU cache,
2073          * we do not need to manually clear the CPU cache lines.  However,
2074          * the caches are only snooped when the render cache is
2075          * flushed/invalidated.  As we always have to emit invalidations
2076          * and flushes when moving into and out of the RENDER domain, correct
2077          * snooping behaviour occurs naturally as the result of our domain
2078          * tracking.
2079          */
2080         if (obj->cache_level != I915_CACHE_NONE)
2081                 return;
2082
2083         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2084 }
2085
2086 /** Flushes the GTT write domain for the object if it's dirty. */
2087 static void
2088 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2089 {
2090         uint32_t old_write_domain;
2091
2092         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2093                 return;
2094
2095         /* No actual flushing is required for the GTT write domain.  Writes
2096          * to it immediately go to main memory as far as we know, so there's
2097          * no chipset flush.  It also doesn't land in render cache.
2098          *
2099          * However, we do have to enforce the order so that all writes through
2100          * the GTT land before any writes to the device, such as updates to
2101          * the GATT itself.
2102          */
2103         cpu_sfence();
2104
2105         old_write_domain = obj->base.write_domain;
2106         obj->base.write_domain = 0;
2107 }
2108
2109 /** Flushes the CPU write domain for the object if it's dirty. */
2110 static void
2111 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2112 {
2113         uint32_t old_write_domain;
2114
2115         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2116                 return;
2117
2118         i915_gem_clflush_object(obj);
2119         intel_gtt_chipset_flush();
2120         old_write_domain = obj->base.write_domain;
2121         obj->base.write_domain = 0;
2122 }
2123
2124 static int
2125 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2126 {
2127
2128         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2129                 return (0);
2130         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
2131 }
2132
2133 /**
2134  * Moves a single object to the GTT read, and possibly write domain.
2135  *
2136  * This function returns when the move is complete, including waiting on
2137  * flushes to occur.
2138  */
2139 int
2140 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2141 {
2142         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2143         uint32_t old_write_domain, old_read_domains;
2144         int ret;
2145
2146         /* Not valid to be called on unbound objects. */
2147         if (obj->gtt_space == NULL)
2148                 return -EINVAL;
2149
2150         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2151                 return 0;
2152
2153         ret = i915_gem_object_flush_gpu_write_domain(obj);
2154         if (ret)
2155                 return ret;
2156
2157         ret = i915_gem_object_wait_rendering(obj, !write);
2158         if (ret)
2159                 return ret;
2160
2161         i915_gem_object_flush_cpu_write_domain(obj);
2162
2163         old_write_domain = obj->base.write_domain;
2164         old_read_domains = obj->base.read_domains;
2165
2166         /* It should now be out of any other write domains, and we can update
2167          * the domain values for our changes.
2168          */
2169         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2170         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2171         if (write) {
2172                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2173                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2174                 obj->dirty = 1;
2175         }
2176
2177         /* And bump the LRU for this access */
2178         if (i915_gem_object_is_inactive(obj))
2179                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2180
2181         return 0;
2182 }
2183
2184 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2185                                     enum i915_cache_level cache_level)
2186 {
2187         struct drm_device *dev = obj->base.dev;
2188         drm_i915_private_t *dev_priv = dev->dev_private;
2189         int ret;
2190
2191         if (obj->cache_level == cache_level)
2192                 return 0;
2193
2194         if (obj->pin_count) {
2195                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2196                 return -EBUSY;
2197         }
2198
2199         if (obj->gtt_space) {
2200                 ret = i915_gem_object_finish_gpu(obj);
2201                 if (ret != 0)
2202                         return (ret);
2203
2204                 i915_gem_object_finish_gtt(obj);
2205
2206                 /* Before SandyBridge, you could not use tiling or fence
2207                  * registers with snooped memory, so relinquish any fences
2208                  * currently pointing to our region in the aperture.
2209                  */
2210                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2211                         ret = i915_gem_object_put_fence(obj);
2212                         if (ret)
2213                                 return ret;
2214                 }
2215
2216                 if (obj->has_global_gtt_mapping)
2217                         i915_gem_gtt_bind_object(obj, cache_level);
2218                 if (obj->has_aliasing_ppgtt_mapping)
2219                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2220                                                obj, cache_level);
2221         }
2222
2223         if (cache_level == I915_CACHE_NONE) {
2224                 u32 old_read_domains, old_write_domain;
2225
2226                 /* If we're coming from LLC cached, then we haven't
2227                  * actually been tracking whether the data is in the
2228                  * CPU cache or not, since we only allow one bit set
2229                  * in obj->write_domain and have been skipping the clflushes.
2230                  * Just set it to the CPU cache for now.
2231                  */
2232                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
2233                     ("obj %p in CPU write domain", obj));
2234                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
2235                     ("obj %p in CPU read domain", obj));
2236
2237                 old_read_domains = obj->base.read_domains;
2238                 old_write_domain = obj->base.write_domain;
2239
2240                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2241                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2242
2243         }
2244
2245         obj->cache_level = cache_level;
2246         return 0;
2247 }
2248
2249 /*
2250  * Prepare buffer for display plane (scanout, cursors, etc).
2251  * Can be called from an uninterruptible phase (modesetting) and allows
2252  * any flushes to be pipelined (for pageflips).
2253  */
2254 int
2255 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2256                                      u32 alignment,
2257                                      struct intel_ring_buffer *pipelined)
2258 {
2259         u32 old_read_domains, old_write_domain;
2260         int ret;
2261
2262         ret = i915_gem_object_flush_gpu_write_domain(obj);
2263         if (ret)
2264                 return ret;
2265
2266         if (pipelined != obj->ring) {
2267                 ret = i915_gem_object_sync(obj, pipelined);
2268                 if (ret)
2269                         return ret;
2270         }
2271
2272         /* The display engine is not coherent with the LLC cache on gen6.  As
2273          * a result, we make sure that the pinning that is about to occur is
2274          * done with uncached PTEs. This is lowest common denominator for all
2275          * chipsets.
2276          *
2277          * However for gen6+, we could do better by using the GFDT bit instead
2278          * of uncaching, which would allow us to flush all the LLC-cached data
2279          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2280          */
2281         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2282         if (ret)
2283                 return ret;
2284
2285         /* As the user may map the buffer once pinned in the display plane
2286          * (e.g. libkms for the bootup splash), we have to ensure that we
2287          * always use map_and_fenceable for all scanout buffers.
2288          */
2289         ret = i915_gem_object_pin(obj, alignment, true);
2290         if (ret)
2291                 return ret;
2292
2293         i915_gem_object_flush_cpu_write_domain(obj);
2294
2295         old_write_domain = obj->base.write_domain;
2296         old_read_domains = obj->base.read_domains;
2297
2298         /* It should now be out of any other write domains, and we can update
2299          * the domain values for our changes.
2300          */
2301         obj->base.write_domain = 0;
2302         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2303
2304         return 0;
2305 }
2306
2307 int
2308 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2309 {
2310         int ret;
2311
2312         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2313                 return 0;
2314
2315         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2316                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2317                 if (ret)
2318                         return ret;
2319         }
2320
2321         ret = i915_gem_object_wait_rendering(obj, false);
2322         if (ret)
2323                 return ret;
2324
2325         /* Ensure that we invalidate the GPU's caches and TLBs. */
2326         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2327         return 0;
2328 }
2329
2330 /**
2331  * Moves a single object to the CPU read, and possibly write domain.
2332  *
2333  * This function returns when the move is complete, including waiting on
2334  * flushes to occur.
2335  */
2336 int
2337 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2338 {
2339         uint32_t old_write_domain, old_read_domains;
2340         int ret;
2341
2342         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2343                 return 0;
2344
2345         ret = i915_gem_object_flush_gpu_write_domain(obj);
2346         if (ret)
2347                 return ret;
2348
2349         ret = i915_gem_object_wait_rendering(obj, !write);
2350         if (ret)
2351                 return ret;
2352
2353         i915_gem_object_flush_gtt_write_domain(obj);
2354
2355         old_write_domain = obj->base.write_domain;
2356         old_read_domains = obj->base.read_domains;
2357
2358         /* Flush the CPU cache if it's still invalid. */
2359         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2360                 i915_gem_clflush_object(obj);
2361
2362                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2363         }
2364
2365         /* It should now be out of any other write domains, and we can update
2366          * the domain values for our changes.
2367          */
2368         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2369
2370         /* If we're writing through the CPU, then the GPU read domains will
2371          * need to be invalidated at next use.
2372          */
2373         if (write) {
2374                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2375                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2376         }
2377
2378         return 0;
2379 }
2380
2381 /* Throttle our rendering by waiting until the ring has completed our requests
2382  * emitted over 20 msec ago.
2383  *
2384  * Note that if we were to use the current jiffies each time around the loop,
2385  * we wouldn't escape the function with any frames outstanding if the time to
2386  * render a frame was over 20ms.
2387  *
2388  * This should get us reasonable parallelism between CPU and GPU but also
2389  * relatively low latency when blocking on a particular request to finish.
2390  */
2391 static int
2392 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2393 {
2394         struct drm_i915_private *dev_priv = dev->dev_private;
2395         struct drm_i915_file_private *file_priv = file->driver_priv;
2396         unsigned long recent_enough = ticks - (20 * hz / 1000);
2397         struct drm_i915_gem_request *request;
2398         struct intel_ring_buffer *ring = NULL;
2399         u32 seqno = 0;
2400         int ret;
2401
2402         if (atomic_read(&dev_priv->mm.wedged))
2403                 return -EIO;
2404
2405         spin_lock(&file_priv->mm.lock);
2406         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2407                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2408                         break;
2409
2410                 ring = request->ring;
2411                 seqno = request->seqno;
2412         }
2413         spin_unlock(&file_priv->mm.lock);
2414
2415         if (seqno == 0)
2416                 return 0;
2417
2418         ret = __wait_seqno(ring, seqno, true, NULL);
2419
2420         if (ret == 0)
2421                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2422
2423         return ret;
2424 }
2425
2426 int
2427 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
2428      bool map_and_fenceable)
2429 {
2430         struct drm_device *dev;
2431         struct drm_i915_private *dev_priv;
2432         int ret;
2433
2434         dev = obj->base.dev;
2435         dev_priv = dev->dev_private;
2436
2437         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
2438             ("Max pin count"));
2439
2440         if (obj->gtt_space != NULL) {
2441                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
2442                     (map_and_fenceable && !obj->map_and_fenceable)) {
2443                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
2444                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2445                              " obj->map_and_fenceable=%d\n",
2446                              obj->gtt_offset, alignment,
2447                              map_and_fenceable,
2448                              obj->map_and_fenceable);
2449                         ret = i915_gem_object_unbind(obj);
2450                         if (ret != 0)
2451                                 return (ret);
2452                 }
2453         }
2454
2455         if (obj->gtt_space == NULL) {
2456                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
2457                     map_and_fenceable);
2458                 if (ret)
2459                         return (ret);
2460         }
2461
2462         if (obj->pin_count++ == 0 && !obj->active)
2463                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2464         obj->pin_mappable |= map_and_fenceable;
2465
2466 #if 1
2467         KIB_NOTYET();
2468 #else
2469         WARN_ON(i915_verify_lists(dev));
2470 #endif
2471         return (0);
2472 }
2473
2474 void
2475 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
2476 {
2477         struct drm_device *dev;
2478         drm_i915_private_t *dev_priv;
2479
2480         dev = obj->base.dev;
2481         dev_priv = dev->dev_private;
2482
2483 #if 1
2484         KIB_NOTYET();
2485 #else
2486         WARN_ON(i915_verify_lists(dev));
2487 #endif
2488
2489         KASSERT(obj->pin_count != 0, ("zero pin count"));
2490         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
2491
2492         if (--obj->pin_count == 0) {
2493                 if (!obj->active)
2494                         list_move_tail(&obj->mm_list,
2495                             &dev_priv->mm.inactive_list);
2496                 obj->pin_mappable = false;
2497         }
2498 #if 1
2499         KIB_NOTYET();
2500 #else
2501         WARN_ON(i915_verify_lists(dev));
2502 #endif
2503 }
2504
2505 int
2506 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2507     struct drm_file *file)
2508 {
2509         struct drm_i915_gem_pin *args;
2510         struct drm_i915_gem_object *obj;
2511         struct drm_gem_object *gobj;
2512         int ret;
2513
2514         args = data;
2515
2516         ret = i915_mutex_lock_interruptible(dev);
2517         if (ret != 0)
2518                 return ret;
2519
2520         gobj = drm_gem_object_lookup(dev, file, args->handle);
2521         if (gobj == NULL) {
2522                 ret = -ENOENT;
2523                 goto unlock;
2524         }
2525         obj = to_intel_bo(gobj);
2526
2527         if (obj->madv != I915_MADV_WILLNEED) {
2528                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2529                 ret = -EINVAL;
2530                 goto out;
2531         }
2532
2533         if (obj->pin_filp != NULL && obj->pin_filp != file) {
2534                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2535                     args->handle);
2536                 ret = -EINVAL;
2537                 goto out;
2538         }
2539
2540         obj->user_pin_count++;
2541         obj->pin_filp = file;
2542         if (obj->user_pin_count == 1) {
2543                 ret = i915_gem_object_pin(obj, args->alignment, true);
2544                 if (ret != 0)
2545                         goto out;
2546         }
2547
2548         /* XXX - flush the CPU caches for pinned objects
2549          * as the X server doesn't manage domains yet
2550          */
2551         i915_gem_object_flush_cpu_write_domain(obj);
2552         args->offset = obj->gtt_offset;
2553 out:
2554         drm_gem_object_unreference(&obj->base);
2555 unlock:
2556         DRM_UNLOCK(dev);
2557         return (ret);
2558 }
2559
2560 int
2561 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2562     struct drm_file *file)
2563 {
2564         struct drm_i915_gem_pin *args;
2565         struct drm_i915_gem_object *obj;
2566         int ret;
2567
2568         args = data;
2569         ret = i915_mutex_lock_interruptible(dev);
2570         if (ret != 0)
2571                 return (ret);
2572
2573         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2574         if (&obj->base == NULL) {
2575                 ret = -ENOENT;
2576                 goto unlock;
2577         }
2578
2579         if (obj->pin_filp != file) {
2580                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2581                     args->handle);
2582                 ret = -EINVAL;
2583                 goto out;
2584         }
2585         obj->user_pin_count--;
2586         if (obj->user_pin_count == 0) {
2587                 obj->pin_filp = NULL;
2588                 i915_gem_object_unpin(obj);
2589         }
2590
2591 out:
2592         drm_gem_object_unreference(&obj->base);
2593 unlock:
2594         DRM_UNLOCK(dev);
2595         return (ret);
2596 }
2597
2598 int
2599 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2600                     struct drm_file *file)
2601 {
2602         struct drm_i915_gem_busy *args = data;
2603         struct drm_i915_gem_object *obj;
2604         int ret;
2605
2606         ret = i915_mutex_lock_interruptible(dev);
2607         if (ret)
2608                 return ret;
2609
2610         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2611         if (&obj->base == NULL) {
2612                 ret = -ENOENT;
2613                 goto unlock;
2614         }
2615
2616         /* Count all active objects as busy, even if they are currently not used
2617          * by the gpu. Users of this interface expect objects to eventually
2618          * become non-busy without any further actions, therefore emit any
2619          * necessary flushes here.
2620          */
2621         ret = i915_gem_object_flush_active(obj);
2622
2623         args->busy = obj->active;
2624         if (obj->ring) {
2625                 args->busy |= intel_ring_flag(obj->ring) << 17;
2626         }
2627
2628         drm_gem_object_unreference(&obj->base);
2629 unlock:
2630         DRM_UNLOCK(dev);
2631         return ret;
2632 }
2633
2634 int
2635 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2636     struct drm_file *file_priv)
2637 {
2638
2639         return (i915_gem_ring_throttle(dev, file_priv));
2640 }
2641
2642 int
2643 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
2644                        struct drm_file *file_priv)
2645 {
2646         struct drm_i915_gem_madvise *args = data;
2647         struct drm_i915_gem_object *obj;
2648         int ret;
2649
2650         switch (args->madv) {
2651         case I915_MADV_DONTNEED:
2652         case I915_MADV_WILLNEED:
2653             break;
2654         default:
2655             return -EINVAL;
2656         }
2657
2658         ret = i915_mutex_lock_interruptible(dev);
2659         if (ret)
2660                 return ret;
2661
2662         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
2663         if (&obj->base == NULL) {
2664                 ret = -ENOENT;
2665                 goto unlock;
2666         }
2667
2668         if (obj->pin_count) {
2669                 ret = -EINVAL;
2670                 goto out;
2671         }
2672
2673         if (obj->madv != __I915_MADV_PURGED)
2674                 obj->madv = args->madv;
2675
2676         /* if the object is no longer attached, discard its backing storage */
2677         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
2678                 i915_gem_object_truncate(obj);
2679
2680         args->retained = obj->madv != __I915_MADV_PURGED;
2681
2682 out:
2683         drm_gem_object_unreference(&obj->base);
2684 unlock:
2685         DRM_UNLOCK(dev);
2686         return ret;
2687 }
2688
2689 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
2690                                                   size_t size)
2691 {
2692         struct drm_i915_private *dev_priv;
2693         struct drm_i915_gem_object *obj;
2694
2695         dev_priv = dev->dev_private;
2696
2697         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
2698
2699         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
2700                 drm_free(obj, DRM_I915_GEM);
2701                 return (NULL);
2702         }
2703
2704         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2705         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2706
2707         if (HAS_LLC(dev))
2708                 obj->cache_level = I915_CACHE_LLC;
2709         else
2710                 obj->cache_level = I915_CACHE_NONE;
2711         obj->base.driver_private = NULL;
2712         obj->fence_reg = I915_FENCE_REG_NONE;
2713         INIT_LIST_HEAD(&obj->mm_list);
2714         INIT_LIST_HEAD(&obj->gtt_list);
2715         INIT_LIST_HEAD(&obj->ring_list);
2716         INIT_LIST_HEAD(&obj->exec_list);
2717         INIT_LIST_HEAD(&obj->gpu_write_list);
2718         obj->madv = I915_MADV_WILLNEED;
2719         /* Avoid an unnecessary call to unbind on the first bind. */
2720         obj->map_and_fenceable = true;
2721
2722         i915_gem_info_add_obj(dev_priv, size);
2723
2724         return (obj);
2725 }
2726
2727 int i915_gem_init_object(struct drm_gem_object *obj)
2728 {
2729
2730         kprintf("i915_gem_init_object called\n");
2731         return (0);
2732 }
2733
2734 static void
2735 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
2736 {
2737         struct drm_device *dev;
2738         drm_i915_private_t *dev_priv;
2739         int ret;
2740
2741         dev = obj->base.dev;
2742         dev_priv = dev->dev_private;
2743
2744         ret = i915_gem_object_unbind(obj);
2745         if (ret == -ERESTART) {
2746                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
2747                 return;
2748         }
2749
2750         drm_gem_free_mmap_offset(&obj->base);
2751         drm_gem_object_release(&obj->base);
2752         i915_gem_info_remove_obj(dev_priv, obj->base.size);
2753
2754         drm_free(obj->bit_17, DRM_I915_GEM);
2755         drm_free(obj, DRM_I915_GEM);
2756 }
2757
2758 void
2759 i915_gem_free_object(struct drm_gem_object *gem_obj)
2760 {
2761         struct drm_i915_gem_object *obj;
2762         struct drm_device *dev;
2763
2764         obj = to_intel_bo(gem_obj);
2765         dev = obj->base.dev;
2766
2767         while (obj->pin_count > 0)
2768                 i915_gem_object_unpin(obj);
2769
2770         if (obj->phys_obj != NULL)
2771                 i915_gem_detach_phys_object(dev, obj);
2772
2773         i915_gem_free_object_tail(obj);
2774 }
2775
2776 int
2777 i915_gem_do_init(struct drm_device *dev, unsigned long start,
2778     unsigned long mappable_end, unsigned long end)
2779 {
2780         drm_i915_private_t *dev_priv;
2781         unsigned long mappable;
2782         int error;
2783
2784         dev_priv = dev->dev_private;
2785         mappable = min(end, mappable_end) - start;
2786
2787         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
2788
2789         dev_priv->mm.gtt_start = start;
2790         dev_priv->mm.gtt_mappable_end = mappable_end;
2791         dev_priv->mm.gtt_end = end;
2792         dev_priv->mm.gtt_total = end - start;
2793         dev_priv->mm.mappable_gtt_total = mappable;
2794
2795         /* Take over this portion of the GTT */
2796         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
2797         device_printf(dev->dev,
2798             "taking over the fictitious range 0x%lx-0x%lx\n",
2799             dev->agp->base + start, dev->agp->base + start + mappable);
2800         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
2801             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2802         return (error);
2803 }
2804
2805 int
2806 i915_gem_idle(struct drm_device *dev)
2807 {
2808         drm_i915_private_t *dev_priv;
2809         int ret;
2810
2811         dev_priv = dev->dev_private;
2812         if (dev_priv->mm.suspended)
2813                 return (0);
2814
2815         ret = i915_gpu_idle(dev);
2816         if (ret != 0)
2817                 return (ret);
2818
2819         /* Under UMS, be paranoid and evict. */
2820         if (!drm_core_check_feature(dev, DRIVER_MODESET))
2821                 i915_gem_evict_everything(dev);
2822
2823         i915_gem_reset_fences(dev);
2824
2825         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2826          * We need to replace this with a semaphore, or something.
2827          * And not confound mm.suspended!
2828          */
2829         dev_priv->mm.suspended = 1;
2830         del_timer_sync(&dev_priv->hangcheck_timer);
2831
2832         i915_kernel_lost_context(dev);
2833         i915_gem_cleanup_ringbuffer(dev);
2834
2835         /* Cancel the retire work handler, which should be idle now. */
2836         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2837
2838         return (ret);
2839 }
2840
2841 void i915_gem_l3_remap(struct drm_device *dev)
2842 {
2843         drm_i915_private_t *dev_priv = dev->dev_private;
2844         u32 misccpctl;
2845         int i;
2846
2847         if (!HAS_L3_GPU_CACHE(dev))
2848                 return;
2849
2850         if (!dev_priv->l3_parity.remap_info)
2851                 return;
2852
2853         misccpctl = I915_READ(GEN7_MISCCPCTL);
2854         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
2855         POSTING_READ(GEN7_MISCCPCTL);
2856
2857         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
2858                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
2859                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
2860                         DRM_DEBUG("0x%x was already programmed to %x\n",
2861                                   GEN7_L3LOG_BASE + i, remap);
2862                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
2863                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
2864                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
2865         }
2866
2867         /* Make sure all the writes land before disabling dop clock gating */
2868         POSTING_READ(GEN7_L3LOG_BASE);
2869
2870         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
2871 }
2872
2873 void
2874 i915_gem_init_swizzling(struct drm_device *dev)
2875 {
2876         drm_i915_private_t *dev_priv;
2877
2878         dev_priv = dev->dev_private;
2879
2880         if (INTEL_INFO(dev)->gen < 5 ||
2881             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
2882                 return;
2883
2884         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
2885                                  DISP_TILE_SURFACE_SWIZZLING);
2886
2887         if (IS_GEN5(dev))
2888                 return;
2889
2890         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
2891         if (IS_GEN6(dev))
2892                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
2893         else
2894                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
2895 }
2896
2897 static bool
2898 intel_enable_blt(struct drm_device *dev)
2899 {
2900         int revision;
2901
2902         if (!HAS_BLT(dev))
2903                 return false;
2904
2905         /* The blitter was dysfunctional on early prototypes */
2906         revision = pci_read_config(dev->dev, PCIR_REVID, 1);
2907         if (IS_GEN6(dev) && revision < 8) {
2908                 DRM_INFO("BLT not supported on this pre-production hardware;"
2909                          " graphics performance will be degraded.\n");
2910                 return false;
2911         }
2912
2913         return true;
2914 }
2915
2916 int
2917 i915_gem_init_hw(struct drm_device *dev)
2918 {
2919         drm_i915_private_t *dev_priv = dev->dev_private;
2920         int ret;
2921
2922         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
2923                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
2924
2925         i915_gem_l3_remap(dev);
2926
2927         i915_gem_init_swizzling(dev);
2928
2929         ret = intel_init_render_ring_buffer(dev);
2930         if (ret)
2931                 return ret;
2932
2933         if (HAS_BSD(dev)) {
2934                 ret = intel_init_bsd_ring_buffer(dev);
2935                 if (ret)
2936                         goto cleanup_render_ring;
2937         }
2938
2939         if (intel_enable_blt(dev)) {
2940                 ret = intel_init_blt_ring_buffer(dev);
2941                 if (ret)
2942                         goto cleanup_bsd_ring;
2943         }
2944
2945         dev_priv->next_seqno = 1;
2946
2947         /*
2948          * XXX: There was some w/a described somewhere suggesting loading
2949          * contexts before PPGTT.
2950          */
2951 #if 0   /* XXX: HW context support */
2952         i915_gem_context_init(dev);
2953 #endif
2954         i915_gem_init_ppgtt(dev);
2955
2956         return 0;
2957
2958 cleanup_bsd_ring:
2959         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
2960 cleanup_render_ring:
2961         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
2962         return ret;
2963 }
2964
2965 void
2966 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2967 {
2968         drm_i915_private_t *dev_priv;
2969         int i;
2970
2971         dev_priv = dev->dev_private;
2972         for (i = 0; i < I915_NUM_RINGS; i++)
2973                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
2974 }
2975
2976 int
2977 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2978                        struct drm_file *file_priv)
2979 {
2980         drm_i915_private_t *dev_priv = dev->dev_private;
2981         int ret;
2982
2983         if (drm_core_check_feature(dev, DRIVER_MODESET))
2984                 return 0;
2985
2986         if (atomic_read(&dev_priv->mm.wedged)) {
2987                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2988                 atomic_set(&dev_priv->mm.wedged, 0);
2989         }
2990
2991         DRM_LOCK(dev);
2992         dev_priv->mm.suspended = 0;
2993
2994         ret = i915_gem_init_hw(dev);
2995         if (ret != 0) {
2996                 DRM_UNLOCK(dev);
2997                 return ret;
2998         }
2999
3000         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
3001         DRM_UNLOCK(dev);
3002
3003         ret = drm_irq_install(dev);
3004         if (ret)
3005                 goto cleanup_ringbuffer;
3006
3007         return 0;
3008
3009 cleanup_ringbuffer:
3010         DRM_LOCK(dev);
3011         i915_gem_cleanup_ringbuffer(dev);
3012         dev_priv->mm.suspended = 1;
3013         DRM_UNLOCK(dev);
3014
3015         return ret;
3016 }
3017
3018 int
3019 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3020     struct drm_file *file_priv)
3021 {
3022
3023         if (drm_core_check_feature(dev, DRIVER_MODESET))
3024                 return 0;
3025
3026         drm_irq_uninstall(dev);
3027         return (i915_gem_idle(dev));
3028 }
3029
3030 void
3031 i915_gem_lastclose(struct drm_device *dev)
3032 {
3033         int ret;
3034
3035         if (drm_core_check_feature(dev, DRIVER_MODESET))
3036                 return;
3037
3038         ret = i915_gem_idle(dev);
3039         if (ret != 0)
3040                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3041 }
3042
3043 static void
3044 init_ring_lists(struct intel_ring_buffer *ring)
3045 {
3046
3047         INIT_LIST_HEAD(&ring->active_list);
3048         INIT_LIST_HEAD(&ring->request_list);
3049         INIT_LIST_HEAD(&ring->gpu_write_list);
3050 }
3051
3052 void
3053 i915_gem_load(struct drm_device *dev)
3054 {
3055         int i;
3056         drm_i915_private_t *dev_priv = dev->dev_private;
3057
3058         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3059         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3060         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3061         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3062         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3063         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3064         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3065         for (i = 0; i < I915_NUM_RINGS; i++)
3066                 init_ring_lists(&dev_priv->ring[i]);
3067         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3068                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3069         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3070                           i915_gem_retire_work_handler);
3071         init_completion(&dev_priv->error_completion);
3072
3073         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3074         if (IS_GEN3(dev)) {
3075                 I915_WRITE(MI_ARB_STATE,
3076                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3077         }
3078
3079         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3080
3081         /* Old X drivers will take 0-2 for front, back, depth buffers */
3082         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3083                 dev_priv->fence_reg_start = 3;
3084
3085         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3086                 dev_priv->num_fence_regs = 16;
3087         else
3088                 dev_priv->num_fence_regs = 8;
3089
3090         /* Initialize fence registers to zero */
3091         i915_gem_reset_fences(dev);
3092
3093         i915_gem_detect_bit_6_swizzle(dev);
3094
3095         dev_priv->mm.interruptible = true;
3096
3097         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
3098             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
3099 }
3100
3101 static int
3102 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
3103 {
3104         drm_i915_private_t *dev_priv;
3105         struct drm_i915_gem_phys_object *phys_obj;
3106         int ret;
3107
3108         dev_priv = dev->dev_private;
3109         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
3110                 return (0);
3111
3112         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3113             M_WAITOK | M_ZERO);
3114
3115         phys_obj->id = id;
3116
3117         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3118         if (phys_obj->handle == NULL) {
3119                 ret = -ENOMEM;
3120                 goto free_obj;
3121         }
3122         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3123             size / PAGE_SIZE, PAT_WRITE_COMBINING);
3124
3125         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3126
3127         return (0);
3128
3129 free_obj:
3130         drm_free(phys_obj, DRM_I915_GEM);
3131         return (ret);
3132 }
3133
3134 static void
3135 i915_gem_free_phys_object(struct drm_device *dev, int id)
3136 {
3137         drm_i915_private_t *dev_priv;
3138         struct drm_i915_gem_phys_object *phys_obj;
3139
3140         dev_priv = dev->dev_private;
3141         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3142                 return;
3143
3144         phys_obj = dev_priv->mm.phys_objs[id - 1];
3145         if (phys_obj->cur_obj != NULL)
3146                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3147
3148         drm_pci_free(dev, phys_obj->handle);
3149         drm_free(phys_obj, DRM_I915_GEM);
3150         dev_priv->mm.phys_objs[id - 1] = NULL;
3151 }
3152
3153 void
3154 i915_gem_free_all_phys_object(struct drm_device *dev)
3155 {
3156         int i;
3157
3158         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3159                 i915_gem_free_phys_object(dev, i);
3160 }
3161
3162 void
3163 i915_gem_detach_phys_object(struct drm_device *dev,
3164     struct drm_i915_gem_object *obj)
3165 {
3166         vm_page_t m;
3167         struct sf_buf *sf;
3168         char *vaddr, *dst;
3169         int i, page_count;
3170
3171         if (obj->phys_obj == NULL)
3172                 return;
3173         vaddr = obj->phys_obj->handle->vaddr;
3174
3175         page_count = obj->base.size / PAGE_SIZE;
3176         VM_OBJECT_LOCK(obj->base.vm_obj);
3177         for (i = 0; i < page_count; i++) {
3178                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3179                 if (m == NULL)
3180                         continue; /* XXX */
3181
3182                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3183                 sf = sf_buf_alloc(m);
3184                 if (sf != NULL) {
3185                         dst = (char *)sf_buf_kva(sf);
3186                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3187                         sf_buf_free(sf);
3188                 }
3189                 drm_clflush_pages(&m, 1);
3190
3191                 VM_OBJECT_LOCK(obj->base.vm_obj);
3192                 vm_page_reference(m);
3193                 vm_page_dirty(m);
3194                 vm_page_busy_wait(m, FALSE, "i915gem");
3195                 vm_page_unwire(m, 0);
3196                 vm_page_wakeup(m);
3197                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3198         }
3199         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3200         intel_gtt_chipset_flush();
3201
3202         obj->phys_obj->cur_obj = NULL;
3203         obj->phys_obj = NULL;
3204 }
3205
3206 int
3207 i915_gem_attach_phys_object(struct drm_device *dev,
3208                             struct drm_i915_gem_object *obj,
3209                             int id,
3210                             int align)
3211 {
3212         drm_i915_private_t *dev_priv;
3213         vm_page_t m;
3214         struct sf_buf *sf;
3215         char *dst, *src;
3216         int i, page_count, ret;
3217
3218         if (id > I915_MAX_PHYS_OBJECT)
3219                 return (-EINVAL);
3220
3221         if (obj->phys_obj != NULL) {
3222                 if (obj->phys_obj->id == id)
3223                         return (0);
3224                 i915_gem_detach_phys_object(dev, obj);
3225         }
3226
3227         dev_priv = dev->dev_private;
3228         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3229                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3230                 if (ret != 0) {
3231                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3232                                   id, obj->base.size);
3233                         return (ret);
3234                 }
3235         }
3236
3237         /* bind to the object */
3238         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3239         obj->phys_obj->cur_obj = obj;
3240
3241         page_count = obj->base.size / PAGE_SIZE;
3242
3243         VM_OBJECT_LOCK(obj->base.vm_obj);
3244         ret = 0;
3245         for (i = 0; i < page_count; i++) {
3246                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3247                 if (m == NULL) {
3248                         ret = -EIO;
3249                         break;
3250                 }
3251                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3252                 sf = sf_buf_alloc(m);
3253                 src = (char *)sf_buf_kva(sf);
3254                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3255                 memcpy(dst, src, PAGE_SIZE);
3256                 sf_buf_free(sf);
3257
3258                 VM_OBJECT_LOCK(obj->base.vm_obj);
3259
3260                 vm_page_reference(m);
3261                 vm_page_busy_wait(m, FALSE, "i915gem");
3262                 vm_page_unwire(m, 0);
3263                 vm_page_wakeup(m);
3264                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3265         }
3266         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3267
3268         return (0);
3269 }
3270
3271 static int
3272 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3273     uint64_t data_ptr, uint64_t offset, uint64_t size,
3274     struct drm_file *file_priv)
3275 {
3276         char *user_data, *vaddr;
3277         int ret;
3278
3279         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3280         user_data = (char *)(uintptr_t)data_ptr;
3281
3282         if (copyin_nofault(user_data, vaddr, size) != 0) {
3283                 /* The physical object once assigned is fixed for the lifetime
3284                  * of the obj, so we can safely drop the lock and continue
3285                  * to access vaddr.
3286                  */
3287                 DRM_UNLOCK(dev);
3288                 ret = -copyin(user_data, vaddr, size);
3289                 DRM_LOCK(dev);
3290                 if (ret != 0)
3291                         return (ret);
3292         }
3293
3294         intel_gtt_chipset_flush();
3295         return (0);
3296 }
3297
3298 void
3299 i915_gem_release(struct drm_device *dev, struct drm_file *file)
3300 {
3301         struct drm_i915_file_private *file_priv;
3302         struct drm_i915_gem_request *request;
3303
3304         file_priv = file->driver_priv;
3305
3306         /* Clean up our request list when the client is going away, so that
3307          * later retire_requests won't dereference our soon-to-be-gone
3308          * file_priv.
3309          */
3310         spin_lock(&file_priv->mm.lock);
3311         while (!list_empty(&file_priv->mm.request_list)) {
3312                 request = list_first_entry(&file_priv->mm.request_list,
3313                                            struct drm_i915_gem_request,
3314                                            client_list);
3315                 list_del(&request->client_list);
3316                 request->file_priv = NULL;
3317         }
3318         spin_unlock(&file_priv->mm.lock);
3319 }
3320
3321 static int
3322 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
3323     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
3324     struct drm_file *file)
3325 {
3326         vm_object_t vm_obj;
3327         vm_page_t m;
3328         struct sf_buf *sf;
3329         vm_offset_t mkva;
3330         vm_pindex_t obj_pi;
3331         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
3332
3333         if (obj->gtt_offset != 0 && rw == UIO_READ)
3334                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
3335         else
3336                 do_bit17_swizzling = 0;
3337
3338         obj->dirty = 1;
3339         vm_obj = obj->base.vm_obj;
3340         ret = 0;
3341
3342         VM_OBJECT_LOCK(vm_obj);
3343         vm_object_pip_add(vm_obj, 1);
3344         while (size > 0) {
3345                 obj_pi = OFF_TO_IDX(offset);
3346                 obj_po = offset & PAGE_MASK;
3347
3348                 m = i915_gem_wire_page(vm_obj, obj_pi);
3349                 VM_OBJECT_UNLOCK(vm_obj);
3350
3351                 sf = sf_buf_alloc(m);
3352                 mkva = sf_buf_kva(sf);
3353                 length = min(size, PAGE_SIZE - obj_po);
3354                 while (length > 0) {
3355                         if (do_bit17_swizzling &&
3356                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
3357                                 cnt = roundup2(obj_po + 1, 64);
3358                                 cnt = min(cnt - obj_po, length);
3359                                 swizzled_po = obj_po ^ 64;
3360                         } else {
3361                                 cnt = length;
3362                                 swizzled_po = obj_po;
3363                         }
3364                         if (rw == UIO_READ)
3365                                 ret = -copyout_nofault(
3366                                     (char *)mkva + swizzled_po,
3367                                     (void *)(uintptr_t)data_ptr, cnt);
3368                         else
3369                                 ret = -copyin_nofault(
3370                                     (void *)(uintptr_t)data_ptr,
3371                                     (char *)mkva + swizzled_po, cnt);
3372                         if (ret != 0)
3373                                 break;
3374                         data_ptr += cnt;
3375                         size -= cnt;
3376                         length -= cnt;
3377                         offset += cnt;
3378                         obj_po += cnt;
3379                 }
3380                 sf_buf_free(sf);
3381                 VM_OBJECT_LOCK(vm_obj);
3382                 if (rw == UIO_WRITE)
3383                         vm_page_dirty(m);
3384                 vm_page_reference(m);
3385                 vm_page_busy_wait(m, FALSE, "i915gem");
3386                 vm_page_unwire(m, 1);
3387                 vm_page_wakeup(m);
3388                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3389
3390                 if (ret != 0)
3391                         break;
3392         }
3393         vm_object_pip_wakeup(vm_obj);
3394         VM_OBJECT_UNLOCK(vm_obj);
3395
3396         return (ret);
3397 }
3398
3399 static int
3400 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
3401     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
3402 {
3403         vm_offset_t mkva;
3404         int ret;
3405
3406         /*
3407          * Pass the unaligned physical address and size to pmap_mapdev_attr()
3408          * so it can properly calculate whether an extra page needs to be
3409          * mapped or not to cover the requested range.  The function will
3410          * add the page offset into the returned mkva for us.
3411          */
3412         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
3413             offset, size, PAT_WRITE_COMBINING);
3414         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
3415         pmap_unmapdev(mkva, size);
3416         return (ret);
3417 }
3418
3419 static int
3420 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
3421     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
3422 {
3423         struct drm_i915_gem_object *obj;
3424         vm_page_t *ma;
3425         vm_offset_t start, end;
3426         int npages, ret;
3427
3428         if (size == 0)
3429                 return (0);
3430         start = trunc_page(data_ptr);
3431         end = round_page(data_ptr + size);
3432         npages = howmany(end - start, PAGE_SIZE);
3433         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
3434             M_ZERO);
3435         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
3436             (vm_offset_t)data_ptr, size,
3437             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
3438         if (npages == -1) {
3439                 ret = -EFAULT;
3440                 goto free_ma;
3441         }
3442
3443         ret = i915_mutex_lock_interruptible(dev);
3444         if (ret != 0)
3445                 goto unlocked;
3446
3447         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
3448         if (&obj->base == NULL) {
3449                 ret = -ENOENT;
3450                 goto unlock;
3451         }
3452         if (offset > obj->base.size || size > obj->base.size - offset) {
3453                 ret = -EINVAL;
3454                 goto out;
3455         }
3456
3457         if (rw == UIO_READ) {
3458                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3459                     UIO_READ, file);
3460         } else {
3461                 if (obj->phys_obj) {
3462                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
3463                             size, file);
3464                 } else if (obj->gtt_space &&
3465                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3466                         ret = i915_gem_object_pin(obj, 0, true);
3467                         if (ret != 0)
3468                                 goto out;
3469                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
3470                         if (ret != 0)
3471                                 goto out_unpin;
3472                         ret = i915_gem_object_put_fence(obj);
3473                         if (ret != 0)
3474                                 goto out_unpin;
3475                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
3476                             offset, file);
3477 out_unpin:
3478                         i915_gem_object_unpin(obj);
3479                 } else {
3480                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
3481                         if (ret != 0)
3482                                 goto out;
3483                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3484                             UIO_WRITE, file);
3485                 }
3486         }
3487 out:
3488         drm_gem_object_unreference(&obj->base);
3489 unlock:
3490         DRM_UNLOCK(dev);
3491 unlocked:
3492         vm_page_unhold_pages(ma, npages);
3493 free_ma:
3494         drm_free(ma, DRM_I915_GEM);
3495         return (ret);
3496 }
3497
3498 static int
3499 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
3500     vm_ooffset_t foff, struct ucred *cred, u_short *color)
3501 {
3502
3503         *color = 0; /* XXXKIB */
3504         return (0);
3505 }
3506
3507 int i915_intr_pf;
3508
3509 static int
3510 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
3511     vm_page_t *mres)
3512 {
3513         struct drm_gem_object *gem_obj;
3514         struct drm_i915_gem_object *obj;
3515         struct drm_device *dev;
3516         drm_i915_private_t *dev_priv;
3517         vm_page_t m, oldm;
3518         int cause, ret;
3519         bool write;
3520
3521         gem_obj = vm_obj->handle;
3522         obj = to_intel_bo(gem_obj);
3523         dev = obj->base.dev;
3524         dev_priv = dev->dev_private;
3525 #if 0
3526         write = (prot & VM_PROT_WRITE) != 0;
3527 #else
3528         write = true;
3529 #endif
3530         vm_object_pip_add(vm_obj, 1);
3531
3532         /*
3533          * Remove the placeholder page inserted by vm_fault() from the
3534          * object before dropping the object lock. If
3535          * i915_gem_release_mmap() is active in parallel on this gem
3536          * object, then it owns the drm device sx and might find the
3537          * placeholder already. Then, since the page is busy,
3538          * i915_gem_release_mmap() sleeps waiting for the busy state
3539          * of the page cleared. We will be not able to acquire drm
3540          * device lock until i915_gem_release_mmap() is able to make a
3541          * progress.
3542          */
3543         if (*mres != NULL) {
3544                 oldm = *mres;
3545                 vm_page_remove(oldm);
3546                 *mres = NULL;
3547         } else
3548                 oldm = NULL;
3549 retry:
3550         VM_OBJECT_UNLOCK(vm_obj);
3551 unlocked_vmobj:
3552         cause = ret = 0;
3553         m = NULL;
3554
3555         if (i915_intr_pf) {
3556                 ret = i915_mutex_lock_interruptible(dev);
3557                 if (ret != 0) {
3558                         cause = 10;
3559                         goto out;
3560                 }
3561         } else
3562                 DRM_LOCK(dev);
3563
3564         /*
3565          * Since the object lock was dropped, other thread might have
3566          * faulted on the same GTT address and instantiated the
3567          * mapping for the page.  Recheck.
3568          */
3569         VM_OBJECT_LOCK(vm_obj);
3570         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
3571         if (m != NULL) {
3572                 if ((m->flags & PG_BUSY) != 0) {
3573                         DRM_UNLOCK(dev);
3574 #if 0 /* XXX */
3575                         vm_page_sleep(m, "915pee");
3576 #endif
3577                         goto retry;
3578                 }
3579                 goto have_page;
3580         } else
3581                 VM_OBJECT_UNLOCK(vm_obj);
3582
3583         /* Now bind it into the GTT if needed */
3584         if (!obj->map_and_fenceable) {
3585                 ret = i915_gem_object_unbind(obj);
3586                 if (ret != 0) {
3587                         cause = 20;
3588                         goto unlock;
3589                 }
3590         }
3591         if (!obj->gtt_space) {
3592                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
3593                 if (ret != 0) {
3594                         cause = 30;
3595                         goto unlock;
3596                 }
3597
3598                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
3599                 if (ret != 0) {
3600                         cause = 40;
3601                         goto unlock;
3602                 }
3603         }
3604
3605         if (obj->tiling_mode == I915_TILING_NONE)
3606                 ret = i915_gem_object_put_fence(obj);
3607         else
3608                 ret = i915_gem_object_get_fence(obj, NULL);
3609         if (ret != 0) {
3610                 cause = 50;
3611                 goto unlock;
3612         }
3613
3614         if (i915_gem_object_is_inactive(obj))
3615                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3616
3617         obj->fault_mappable = true;
3618         VM_OBJECT_LOCK(vm_obj);
3619         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
3620             offset);
3621         if (m == NULL) {
3622                 cause = 60;
3623                 ret = -EFAULT;
3624                 goto unlock;
3625         }
3626         KASSERT((m->flags & PG_FICTITIOUS) != 0,
3627             ("not fictitious %p", m));
3628         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
3629
3630         if ((m->flags & PG_BUSY) != 0) {
3631                 DRM_UNLOCK(dev);
3632 #if 0 /* XXX */
3633                 vm_page_sleep(m, "915pbs");
3634 #endif
3635                 goto retry;
3636         }
3637         m->valid = VM_PAGE_BITS_ALL;
3638         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
3639 have_page:
3640         *mres = m;
3641         vm_page_busy_try(m, false);
3642
3643         DRM_UNLOCK(dev);
3644         if (oldm != NULL) {
3645                 vm_page_free(oldm);
3646         }
3647         vm_object_pip_wakeup(vm_obj);
3648         return (VM_PAGER_OK);
3649
3650 unlock:
3651         DRM_UNLOCK(dev);
3652 out:
3653         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
3654         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
3655                 goto unlocked_vmobj;
3656         }
3657         VM_OBJECT_LOCK(vm_obj);
3658         vm_object_pip_wakeup(vm_obj);
3659         return (VM_PAGER_ERROR);
3660 }
3661
3662 static void
3663 i915_gem_pager_dtor(void *handle)
3664 {
3665         struct drm_gem_object *obj;
3666         struct drm_device *dev;
3667
3668         obj = handle;
3669         dev = obj->dev;
3670
3671         DRM_LOCK(dev);
3672         drm_gem_free_mmap_offset(obj);
3673         i915_gem_release_mmap(to_intel_bo(obj));
3674         drm_gem_object_unreference(obj);
3675         DRM_UNLOCK(dev);
3676 }
3677
3678 struct cdev_pager_ops i915_gem_pager_ops = {
3679         .cdev_pg_fault  = i915_gem_pager_fault,
3680         .cdev_pg_ctor   = i915_gem_pager_ctor,
3681         .cdev_pg_dtor   = i915_gem_pager_dtor
3682 };
3683
3684 #define GEM_PARANOID_CHECK_GTT 0
3685 #if GEM_PARANOID_CHECK_GTT
3686 static void
3687 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
3688     int page_count)
3689 {
3690         struct drm_i915_private *dev_priv;
3691         vm_paddr_t pa;
3692         unsigned long start, end;
3693         u_int i;
3694         int j;
3695
3696         dev_priv = dev->dev_private;
3697         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
3698         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
3699         for (i = start; i < end; i++) {
3700                 pa = intel_gtt_read_pte_paddr(i);
3701                 for (j = 0; j < page_count; j++) {
3702                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
3703                                 panic("Page %p in GTT pte index %d pte %x",
3704                                     ma[i], i, intel_gtt_read_pte(i));
3705                         }
3706                 }
3707         }
3708 }
3709 #endif
3710
3711 static void
3712 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
3713     uint32_t flush_domains)
3714 {
3715         struct drm_i915_gem_object *obj, *next;
3716         uint32_t old_write_domain;
3717
3718         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
3719             gpu_write_list) {
3720                 if (obj->base.write_domain & flush_domains) {
3721                         old_write_domain = obj->base.write_domain;
3722                         obj->base.write_domain = 0;
3723                         list_del_init(&obj->gpu_write_list);
3724                         i915_gem_object_move_to_active(obj, ring,
3725                             i915_gem_next_request_seqno(ring));
3726                 }
3727         }
3728 }
3729
3730 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3731
3732 static vm_page_t
3733 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3734 {
3735         vm_page_t m;
3736         int rv;
3737
3738         VM_OBJECT_LOCK_ASSERT_OWNED(object);
3739         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
3740         if (m->valid != VM_PAGE_BITS_ALL) {
3741                 if (vm_pager_has_page(object, pindex)) {
3742                         rv = vm_pager_get_page(object, &m, 1);
3743                         m = vm_page_lookup(object, pindex);
3744                         if (m == NULL)
3745                                 return (NULL);
3746                         if (rv != VM_PAGER_OK) {
3747                                 vm_page_free(m);
3748                                 return (NULL);
3749                         }
3750                 } else {
3751                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
3752                         m->valid = VM_PAGE_BITS_ALL;
3753                         m->dirty = 0;
3754                 }
3755         }
3756         vm_page_wire(m);
3757         vm_page_wakeup(m);
3758         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3759         return (m);
3760 }
3761
3762 int
3763 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
3764     uint32_t flush_domains)
3765 {
3766         int ret;
3767
3768         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
3769                 return 0;
3770
3771         ret = ring->flush(ring, invalidate_domains, flush_domains);
3772         if (ret)
3773                 return ret;
3774
3775         if (flush_domains & I915_GEM_GPU_DOMAINS)
3776                 i915_gem_process_flushing_list(ring, flush_domains);
3777         return 0;
3778 }
3779
3780 u32
3781 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
3782 {
3783         if (ring->outstanding_lazy_request == 0)
3784                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
3785
3786         return ring->outstanding_lazy_request;
3787 }
3788
3789 static void
3790 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3791 {
3792         drm_i915_private_t *dev_priv = dev->dev_private;
3793         uint32_t fence_reg = reg - dev_priv->fence_regs;
3794
3795         switch (INTEL_INFO(dev)->gen) {
3796         case 7:
3797         case 6:
3798                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3799                 break;
3800         case 5:
3801         case 4:
3802                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3803                 break;
3804         case 3:
3805                 if (fence_reg >= 8)
3806                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3807                 else
3808         case 2:
3809                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3810
3811                 I915_WRITE(fence_reg, 0);
3812                 break;
3813         }
3814
3815         list_del_init(&reg->lru_list);
3816         reg->obj = NULL;
3817         reg->setup_seqno = 0;
3818         reg->pin_count = 0;
3819 }
3820
3821 static int
3822 i915_gpu_is_active(struct drm_device *dev)
3823 {
3824         drm_i915_private_t *dev_priv;
3825
3826         dev_priv = dev->dev_private;
3827         return (!list_empty(&dev_priv->mm.flushing_list) ||
3828             !list_empty(&dev_priv->mm.active_list));
3829 }
3830
3831 static void
3832 i915_gem_lowmem(void *arg)
3833 {
3834         struct drm_device *dev;
3835         struct drm_i915_private *dev_priv;
3836         struct drm_i915_gem_object *obj, *next;
3837         int cnt, cnt_fail, cnt_total;
3838
3839         dev = arg;
3840         dev_priv = dev->dev_private;
3841
3842         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3843                 return;
3844
3845 rescan:
3846         /* first scan for clean buffers */
3847         i915_gem_retire_requests(dev);
3848
3849         cnt_total = cnt_fail = cnt = 0;
3850
3851         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3852             mm_list) {
3853                 if (i915_gem_object_is_purgeable(obj)) {
3854                         if (i915_gem_object_unbind(obj) != 0)
3855                                 cnt_total++;
3856                 } else
3857                         cnt_total++;
3858         }
3859
3860         /* second pass, evict/count anything still on the inactive list */
3861         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3862             mm_list) {
3863                 if (i915_gem_object_unbind(obj) == 0)
3864                         cnt++;
3865                 else
3866                         cnt_fail++;
3867         }
3868
3869         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3870                 /*
3871                  * We are desperate for pages, so as a last resort, wait
3872                  * for the GPU to finish and discard whatever we can.
3873                  * This has a dramatic impact to reduce the number of
3874                  * OOM-killer events whilst running the GPU aggressively.
3875                  */
3876                 if (i915_gpu_idle(dev) == 0)
3877                         goto rescan;
3878         }
3879         DRM_UNLOCK(dev);
3880 }
3881
3882 void
3883 i915_gem_unload(struct drm_device *dev)
3884 {
3885         struct drm_i915_private *dev_priv;
3886
3887         dev_priv = dev->dev_private;
3888         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3889 }