sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64 #include <linux/completion.h>
  65 #include <linux/jiffies.h>
  66 #include <linux/time.h>
  67
  68 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  69 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  70 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  71     unsigned alignment, bool map_and_fenceable);
  72
  73 static int i915_gem_phys_pwrite(struct drm_device *dev,
  74     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  75     uint64_t size, struct drm_file *file_priv);
  76
  77 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  78     int tiling_mode);
  79 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  80     uint32_t size, int tiling_mode);
  81 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  82     int flags);
  83 static int i915_gem_object_set_cpu_read_domain_range(
  84     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  85 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  86 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  87 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  88 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  89 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  90 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  91 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  92     uint32_t flush_domains);
  93 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  94     struct drm_i915_fence_reg *reg);
  95 static void i915_gem_reset_fences(struct drm_device *dev);
  96 static void i915_gem_lowmem(void *arg);
  97
  98 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
  99     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file);
 100
 101 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
 102 long i915_gem_wired_pages_cnt;
 103
 104 /* some bookkeeping */
 105 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 106                                   size_t size)
 107 {
 108
 109         dev_priv->mm.object_count++;
 110         dev_priv->mm.object_memory += size;
 111 }
 112
 113 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 114                                      size_t size)
 115 {
 116
 117         dev_priv->mm.object_count--;
 118         dev_priv->mm.object_memory -= size;
 119 }
 120
 121 static int
 122 i915_gem_wait_for_error(struct drm_device *dev)
 123 {
 124         struct drm_i915_private *dev_priv = dev->dev_private;
 125         struct completion *x = &dev_priv->error_completion;
 126         int ret;
 127
 128         if (!atomic_read(&dev_priv->mm.wedged))
 129                 return 0;
 130
 131         /*
 132          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 133          * userspace. If it takes that long something really bad is going on and
 134          * we should simply try to bail out and fail as gracefully as possible.
 135          */
 136         ret = wait_for_completion_interruptible_timeout(x, 10*hz);
 137         if (ret == 0) {
 138                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 139                 return -EIO;
 140         } else if (ret < 0) {
 141                 return ret;
 142         }
 143
 144         if (atomic_read(&dev_priv->mm.wedged)) {
 145                 /* GPU is hung, bump the completion count to account for
 146                  * the token we just consumed so that we never hit zero and
 147                  * end up waiting upon a subsequent completion event that
 148                  * will never happen.
 149                  */
 150                 spin_lock(&x->wait.lock);
 151                 x->done++;
 152                 spin_unlock(&x->wait.lock);
 153         }
 154         return 0;
 155 }
 156
 157 int i915_mutex_lock_interruptible(struct drm_device *dev)
 158 {
 159         int ret;
 160
 161         ret = i915_gem_wait_for_error(dev);
 162         if (ret != 0)
 163                 return (ret);
 164
 165         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 166         if (ret)
 167                 return -EINTR;
 168
 169 #if 0
 170         WARN_ON(i915_verify_lists(dev));
 171 #endif
 172         return 0;
 173 }
 174
 175 static inline bool
 176 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 177 {
 178         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
 179 }
 180
 181 int
 182 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 183     struct drm_file *file)
 184 {
 185         struct drm_i915_gem_init *args;
 186         drm_i915_private_t *dev_priv;
 187
 188         dev_priv = dev->dev_private;
 189         args = data;
 190
 191         if (args->gtt_start >= args->gtt_end ||
 192             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 193                 return (-EINVAL);
 194
 195         /*
 196          * XXXKIB. The second-time initialization should be guarded
 197          * against.
 198          */
 199         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 200         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 201         lockmgr(&dev->dev_lock, LK_RELEASE);
 202
 203         return 0;
 204 }
 205
 206 int
 207 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 208                             struct drm_file *file)
 209 {
 210         struct drm_i915_private *dev_priv;
 211         struct drm_i915_gem_get_aperture *args;
 212         struct drm_i915_gem_object *obj;
 213         size_t pinned;
 214
 215         dev_priv = dev->dev_private;
 216         args = data;
 217
 218         if (!(dev->driver->driver_features & DRIVER_GEM))
 219                 return (-ENODEV);
 220
 221         pinned = 0;
 222         DRM_LOCK(dev);
 223         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 224                 pinned += obj->gtt_space->size;
 225         DRM_UNLOCK(dev);
 226
 227         args->aper_size = dev_priv->mm.gtt_total;
 228         args->aper_available_size = args->aper_size - pinned;
 229
 230         return (0);
 231 }
 232
 233 static int
 234 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 235     uint32_t *handle_p)
 236 {
 237         struct drm_i915_gem_object *obj;
 238         uint32_t handle;
 239         int ret;
 240
 241         size = roundup(size, PAGE_SIZE);
 242         if (size == 0)
 243                 return (-EINVAL);
 244
 245         obj = i915_gem_alloc_object(dev, size);
 246         if (obj == NULL)
 247                 return (-ENOMEM);
 248
 249         handle = 0;
 250         ret = drm_gem_handle_create(file, &obj->base, &handle);
 251         if (ret != 0) {
 252                 drm_gem_object_release(&obj->base);
 253                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 254                 drm_free(obj, DRM_I915_GEM);
 255                 return (-ret);
 256         }
 257
 258         /* drop reference from allocate - handle holds it now */
 259         drm_gem_object_unreference(&obj->base);
 260         *handle_p = handle;
 261         return (0);
 262 }
 263
 264 int
 265 i915_gem_dumb_create(struct drm_file *file,
 266                      struct drm_device *dev,
 267                      struct drm_mode_create_dumb *args)
 268 {
 269
 270         /* have to work out size/pitch and return them */
 271         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 272         args->size = args->pitch * args->height;
 273         return (i915_gem_create(file, dev, args->size, &args->handle));
 274 }
 275
 276 int i915_gem_dumb_destroy(struct drm_file *file,
 277                           struct drm_device *dev,
 278                           uint32_t handle)
 279 {
 280
 281         return (drm_gem_handle_delete(file, handle));
 282 }
 283
 284 /**
 285  * Creates a new mm object and returns a handle to it.
 286  */
 287 int
 288 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 289                       struct drm_file *file)
 290 {
 291         struct drm_i915_gem_create *args = data;
 292
 293         return (i915_gem_create(file, dev, args->size, &args->handle));
 294 }
 295
 296 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 297 {
 298         drm_i915_private_t *dev_priv;
 299
 300         dev_priv = obj->base.dev->dev_private;
 301         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 302             obj->tiling_mode != I915_TILING_NONE);
 303 }
 304
 305 /**
 306  * Reads data from the object referenced by handle.
 307  *
 308  * On error, the contents of *data are undefined.
 309  */
 310 int
 311 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 312                      struct drm_file *file)
 313 {
 314         struct drm_i915_gem_pread *args;
 315
 316         args = data;
 317         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 318             args->offset, UIO_READ, file));
 319 }
 320
 321 /**
 322  * Writes data to the object referenced by handle.
 323  *
 324  * On error, the contents of the buffer that were to be modified are undefined.
 325  */
 326 int
 327 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 328                       struct drm_file *file)
 329 {
 330         struct drm_i915_gem_pwrite *args;
 331
 332         args = data;
 333         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 334             args->offset, UIO_WRITE, file));
 335 }
 336
 337 int
 338 i915_gem_check_wedge(struct drm_i915_private *dev_priv,
 339                      bool interruptible)
 340 {
 341         if (atomic_read(&dev_priv->mm.wedged)) {
 342                 struct completion *x = &dev_priv->error_completion;
 343                 bool recovery_complete;
 344
 345                 /* Give the error handler a chance to run. */
 346                 spin_lock(&x->wait.lock);
 347                 recovery_complete = x->done > 0;
 348                 spin_unlock(&x->wait.lock);
 349
 350                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 351                  * -EIO unconditionally for these. */
 352                 if (!interruptible)
 353                         return -EIO;
 354
 355                 /* Recovery complete, but still wedged means reset failure. */
 356                 if (recovery_complete)
 357                         return -EIO;
 358
 359                 return -EAGAIN;
 360         }
 361
 362         return 0;
 363 }
 364
 365 /**
 366  * __wait_seqno - wait until execution of seqno has finished
 367  * @ring: the ring expected to report seqno
 368  * @seqno: duh!
 369  * @interruptible: do an interruptible wait (normally yes)
 370  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 371  *
 372  * Returns 0 if the seqno was found within the alloted time. Else returns the
 373  * errno with remaining time filled in timeout argument.
 374  */
 375 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
 376                         bool interruptible, struct timespec *timeout)
 377 {
 378         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 379         struct timespec before, now, wait_time={1,0};
 380         unsigned long timeout_jiffies;
 381         long end;
 382         bool wait_forever = true;
 383         int ret;
 384
 385         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
 386                 return 0;
 387
 388         if (timeout != NULL) {
 389                 wait_time = *timeout;
 390                 wait_forever = false;
 391         }
 392
 393         timeout_jiffies = timespec_to_jiffies(&wait_time);
 394
 395         if (WARN_ON(!ring->irq_get(ring)))
 396                 return -ENODEV;
 397
 398         /* Record current time in case interrupted by signal, or wedged * */
 399         getrawmonotonic(&before);
 400
 401 #define EXIT_COND \
 402         (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
 403         atomic_read(&dev_priv->mm.wedged))
 404         do {
 405                 if (interruptible)
 406                         end = wait_event_interruptible_timeout(ring->irq_queue,
 407                                                                EXIT_COND,
 408                                                                timeout_jiffies);
 409                 else
 410                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
 411                                                  timeout_jiffies);
 412
 413                 ret = i915_gem_check_wedge(dev_priv, interruptible);
 414                 if (ret)
 415                         end = ret;
 416         } while (end == 0 && wait_forever);
 417
 418         getrawmonotonic(&now);
 419
 420         ring->irq_put(ring);
 421 #undef EXIT_COND
 422
 423         if (timeout) {
 424                 struct timespec sleep_time = timespec_sub(now, before);
 425                 *timeout = timespec_sub(*timeout, sleep_time);
 426         }
 427
 428         switch (end) {
 429         case -EIO:
 430         case -EAGAIN: /* Wedged */
 431         case -ERESTARTSYS: /* Signal */
 432                 return (int)end;
 433         case 0: /* Timeout */
 434                 if (timeout)
 435                         set_normalized_timespec(timeout, 0, 0);
 436                 return -ETIMEDOUT;      /* -ETIME on Linux */
 437         default: /* Completed */
 438                 WARN_ON(end < 0); /* We're not aware of other errors */
 439                 return 0;
 440         }
 441 }
 442
 443 /**
 444  * Waits for a sequence number to be signaled, and cleans up the
 445  * request and object lists appropriately for that event.
 446  */
 447 int
 448 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
 449 {
 450         struct drm_device *dev = ring->dev;
 451         struct drm_i915_private *dev_priv = dev->dev_private;
 452         bool interruptible = dev_priv->mm.interruptible;
 453         struct drm_i915_gem_request *request;
 454         int ret;
 455
 456         DRM_LOCK_ASSERT(dev);
 457         BUG_ON(seqno == 0);
 458
 459         ret = i915_gem_check_wedge(dev_priv, interruptible);
 460         if (ret)
 461                 return ret;
 462
 463         if (seqno == ring->outstanding_lazy_request) {
 464                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
 465                     M_WAITOK | M_ZERO);
 466                 if (request == NULL)
 467                         return (-ENOMEM);
 468
 469                 ret = i915_add_request(ring, NULL, request);
 470                 if (ret != 0) {
 471                         drm_free(request, DRM_I915_GEM);
 472                         return (ret);
 473                 }
 474
 475                 seqno = request->seqno;
 476         }
 477
 478         return __wait_seqno(ring, seqno, interruptible, NULL);
 479 }
 480
 481 /**
 482  * Ensures that all rendering to the object has completed and the object is
 483  * safe to unbind from the GTT or access from the CPU.
 484  */
 485 int
 486 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 487 {
 488         u32 seqno;
 489         int ret;
 490
 491
 492         seqno = obj->last_rendering_seqno;
 493         if (seqno == 0)
 494                 return 0;
 495
 496         if (obj->active) {
 497                 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
 498                 if (ret != 0)
 499                         return (ret);
 500         }
 501
 502         /* Manually manage the write flush as we may have not yet
 503          * retired the buffer.
 504          */
 505         if (obj->last_rendering_seqno &&
 506             i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
 507                 obj->last_rendering_seqno = 0;
 508                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
 509         }
 510
 511         return 0;
 512 }
 513
 514 /**
 515  * Called when user space prepares to use an object with the CPU, either
 516  * through the mmap ioctl's mapping or a GTT mapping.
 517  */
 518 int
 519 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 520                           struct drm_file *file)
 521 {
 522         struct drm_i915_gem_set_domain *args;
 523         struct drm_i915_gem_object *obj;
 524         uint32_t read_domains;
 525         uint32_t write_domain;
 526         int ret;
 527
 528         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 529                 return (-ENODEV);
 530
 531         args = data;
 532         read_domains = args->read_domains;
 533         write_domain = args->write_domain;
 534
 535         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
 536             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
 537             (write_domain != 0 && read_domains != write_domain))
 538                 return (-EINVAL);
 539
 540         ret = i915_mutex_lock_interruptible(dev);
 541         if (ret != 0)
 542                 return (ret);
 543
 544         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 545         if (&obj->base == NULL) {
 546                 ret = -ENOENT;
 547                 goto unlock;
 548         }
 549
 550         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
 551                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 552                 if (ret == -EINVAL)
 553                         ret = 0;
 554         } else
 555                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 556
 557         drm_gem_object_unreference(&obj->base);
 558 unlock:
 559         DRM_UNLOCK(dev);
 560         return (ret);
 561 }
 562
 563 /**
 564  * Called when user space has done writes to this buffer
 565  */
 566 int
 567 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 568                          struct drm_file *file)
 569 {
 570         struct drm_i915_gem_sw_finish *args;
 571         struct drm_i915_gem_object *obj;
 572         int ret;
 573
 574         args = data;
 575         ret = 0;
 576         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 577                 return (ENODEV);
 578         ret = i915_mutex_lock_interruptible(dev);
 579         if (ret != 0)
 580                 return (ret);
 581         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 582         if (&obj->base == NULL) {
 583                 ret = -ENOENT;
 584                 goto unlock;
 585         }
 586         if (obj->pin_count != 0)
 587                 i915_gem_object_flush_cpu_write_domain(obj);
 588         drm_gem_object_unreference(&obj->base);
 589 unlock:
 590         DRM_UNLOCK(dev);
 591         return (ret);
 592 }
 593
 594 /**
 595  * Maps the contents of an object, returning the address it is mapped
 596  * into.
 597  *
 598  * While the mapping holds a reference on the contents of the object, it doesn't
 599  * imply a ref on the object itself.
 600  */
 601 int
 602 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 603                     struct drm_file *file)
 604 {
 605         struct drm_i915_gem_mmap *args;
 606         struct drm_gem_object *obj;
 607         struct proc *p;
 608         vm_map_t map;
 609         vm_offset_t addr;
 610         vm_size_t size;
 611         int error, rv;
 612
 613         args = data;
 614
 615         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 616                 return (-ENODEV);
 617
 618         obj = drm_gem_object_lookup(dev, file, args->handle);
 619         if (obj == NULL)
 620                 return (-ENOENT);
 621         error = 0;
 622         if (args->size == 0)
 623                 goto out;
 624         p = curproc;
 625         map = &p->p_vmspace->vm_map;
 626         size = round_page(args->size);
 627         PROC_LOCK(p);
 628         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
 629                 PROC_UNLOCK(p);
 630                 error = ENOMEM;
 631                 goto out;
 632         }
 633         PROC_UNLOCK(p);
 634
 635         addr = 0;
 636         vm_object_hold(obj->vm_obj);
 637         vm_object_reference_locked(obj->vm_obj);
 638         vm_object_drop(obj->vm_obj);
 639         DRM_UNLOCK(dev);
 640         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
 641             PAGE_SIZE, /* align */
 642             TRUE, /* fitit */
 643             VM_MAPTYPE_NORMAL, /* maptype */
 644             VM_PROT_READ | VM_PROT_WRITE, /* prot */
 645             VM_PROT_READ | VM_PROT_WRITE, /* max */
 646             MAP_SHARED /* cow */);
 647         if (rv != KERN_SUCCESS) {
 648                 vm_object_deallocate(obj->vm_obj);
 649                 error = -vm_mmap_to_errno(rv);
 650         } else {
 651                 args->addr_ptr = (uint64_t)addr;
 652         }
 653         DRM_LOCK(dev);
 654 out:
 655         drm_gem_object_unreference(obj);
 656         return (error);
 657 }
 658
 659 /**
 660  * i915_gem_release_mmap - remove physical page mappings
 661  * @obj: obj in question
 662  *
 663  * Preserve the reservation of the mmapping with the DRM core code, but
 664  * relinquish ownership of the pages back to the system.
 665  *
 666  * It is vital that we remove the page mapping if we have mapped a tiled
 667  * object through the GTT and then lose the fence register due to
 668  * resource pressure. Similarly if the object has been moved out of the
 669  * aperture, than pages mapped into userspace must be revoked. Removing the
 670  * mapping will then trigger a page fault on the next user access, allowing
 671  * fixup by i915_gem_fault().
 672  */
 673 void
 674 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 675 {
 676         vm_object_t devobj;
 677         vm_page_t m;
 678         int i, page_count;
 679
 680         if (!obj->fault_mappable)
 681                 return;
 682
 683         devobj = cdev_pager_lookup(obj);
 684         if (devobj != NULL) {
 685                 page_count = OFF_TO_IDX(obj->base.size);
 686
 687                 VM_OBJECT_LOCK(devobj);
 688                 for (i = 0; i < page_count; i++) {
 689                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
 690                         if (m == NULL)
 691                                 continue;
 692                         cdev_pager_free_page(devobj, m);
 693                 }
 694                 VM_OBJECT_UNLOCK(devobj);
 695                 vm_object_deallocate(devobj);
 696         }
 697
 698         obj->fault_mappable = false;
 699 }
 700
 701 static uint32_t
 702 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 703 {
 704         uint32_t gtt_size;
 705
 706         if (INTEL_INFO(dev)->gen >= 4 ||
 707             tiling_mode == I915_TILING_NONE)
 708                 return (size);
 709
 710         /* Previous chips need a power-of-two fence region when tiling */
 711         if (INTEL_INFO(dev)->gen == 3)
 712                 gtt_size = 1024*1024;
 713         else
 714                 gtt_size = 512*1024;
 715
 716         while (gtt_size < size)
 717                 gtt_size <<= 1;
 718
 719         return (gtt_size);
 720 }
 721
 722 /**
 723  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 724  * @obj: object to check
 725  *
 726  * Return the required GTT alignment for an object, taking into account
 727  * potential fence register mapping.
 728  */
 729 static uint32_t
 730 i915_gem_get_gtt_alignment(struct drm_device *dev,
 731                            uint32_t size,
 732                            int tiling_mode)
 733 {
 734
 735         /*
 736          * Minimum alignment is 4k (GTT page size), but might be greater
 737          * if a fence register is needed for the object.
 738          */
 739         if (INTEL_INFO(dev)->gen >= 4 ||
 740             tiling_mode == I915_TILING_NONE)
 741                 return (4096);
 742
 743         /*
 744          * Previous chips need to be aligned to the size of the smallest
 745          * fence register that can contain the object.
 746          */
 747         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 748 }
 749
 750 /**
 751  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
 752  *                                       unfenced object
 753  * @dev: the device
 754  * @size: size of the object
 755  * @tiling_mode: tiling mode of the object
 756  *
 757  * Return the required GTT alignment for an object, only taking into account
 758  * unfenced tiled surface requirements.
 759  */
 760 uint32_t
 761 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 762                                     uint32_t size,
 763                                     int tiling_mode)
 764 {
 765
 766         if (tiling_mode == I915_TILING_NONE)
 767                 return (4096);
 768
 769         /*
 770          * Minimum alignment is 4k (GTT page size) for sane hw.
 771          */
 772         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
 773                 return (4096);
 774
 775         /*
 776          * Previous hardware however needs to be aligned to a power-of-two
 777          * tile height. The simplest method for determining this is to reuse
 778          * the power-of-tile object size.
 779          */
 780         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 781 }
 782
 783 int
 784 i915_gem_mmap_gtt(struct drm_file *file,
 785                   struct drm_device *dev,
 786                   uint32_t handle,
 787                   uint64_t *offset)
 788 {
 789         struct drm_i915_private *dev_priv;
 790         struct drm_i915_gem_object *obj;
 791         int ret;
 792
 793         if (!(dev->driver->driver_features & DRIVER_GEM))
 794                 return (-ENODEV);
 795
 796         dev_priv = dev->dev_private;
 797
 798         ret = i915_mutex_lock_interruptible(dev);
 799         if (ret != 0)
 800                 return (ret);
 801
 802         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 803         if (&obj->base == NULL) {
 804                 ret = -ENOENT;
 805                 goto unlock;
 806         }
 807
 808         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
 809                 ret = -E2BIG;
 810                 goto out;
 811         }
 812
 813         if (obj->madv != I915_MADV_WILLNEED) {
 814                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
 815                 ret = -EINVAL;
 816                 goto out;
 817         }
 818
 819         ret = drm_gem_create_mmap_offset(&obj->base);
 820         if (ret != 0)
 821                 goto out;
 822
 823         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
 824             DRM_GEM_MAPPING_KEY;
 825 out:
 826         drm_gem_object_unreference(&obj->base);
 827 unlock:
 828         DRM_UNLOCK(dev);
 829         return (ret);
 830 }
 831
 832 /**
 833  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 834  * @dev: DRM device
 835  * @data: GTT mapping ioctl data
 836  * @file: GEM object info
 837  *
 838  * Simply returns the fake offset to userspace so it can mmap it.
 839  * The mmap call will end up in drm_gem_mmap(), which will set things
 840  * up so we can get faults in the handler above.
 841  *
 842  * The fault handler will take care of binding the object into the GTT
 843  * (since it may have been evicted to make room for something), allocating
 844  * a fence register, and mapping the appropriate aperture address into
 845  * userspace.
 846  */
 847 int
 848 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 849                         struct drm_file *file)
 850 {
 851         struct drm_i915_private *dev_priv;
 852         struct drm_i915_gem_mmap_gtt *args;
 853
 854         dev_priv = dev->dev_private;
 855         args = data;
 856
 857         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
 858 }
 859
 860 /* Immediately discard the backing storage */
 861 static void
 862 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 863 {
 864         vm_object_t vm_obj;
 865
 866         vm_obj = obj->base.vm_obj;
 867         VM_OBJECT_LOCK(vm_obj);
 868         vm_object_page_remove(vm_obj, 0, 0, false);
 869         VM_OBJECT_UNLOCK(vm_obj);
 870         obj->madv = __I915_MADV_PURGED;
 871 }
 872
 873 static inline int
 874 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 875 {
 876         return obj->madv == I915_MADV_DONTNEED;
 877 }
 878
 879 static inline void vm_page_reference(vm_page_t m)
 880 {
 881         vm_page_flag_set(m, PG_REFERENCED);
 882 }
 883
 884 static void
 885 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 886 {
 887         vm_page_t m;
 888         int page_count, i;
 889
 890         BUG_ON(obj->madv == __I915_MADV_PURGED);
 891
 892         if (obj->tiling_mode != I915_TILING_NONE)
 893                 i915_gem_object_save_bit_17_swizzle(obj);
 894         if (obj->madv == I915_MADV_DONTNEED)
 895                 obj->dirty = 0;
 896         page_count = obj->base.size / PAGE_SIZE;
 897         VM_OBJECT_LOCK(obj->base.vm_obj);
 898 #if GEM_PARANOID_CHECK_GTT
 899         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
 900 #endif
 901         for (i = 0; i < page_count; i++) {
 902                 m = obj->pages[i];
 903                 if (obj->dirty)
 904                         vm_page_dirty(m);
 905                 if (obj->madv == I915_MADV_WILLNEED)
 906                         vm_page_reference(m);
 907                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
 908                 vm_page_unwire(obj->pages[i], 1);
 909                 vm_page_wakeup(obj->pages[i]);
 910                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 911         }
 912         VM_OBJECT_UNLOCK(obj->base.vm_obj);
 913         obj->dirty = 0;
 914         drm_free(obj->pages, DRM_I915_GEM);
 915         obj->pages = NULL;
 916 }
 917
 918 static int
 919 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 920     int flags)
 921 {
 922         struct drm_device *dev;
 923         vm_object_t vm_obj;
 924         vm_page_t m;
 925         int page_count, i, j;
 926
 927         dev = obj->base.dev;
 928         KASSERT(obj->pages == NULL, ("Obj already has pages"));
 929         page_count = obj->base.size / PAGE_SIZE;
 930         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 931             M_WAITOK);
 932         vm_obj = obj->base.vm_obj;
 933         VM_OBJECT_LOCK(vm_obj);
 934         for (i = 0; i < page_count; i++) {
 935                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
 936                         goto failed;
 937         }
 938         VM_OBJECT_UNLOCK(vm_obj);
 939         if (i915_gem_object_needs_bit17_swizzle(obj))
 940                 i915_gem_object_do_bit_17_swizzle(obj);
 941         return (0);
 942
 943 failed:
 944         for (j = 0; j < i; j++) {
 945                 m = obj->pages[j];
 946                 vm_page_busy_wait(m, FALSE, "i915gem");
 947                 vm_page_unwire(m, 0);
 948                 vm_page_wakeup(m);
 949                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 950         }
 951         VM_OBJECT_UNLOCK(vm_obj);
 952         drm_free(obj->pages, DRM_I915_GEM);
 953         obj->pages = NULL;
 954         return (-EIO);
 955 }
 956
 957 void
 958 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 959     struct intel_ring_buffer *ring, uint32_t seqno)
 960 {
 961         struct drm_device *dev = obj->base.dev;
 962         struct drm_i915_private *dev_priv = dev->dev_private;
 963         struct drm_i915_fence_reg *reg;
 964
 965         obj->ring = ring;
 966         KASSERT(ring != NULL, ("NULL ring"));
 967
 968         /* Add a reference if we're newly entering the active list. */
 969         if (!obj->active) {
 970                 drm_gem_object_reference(&obj->base);
 971                 obj->active = 1;
 972         }
 973
 974         /* Move from whatever list we were on to the tail of execution. */
 975         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 976         list_move_tail(&obj->ring_list, &ring->active_list);
 977
 978         obj->last_rendering_seqno = seqno;
 979         if (obj->fenced_gpu_access) {
 980                 obj->last_fenced_seqno = seqno;
 981                 obj->last_fenced_ring = ring;
 982
 983                 /* Bump MRU to take account of the delayed flush */
 984                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
 985                         reg = &dev_priv->fence_regs[obj->fence_reg];
 986                         list_move_tail(&reg->lru_list,
 987                                        &dev_priv->mm.fence_list);
 988                 }
 989         }
 990 }
 991
 992 static void
 993 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 994 {
 995         list_del_init(&obj->ring_list);
 996         obj->last_rendering_seqno = 0;
 997         obj->last_fenced_seqno = 0;
 998 }
 999
1000 static void
1001 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1002 {
1003         struct drm_device *dev = obj->base.dev;
1004         struct drm_i915_private *dev_priv = dev->dev_private;
1005
1006         if (obj->pin_count != 0)
1007                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1008         else
1009                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1010
1011         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
1012         KASSERT(obj->active, ("Object not active"));
1013         obj->ring = NULL;
1014         obj->last_fenced_ring = NULL;
1015
1016         i915_gem_object_move_off_active(obj);
1017         obj->fenced_gpu_access = false;
1018
1019         obj->active = 0;
1020         obj->pending_gpu_write = false;
1021         drm_gem_object_unreference(&obj->base);
1022
1023 #if 1
1024         KIB_NOTYET();
1025 #else
1026         WARN_ON(i915_verify_lists(dev));
1027 #endif
1028 }
1029
1030 static u32
1031 i915_gem_get_seqno(struct drm_device *dev)
1032 {
1033         drm_i915_private_t *dev_priv = dev->dev_private;
1034         u32 seqno = dev_priv->next_seqno;
1035
1036         /* reserve 0 for non-seqno */
1037         if (++dev_priv->next_seqno == 0)
1038                 dev_priv->next_seqno = 1;
1039
1040         return seqno;
1041 }
1042
1043 int
1044 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
1045      struct drm_i915_gem_request *request)
1046 {
1047         drm_i915_private_t *dev_priv;
1048         struct drm_i915_file_private *file_priv;
1049         uint32_t seqno;
1050         u32 request_ring_position;
1051         int was_empty;
1052         int ret;
1053
1054         KASSERT(request != NULL, ("NULL request in add"));
1055         DRM_LOCK_ASSERT(ring->dev);
1056         dev_priv = ring->dev->dev_private;
1057
1058         seqno = i915_gem_next_request_seqno(ring);
1059         request_ring_position = intel_ring_get_tail(ring);
1060
1061         ret = ring->add_request(ring, &seqno);
1062         if (ret != 0)
1063             return ret;
1064
1065         request->seqno = seqno;
1066         request->ring = ring;
1067         request->tail = request_ring_position;
1068         request->emitted_jiffies = ticks;
1069         was_empty = list_empty(&ring->request_list);
1070         list_add_tail(&request->list, &ring->request_list);
1071
1072         if (file != NULL) {
1073                 file_priv = file->driver_priv;
1074
1075                 spin_lock(&file_priv->mm.lock);
1076                 request->file_priv = file_priv;
1077                 list_add_tail(&request->client_list,
1078                     &file_priv->mm.request_list);
1079                 spin_unlock(&file_priv->mm.lock);
1080         }
1081
1082         ring->outstanding_lazy_request = 0;
1083
1084         if (!dev_priv->mm.suspended) {
1085                 if (i915_enable_hangcheck) {
1086                         mod_timer(&dev_priv->hangcheck_timer,
1087                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
1088                 }
1089                 if (was_empty) {
1090                         queue_delayed_work(dev_priv->wq,
1091                                            &dev_priv->mm.retire_work,
1092                                            round_jiffies_up_relative(hz));
1093                         intel_mark_busy(dev_priv->dev);
1094                 }
1095         }
1096         return (0);
1097 }
1098
1099 static inline void
1100 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1101 {
1102         struct drm_i915_file_private *file_priv = request->file_priv;
1103
1104         if (!file_priv)
1105                 return;
1106
1107         DRM_LOCK_ASSERT(request->ring->dev);
1108
1109         spin_lock(&file_priv->mm.lock);
1110         if (request->file_priv != NULL) {
1111                 list_del(&request->client_list);
1112                 request->file_priv = NULL;
1113         }
1114         spin_unlock(&file_priv->mm.lock);
1115 }
1116
1117 static void
1118 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1119     struct intel_ring_buffer *ring)
1120 {
1121
1122         if (ring->dev != NULL)
1123                 DRM_LOCK_ASSERT(ring->dev);
1124
1125         while (!list_empty(&ring->request_list)) {
1126                 struct drm_i915_gem_request *request;
1127
1128                 request = list_first_entry(&ring->request_list,
1129                     struct drm_i915_gem_request, list);
1130
1131                 list_del(&request->list);
1132                 i915_gem_request_remove_from_client(request);
1133                 drm_free(request, DRM_I915_GEM);
1134         }
1135
1136         while (!list_empty(&ring->active_list)) {
1137                 struct drm_i915_gem_object *obj;
1138
1139                 obj = list_first_entry(&ring->active_list,
1140                     struct drm_i915_gem_object, ring_list);
1141
1142                 obj->base.write_domain = 0;
1143                 list_del_init(&obj->gpu_write_list);
1144                 i915_gem_object_move_to_inactive(obj);
1145         }
1146 }
1147
1148 static void
1149 i915_gem_reset_fences(struct drm_device *dev)
1150 {
1151         struct drm_i915_private *dev_priv = dev->dev_private;
1152         int i;
1153
1154         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1155                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1156                 struct drm_i915_gem_object *obj = reg->obj;
1157
1158                 if (!obj)
1159                         continue;
1160
1161                 if (obj->tiling_mode)
1162                         i915_gem_release_mmap(obj);
1163
1164                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1165                 reg->obj->fenced_gpu_access = false;
1166                 reg->obj->last_fenced_seqno = 0;
1167                 reg->obj->last_fenced_ring = NULL;
1168                 i915_gem_clear_fence_reg(dev, reg);
1169         }
1170 }
1171
1172 void i915_gem_reset(struct drm_device *dev)
1173 {
1174         struct drm_i915_private *dev_priv = dev->dev_private;
1175         struct drm_i915_gem_object *obj;
1176         int i;
1177
1178         for (i = 0; i < I915_NUM_RINGS; i++)
1179                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1180
1181         /* Remove anything from the flushing lists. The GPU cache is likely
1182          * to be lost on reset along with the data, so simply move the
1183          * lost bo to the inactive list.
1184          */
1185         while (!list_empty(&dev_priv->mm.flushing_list)) {
1186                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1187                                       struct drm_i915_gem_object,
1188                                       mm_list);
1189
1190                 obj->base.write_domain = 0;
1191                 list_del_init(&obj->gpu_write_list);
1192                 i915_gem_object_move_to_inactive(obj);
1193         }
1194
1195         /* Move everything out of the GPU domains to ensure we do any
1196          * necessary invalidation upon reuse.
1197          */
1198         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
1199                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1200         }
1201
1202         /* The fence registers are invalidated so clear them out */
1203         i915_gem_reset_fences(dev);
1204 }
1205
1206 static void
1207 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1208 {
1209         struct drm_device *dev = obj->base.dev;
1210         drm_i915_private_t *dev_priv = dev->dev_private;
1211
1212         KASSERT(obj->active, ("Object not active"));
1213         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1214
1215         i915_gem_object_move_off_active(obj);
1216 }
1217
1218 /**
1219  * This function clears the request list as sequence numbers are passed.
1220  */
1221 void
1222 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1223 {
1224         uint32_t seqno;
1225
1226         if (list_empty(&ring->request_list))
1227                 return;
1228
1229         seqno = ring->get_seqno(ring, true);
1230
1231         while (!list_empty(&ring->request_list)) {
1232                 struct drm_i915_gem_request *request;
1233
1234                 request = list_first_entry(&ring->request_list,
1235                                            struct drm_i915_gem_request,
1236                                            list);
1237
1238                 if (!i915_seqno_passed(seqno, request->seqno))
1239                         break;
1240
1241                 /* We know the GPU must have read the request to have
1242                  * sent us the seqno + interrupt, so use the position
1243                  * of tail of the request to update the last known position
1244                  * of the GPU head.
1245                  */
1246                 ring->last_retired_head = request->tail;
1247
1248                 list_del(&request->list);
1249                 i915_gem_request_remove_from_client(request);
1250                 drm_free(request, DRM_I915_GEM);
1251         }
1252
1253         /* Move any buffers on the active list that are no longer referenced
1254          * by the ringbuffer to the flushing/inactive lists as appropriate.
1255          */
1256         while (!list_empty(&ring->active_list)) {
1257                 struct drm_i915_gem_object *obj;
1258
1259                 obj = list_first_entry(&ring->active_list,
1260                                       struct drm_i915_gem_object,
1261                                       ring_list);
1262
1263                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1264                         break;
1265
1266                 if (obj->base.write_domain != 0)
1267                         i915_gem_object_move_to_flushing(obj);
1268                 else
1269                         i915_gem_object_move_to_inactive(obj);
1270         }
1271
1272         if (unlikely(ring->trace_irq_seqno &&
1273                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1274                 ring->irq_put(ring);
1275                 ring->trace_irq_seqno = 0;
1276         }
1277
1278 }
1279
1280 static void
1281 i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
1282
1283 void
1284 i915_gem_retire_requests(struct drm_device *dev)
1285 {
1286         drm_i915_private_t *dev_priv = dev->dev_private;
1287         struct drm_i915_gem_object *obj, *next;
1288         int i;
1289
1290         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1291                 list_for_each_entry_safe(obj, next,
1292                     &dev_priv->mm.deferred_free_list, mm_list)
1293                         i915_gem_free_object_tail(obj);
1294         }
1295
1296         for (i = 0; i < I915_NUM_RINGS; i++)
1297                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1298 }
1299
1300 static void
1301 i915_gem_retire_work_handler(struct work_struct *work)
1302 {
1303         drm_i915_private_t *dev_priv;
1304         struct drm_device *dev;
1305         struct intel_ring_buffer *ring;
1306         bool idle;
1307         int i;
1308
1309         dev_priv = container_of(work, drm_i915_private_t,
1310                                 mm.retire_work.work);
1311         dev = dev_priv->dev;
1312
1313         /* Come back later if the device is busy... */
1314         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
1315                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1316                                    round_jiffies_up_relative(hz));
1317                 return;
1318         }
1319
1320         i915_gem_retire_requests(dev);
1321
1322         /* Send a periodic flush down the ring so we don't hold onto GEM
1323          * objects indefinitely.
1324          */
1325         idle = true;
1326         for_each_ring(ring, dev_priv, i) {
1327                 if (ring->gpu_caches_dirty)
1328                         i915_add_request(ring, NULL, NULL);
1329
1330                 idle &= list_empty(&ring->request_list);
1331         }
1332
1333         if (!dev_priv->mm.suspended && !idle)
1334                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1335                                    round_jiffies_up_relative(hz));
1336         if (idle)
1337                 intel_mark_idle(dev);
1338
1339         DRM_UNLOCK(dev);
1340 }
1341
1342 /**
1343  * i915_gem_object_sync - sync an object to a ring.
1344  *
1345  * @obj: object which may be in use on another ring.
1346  * @to: ring we wish to use the object on. May be NULL.
1347  *
1348  * This code is meant to abstract object synchronization with the GPU.
1349  * Calling with NULL implies synchronizing the object with the CPU
1350  * rather than a particular GPU ring.
1351  *
1352  * Returns 0 if successful, else propagates up the lower layer error.
1353  */
1354 int
1355 i915_gem_object_sync(struct drm_i915_gem_object *obj,
1356                      struct intel_ring_buffer *to)
1357 {
1358         struct intel_ring_buffer *from = obj->ring;
1359         u32 seqno;
1360         int ret, idx;
1361
1362         if (from == NULL || to == from)
1363                 return 0;
1364
1365         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
1366                 return i915_gem_object_wait_rendering(obj);
1367
1368         idx = intel_ring_sync_index(from, to);
1369
1370         seqno = obj->last_rendering_seqno;
1371         if (seqno <= from->sync_seqno[idx])
1372                 return 0;
1373
1374         if (seqno == from->outstanding_lazy_request) {
1375                 struct drm_i915_gem_request *request;
1376
1377                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
1378                     M_WAITOK | M_ZERO);
1379                 if (request == NULL)
1380                         return -ENOMEM;
1381
1382                 ret = i915_add_request(from, NULL, request);
1383                 if (ret) {
1384                         kfree(request, DRM_I915_GEM);
1385                         return ret;
1386                 }
1387
1388                 seqno = request->seqno;
1389         }
1390
1391         from->sync_seqno[idx] = seqno;
1392
1393         return to->sync_to(to, from, seqno - 1);
1394 }
1395
1396 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1397 {
1398         u32 old_write_domain, old_read_domains;
1399
1400         /* Act a barrier for all accesses through the GTT */
1401         cpu_mfence();
1402
1403         /* Force a pagefault for domain tracking on next user access */
1404         i915_gem_release_mmap(obj);
1405
1406         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1407                 return;
1408
1409         old_read_domains = obj->base.read_domains;
1410         old_write_domain = obj->base.write_domain;
1411
1412         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1413         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1414
1415 }
1416
1417 int
1418 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1419 {
1420         drm_i915_private_t *dev_priv;
1421         int ret;
1422
1423         dev_priv = obj->base.dev->dev_private;
1424         ret = 0;
1425         if (obj->gtt_space == NULL)
1426                 return (0);
1427         if (obj->pin_count != 0) {
1428                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1429                 return (-EINVAL);
1430         }
1431
1432         ret = i915_gem_object_finish_gpu(obj);
1433         if (ret == -ERESTART || ret == -EINTR)
1434                 return (ret);
1435
1436         i915_gem_object_finish_gtt(obj);
1437
1438         if (ret == 0)
1439                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1440         if (ret == -ERESTART || ret == -EINTR)
1441                 return (ret);
1442         if (ret != 0) {
1443                 i915_gem_clflush_object(obj);
1444                 obj->base.read_domains = obj->base.write_domain =
1445                     I915_GEM_DOMAIN_CPU;
1446         }
1447
1448         ret = i915_gem_object_put_fence(obj);
1449         if (ret == -ERESTART)
1450                 return (ret);
1451
1452         i915_gem_gtt_unbind_object(obj);
1453         if (obj->has_aliasing_ppgtt_mapping) {
1454                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
1455                 obj->has_aliasing_ppgtt_mapping = 0;
1456         }
1457         i915_gem_object_put_pages_gtt(obj);
1458
1459         list_del_init(&obj->gtt_list);
1460         list_del_init(&obj->mm_list);
1461         obj->map_and_fenceable = true;
1462
1463         drm_mm_put_block(obj->gtt_space);
1464         obj->gtt_space = NULL;
1465         obj->gtt_offset = 0;
1466
1467         if (i915_gem_object_is_purgeable(obj))
1468                 i915_gem_object_truncate(obj);
1469
1470         return (ret);
1471 }
1472
1473 int i915_gpu_idle(struct drm_device *dev)
1474 {
1475         drm_i915_private_t *dev_priv = dev->dev_private;
1476         struct intel_ring_buffer *ring;
1477         int ret, i;
1478
1479         /* Flush everything onto the inactive list. */
1480         for_each_ring(ring, dev_priv, i) {
1481                 ret = intel_ring_idle(ring);
1482                 if (ret)
1483                         return ret;
1484         }
1485
1486         return 0;
1487 }
1488
1489 static int
1490 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
1491     struct intel_ring_buffer *pipelined)
1492 {
1493         struct drm_device *dev = obj->base.dev;
1494         drm_i915_private_t *dev_priv = dev->dev_private;
1495         u32 size = obj->gtt_space->size;
1496         int regnum = obj->fence_reg;
1497         uint64_t val;
1498
1499         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1500                          0xfffff000) << 32;
1501         val |= obj->gtt_offset & 0xfffff000;
1502         val |= (uint64_t)((obj->stride / 128) - 1) <<
1503                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
1504
1505         if (obj->tiling_mode == I915_TILING_Y)
1506                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1507         val |= I965_FENCE_REG_VALID;
1508
1509         if (pipelined) {
1510                 int ret = intel_ring_begin(pipelined, 6);
1511                 if (ret)
1512                         return ret;
1513
1514                 intel_ring_emit(pipelined, MI_NOOP);
1515                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1516                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
1517                 intel_ring_emit(pipelined, (u32)val);
1518                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
1519                 intel_ring_emit(pipelined, (u32)(val >> 32));
1520                 intel_ring_advance(pipelined);
1521         } else
1522                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
1523
1524         return 0;
1525 }
1526
1527 static int
1528 i965_write_fence_reg(struct drm_i915_gem_object *obj,
1529     struct intel_ring_buffer *pipelined)
1530 {
1531         struct drm_device *dev = obj->base.dev;
1532         drm_i915_private_t *dev_priv = dev->dev_private;
1533         u32 size = obj->gtt_space->size;
1534         int regnum = obj->fence_reg;
1535         uint64_t val;
1536
1537         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1538                     0xfffff000) << 32;
1539         val |= obj->gtt_offset & 0xfffff000;
1540         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1541         if (obj->tiling_mode == I915_TILING_Y)
1542                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1543         val |= I965_FENCE_REG_VALID;
1544
1545         if (pipelined) {
1546                 int ret = intel_ring_begin(pipelined, 6);
1547                 if (ret)
1548                         return ret;
1549
1550                 intel_ring_emit(pipelined, MI_NOOP);
1551                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1552                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
1553                 intel_ring_emit(pipelined, (u32)val);
1554                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
1555                 intel_ring_emit(pipelined, (u32)(val >> 32));
1556                 intel_ring_advance(pipelined);
1557         } else
1558                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
1559
1560         return 0;
1561 }
1562
1563 static int
1564 i915_write_fence_reg(struct drm_i915_gem_object *obj,
1565     struct intel_ring_buffer *pipelined)
1566 {
1567         struct drm_device *dev = obj->base.dev;
1568         drm_i915_private_t *dev_priv = dev->dev_private;
1569         u32 size = obj->gtt_space->size;
1570         u32 fence_reg, val, pitch_val;
1571         int tile_width;
1572
1573         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
1574             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1575                 kprintf(
1576 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1577                  obj->gtt_offset, obj->map_and_fenceable, size);
1578                 return -EINVAL;
1579         }
1580
1581         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1582                 tile_width = 128;
1583         else
1584                 tile_width = 512;
1585
1586         /* Note: pitch better be a power of two tile widths */
1587         pitch_val = obj->stride / tile_width;
1588         pitch_val = ffs(pitch_val) - 1;
1589
1590         val = obj->gtt_offset;
1591         if (obj->tiling_mode == I915_TILING_Y)
1592                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1593         val |= I915_FENCE_SIZE_BITS(size);
1594         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1595         val |= I830_FENCE_REG_VALID;
1596
1597         fence_reg = obj->fence_reg;
1598         if (fence_reg < 8)
1599                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
1600         else
1601                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
1602
1603         if (pipelined) {
1604                 int ret = intel_ring_begin(pipelined, 4);
1605                 if (ret)
1606                         return ret;
1607
1608                 intel_ring_emit(pipelined, MI_NOOP);
1609                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1610                 intel_ring_emit(pipelined, fence_reg);
1611                 intel_ring_emit(pipelined, val);
1612                 intel_ring_advance(pipelined);
1613         } else
1614                 I915_WRITE(fence_reg, val);
1615
1616         return 0;
1617 }
1618
1619 static int
1620 i830_write_fence_reg(struct drm_i915_gem_object *obj,
1621     struct intel_ring_buffer *pipelined)
1622 {
1623         struct drm_device *dev = obj->base.dev;
1624         drm_i915_private_t *dev_priv = dev->dev_private;
1625         u32 size = obj->gtt_space->size;
1626         int regnum = obj->fence_reg;
1627         uint32_t val;
1628         uint32_t pitch_val;
1629
1630         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
1631             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1632                 kprintf(
1633 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1634                     obj->gtt_offset, size);
1635                 return -EINVAL;
1636         }
1637
1638         pitch_val = obj->stride / 128;
1639         pitch_val = ffs(pitch_val) - 1;
1640
1641         val = obj->gtt_offset;
1642         if (obj->tiling_mode == I915_TILING_Y)
1643                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1644         val |= I830_FENCE_SIZE_BITS(size);
1645         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1646         val |= I830_FENCE_REG_VALID;
1647
1648         if (pipelined) {
1649                 int ret = intel_ring_begin(pipelined, 4);
1650                 if (ret)
1651                         return ret;
1652
1653                 intel_ring_emit(pipelined, MI_NOOP);
1654                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1655                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
1656                 intel_ring_emit(pipelined, val);
1657                 intel_ring_advance(pipelined);
1658         } else
1659                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
1660
1661         return 0;
1662 }
1663
1664 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
1665 {
1666         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
1667 }
1668
1669 static int
1670 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
1671     struct intel_ring_buffer *pipelined)
1672 {
1673         int ret;
1674
1675         if (obj->fenced_gpu_access) {
1676                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1677                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
1678                             obj->base.write_domain);
1679                         if (ret)
1680                                 return ret;
1681                 }
1682
1683                 obj->fenced_gpu_access = false;
1684         }
1685
1686         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
1687                 if (!ring_passed_seqno(obj->last_fenced_ring,
1688                                        obj->last_fenced_seqno)) {
1689                         ret = i915_wait_seqno(obj->last_fenced_ring,
1690                                                 obj->last_fenced_seqno);
1691                         if (ret)
1692                                 return ret;
1693                 }
1694
1695                 obj->last_fenced_seqno = 0;
1696                 obj->last_fenced_ring = NULL;
1697         }
1698
1699         /* Ensure that all CPU reads are completed before installing a fence
1700          * and all writes before removing the fence.
1701          */
1702         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
1703                 cpu_mfence();
1704
1705         return 0;
1706 }
1707
1708 int
1709 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
1710 {
1711         int ret;
1712
1713         if (obj->tiling_mode)
1714                 i915_gem_release_mmap(obj);
1715
1716         ret = i915_gem_object_flush_fence(obj, NULL);
1717         if (ret)
1718                 return ret;
1719
1720         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1721                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1722
1723                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
1724                         kprintf("%s: pin_count %d\n", __func__,
1725                             dev_priv->fence_regs[obj->fence_reg].pin_count);
1726                 i915_gem_clear_fence_reg(obj->base.dev,
1727                                          &dev_priv->fence_regs[obj->fence_reg]);
1728
1729                 obj->fence_reg = I915_FENCE_REG_NONE;
1730         }
1731
1732         return 0;
1733 }
1734
1735 static struct drm_i915_fence_reg *
1736 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
1737 {
1738         struct drm_i915_private *dev_priv = dev->dev_private;
1739         struct drm_i915_fence_reg *reg, *first, *avail;
1740         int i;
1741
1742         /* First try to find a free reg */
1743         avail = NULL;
1744         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
1745                 reg = &dev_priv->fence_regs[i];
1746                 if (!reg->obj)
1747                         return reg;
1748
1749                 if (!reg->pin_count)
1750                         avail = reg;
1751         }
1752
1753         if (avail == NULL)
1754                 return NULL;
1755
1756         /* None available, try to steal one or wait for a user to finish */
1757         avail = first = NULL;
1758         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1759                 if (reg->pin_count)
1760                         continue;
1761
1762                 if (first == NULL)
1763                         first = reg;
1764
1765                 if (!pipelined ||
1766                     !reg->obj->last_fenced_ring ||
1767                     reg->obj->last_fenced_ring == pipelined) {
1768                         avail = reg;
1769                         break;
1770                 }
1771         }
1772
1773         if (avail == NULL)
1774                 avail = first;
1775
1776         return avail;
1777 }
1778
1779 int
1780 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1781     struct intel_ring_buffer *pipelined)
1782 {
1783         struct drm_device *dev = obj->base.dev;
1784         struct drm_i915_private *dev_priv = dev->dev_private;
1785         struct drm_i915_fence_reg *reg;
1786         int ret;
1787
1788         pipelined = NULL;
1789         ret = 0;
1790
1791         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1792                 reg = &dev_priv->fence_regs[obj->fence_reg];
1793                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1794
1795                 if (obj->tiling_changed) {
1796                         ret = i915_gem_object_flush_fence(obj, pipelined);
1797                         if (ret)
1798                                 return ret;
1799
1800                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1801                                 pipelined = NULL;
1802
1803                         if (pipelined) {
1804                                 reg->setup_seqno =
1805                                         i915_gem_next_request_seqno(pipelined);
1806                                 obj->last_fenced_seqno = reg->setup_seqno;
1807                                 obj->last_fenced_ring = pipelined;
1808                         }
1809
1810                         goto update;
1811                 }
1812
1813                 if (!pipelined) {
1814                         if (reg->setup_seqno) {
1815                                 if (!ring_passed_seqno(obj->last_fenced_ring,
1816                                     reg->setup_seqno)) {
1817                                         ret = i915_wait_seqno(
1818                                             obj->last_fenced_ring,
1819                                             reg->setup_seqno);
1820                                         if (ret)
1821                                                 return ret;
1822                                 }
1823
1824                                 reg->setup_seqno = 0;
1825                         }
1826                 } else if (obj->last_fenced_ring &&
1827                            obj->last_fenced_ring != pipelined) {
1828                         ret = i915_gem_object_flush_fence(obj, pipelined);
1829                         if (ret)
1830                                 return ret;
1831                 }
1832
1833                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1834                         pipelined = NULL;
1835                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
1836
1837                 if (obj->tiling_changed) {
1838                         if (pipelined) {
1839                                 reg->setup_seqno =
1840                                         i915_gem_next_request_seqno(pipelined);
1841                                 obj->last_fenced_seqno = reg->setup_seqno;
1842                                 obj->last_fenced_ring = pipelined;
1843                         }
1844                         goto update;
1845                 }
1846
1847                 return 0;
1848         }
1849
1850         reg = i915_find_fence_reg(dev, pipelined);
1851         if (reg == NULL)
1852                 return -EDEADLK;
1853
1854         ret = i915_gem_object_flush_fence(obj, pipelined);
1855         if (ret)
1856                 return ret;
1857
1858         if (reg->obj) {
1859                 struct drm_i915_gem_object *old = reg->obj;
1860
1861                 drm_gem_object_reference(&old->base);
1862
1863                 if (old->tiling_mode)
1864                         i915_gem_release_mmap(old);
1865
1866                 ret = i915_gem_object_flush_fence(old, pipelined);
1867                 if (ret) {
1868                         drm_gem_object_unreference(&old->base);
1869                         return ret;
1870                 }
1871
1872                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
1873                         pipelined = NULL;
1874
1875                 old->fence_reg = I915_FENCE_REG_NONE;
1876                 old->last_fenced_ring = pipelined;
1877                 old->last_fenced_seqno =
1878                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1879
1880                 drm_gem_object_unreference(&old->base);
1881         } else if (obj->last_fenced_seqno == 0)
1882                 pipelined = NULL;
1883
1884         reg->obj = obj;
1885         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1886         obj->fence_reg = reg - dev_priv->fence_regs;
1887         obj->last_fenced_ring = pipelined;
1888
1889         reg->setup_seqno =
1890                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1891         obj->last_fenced_seqno = reg->setup_seqno;
1892
1893 update:
1894         obj->tiling_changed = false;
1895         switch (INTEL_INFO(dev)->gen) {
1896         case 7:
1897         case 6:
1898                 ret = sandybridge_write_fence_reg(obj, pipelined);
1899                 break;
1900         case 5:
1901         case 4:
1902                 ret = i965_write_fence_reg(obj, pipelined);
1903                 break;
1904         case 3:
1905                 ret = i915_write_fence_reg(obj, pipelined);
1906                 break;
1907         case 2:
1908                 ret = i830_write_fence_reg(obj, pipelined);
1909                 break;
1910         }
1911
1912         return ret;
1913 }
1914
1915 static int
1916 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1917     unsigned alignment, bool map_and_fenceable)
1918 {
1919         struct drm_device *dev;
1920         struct drm_i915_private *dev_priv;
1921         struct drm_mm_node *free_space;
1922         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1923         bool mappable, fenceable;
1924         int ret;
1925
1926         dev = obj->base.dev;
1927         dev_priv = dev->dev_private;
1928
1929         if (obj->madv != I915_MADV_WILLNEED) {
1930                 DRM_ERROR("Attempting to bind a purgeable object\n");
1931                 return (-EINVAL);
1932         }
1933
1934         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1935             obj->tiling_mode);
1936         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1937             obj->tiling_mode);
1938         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1939             obj->base.size, obj->tiling_mode);
1940         if (alignment == 0)
1941                 alignment = map_and_fenceable ? fence_alignment :
1942                     unfenced_alignment;
1943         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1944                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1945                 return (-EINVAL);
1946         }
1947
1948         size = map_and_fenceable ? fence_size : obj->base.size;
1949
1950         /* If the object is bigger than the entire aperture, reject it early
1951          * before evicting everything in a vain attempt to find space.
1952          */
1953         if (obj->base.size > (map_and_fenceable ?
1954             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1955                 DRM_ERROR(
1956 "Attempting to bind an object larger than the aperture\n");
1957                 return (-E2BIG);
1958         }
1959
1960  search_free:
1961         if (map_and_fenceable)
1962                 free_space = drm_mm_search_free_in_range(
1963                     &dev_priv->mm.gtt_space, size, alignment, 0,
1964                     dev_priv->mm.gtt_mappable_end, 0);
1965         else
1966                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1967                     size, alignment, 0);
1968         if (free_space != NULL) {
1969                 int color = 0;
1970                 if (map_and_fenceable)
1971                         obj->gtt_space = drm_mm_get_block_range_generic(
1972                             free_space, size, alignment, color, 0,
1973                             dev_priv->mm.gtt_mappable_end, 1);
1974                 else
1975                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1976                             size, alignment, color, 1);
1977         }
1978         if (obj->gtt_space == NULL) {
1979                 ret = i915_gem_evict_something(dev, size, alignment,
1980                     map_and_fenceable);
1981                 if (ret != 0)
1982                         return (ret);
1983                 goto search_free;
1984         }
1985
1986         /*
1987          * NOTE: i915_gem_object_get_pages_gtt() cannot
1988          *       return ENOMEM, since we used VM_ALLOC_RETRY.
1989          */
1990         ret = i915_gem_object_get_pages_gtt(obj, 0);
1991         if (ret != 0) {
1992                 drm_mm_put_block(obj->gtt_space);
1993                 obj->gtt_space = NULL;
1994                 return (ret);
1995         }
1996
1997         i915_gem_gtt_bind_object(obj, obj->cache_level);
1998         if (ret != 0) {
1999                 i915_gem_object_put_pages_gtt(obj);
2000                 drm_mm_put_block(obj->gtt_space);
2001                 obj->gtt_space = NULL;
2002                 if (i915_gem_evict_everything(dev, false))
2003                         return (ret);
2004                 goto search_free;
2005         }
2006
2007         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2008         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2009
2010         obj->gtt_offset = obj->gtt_space->start;
2011
2012         fenceable =
2013                 obj->gtt_space->size == fence_size &&
2014                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2015
2016         mappable =
2017                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2018         obj->map_and_fenceable = mappable && fenceable;
2019
2020         return (0);
2021 }
2022
2023 void
2024 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2025 {
2026
2027         /* If we don't have a page list set up, then we're not pinned
2028          * to GPU, and we can ignore the cache flush because it'll happen
2029          * again at bind time.
2030          */
2031         if (obj->pages == NULL)
2032                 return;
2033
2034         /* If the GPU is snooping the contents of the CPU cache,
2035          * we do not need to manually clear the CPU cache lines.  However,
2036          * the caches are only snooped when the render cache is
2037          * flushed/invalidated.  As we always have to emit invalidations
2038          * and flushes when moving into and out of the RENDER domain, correct
2039          * snooping behaviour occurs naturally as the result of our domain
2040          * tracking.
2041          */
2042         if (obj->cache_level != I915_CACHE_NONE)
2043                 return;
2044
2045         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2046 }
2047
2048 /** Flushes the GTT write domain for the object if it's dirty. */
2049 static void
2050 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2051 {
2052         uint32_t old_write_domain;
2053
2054         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2055                 return;
2056
2057         /* No actual flushing is required for the GTT write domain.  Writes
2058          * to it immediately go to main memory as far as we know, so there's
2059          * no chipset flush.  It also doesn't land in render cache.
2060          *
2061          * However, we do have to enforce the order so that all writes through
2062          * the GTT land before any writes to the device, such as updates to
2063          * the GATT itself.
2064          */
2065         cpu_sfence();
2066
2067         old_write_domain = obj->base.write_domain;
2068         obj->base.write_domain = 0;
2069 }
2070
2071 /** Flushes the CPU write domain for the object if it's dirty. */
2072 static void
2073 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2074 {
2075         uint32_t old_write_domain;
2076
2077         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2078                 return;
2079
2080         i915_gem_clflush_object(obj);
2081         intel_gtt_chipset_flush();
2082         old_write_domain = obj->base.write_domain;
2083         obj->base.write_domain = 0;
2084 }
2085
2086 static int
2087 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2088 {
2089
2090         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2091                 return (0);
2092         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
2093 }
2094
2095 /**
2096  * Moves a single object to the GTT read, and possibly write domain.
2097  *
2098  * This function returns when the move is complete, including waiting on
2099  * flushes to occur.
2100  */
2101 int
2102 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2103 {
2104         uint32_t old_write_domain, old_read_domains;
2105         int ret;
2106
2107         if (obj->gtt_space == NULL)
2108                 return (-EINVAL);
2109
2110         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2111                 return 0;
2112
2113         ret = i915_gem_object_flush_gpu_write_domain(obj);
2114         if (ret != 0)
2115                 return (ret);
2116
2117         if (obj->pending_gpu_write || write) {
2118                 ret = i915_gem_object_wait_rendering(obj);
2119                 if (ret != 0)
2120                         return (ret);
2121         }
2122
2123         i915_gem_object_flush_cpu_write_domain(obj);
2124
2125         old_write_domain = obj->base.write_domain;
2126         old_read_domains = obj->base.read_domains;
2127
2128         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2129             ("In GTT write domain"));
2130         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2131         if (write) {
2132                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2133                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2134                 obj->dirty = 1;
2135         }
2136
2137         return (0);
2138 }
2139
2140 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2141                                     enum i915_cache_level cache_level)
2142 {
2143         struct drm_device *dev = obj->base.dev;
2144         drm_i915_private_t *dev_priv = dev->dev_private;
2145         int ret;
2146
2147         if (obj->cache_level == cache_level)
2148                 return 0;
2149
2150         if (obj->pin_count) {
2151                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2152                 return -EBUSY;
2153         }
2154
2155         if (obj->gtt_space) {
2156                 ret = i915_gem_object_finish_gpu(obj);
2157                 if (ret != 0)
2158                         return (ret);
2159
2160                 i915_gem_object_finish_gtt(obj);
2161
2162                 /* Before SandyBridge, you could not use tiling or fence
2163                  * registers with snooped memory, so relinquish any fences
2164                  * currently pointing to our region in the aperture.
2165                  */
2166                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2167                         ret = i915_gem_object_put_fence(obj);
2168                         if (ret)
2169                                 return ret;
2170                 }
2171
2172                 if (obj->has_global_gtt_mapping)
2173                         i915_gem_gtt_bind_object(obj, cache_level);
2174                 if (obj->has_aliasing_ppgtt_mapping)
2175                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2176                                                obj, cache_level);
2177         }
2178
2179         if (cache_level == I915_CACHE_NONE) {
2180                 u32 old_read_domains, old_write_domain;
2181
2182                 /* If we're coming from LLC cached, then we haven't
2183                  * actually been tracking whether the data is in the
2184                  * CPU cache or not, since we only allow one bit set
2185                  * in obj->write_domain and have been skipping the clflushes.
2186                  * Just set it to the CPU cache for now.
2187                  */
2188                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
2189                     ("obj %p in CPU write domain", obj));
2190                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
2191                     ("obj %p in CPU read domain", obj));
2192
2193                 old_read_domains = obj->base.read_domains;
2194                 old_write_domain = obj->base.write_domain;
2195
2196                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2197                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2198
2199         }
2200
2201         obj->cache_level = cache_level;
2202         return 0;
2203 }
2204
2205 /*
2206  * Prepare buffer for display plane (scanout, cursors, etc).
2207  * Can be called from an uninterruptible phase (modesetting) and allows
2208  * any flushes to be pipelined (for pageflips).
2209  */
2210 int
2211 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2212                                      u32 alignment,
2213                                      struct intel_ring_buffer *pipelined)
2214 {
2215         u32 old_read_domains, old_write_domain;
2216         int ret;
2217
2218         ret = i915_gem_object_flush_gpu_write_domain(obj);
2219         if (ret != 0)
2220                 return (ret);
2221
2222         if (pipelined != obj->ring) {
2223                 ret = i915_gem_object_sync(obj, pipelined);
2224                 if (ret)
2225                         return ret;
2226         }
2227
2228         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2229         if (ret != 0)
2230                 return (ret);
2231
2232         ret = i915_gem_object_pin(obj, alignment, true);
2233         if (ret != 0)
2234                 return (ret);
2235
2236         i915_gem_object_flush_cpu_write_domain(obj);
2237
2238         old_write_domain = obj->base.write_domain;
2239         old_read_domains = obj->base.read_domains;
2240
2241         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2242             ("obj %p in GTT write domain", obj));
2243         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2244
2245         return (0);
2246 }
2247
2248 int
2249 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2250 {
2251         int ret;
2252
2253         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2254                 return (0);
2255
2256         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2257                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2258                 if (ret != 0)
2259                         return (ret);
2260         }
2261
2262         ret = i915_gem_object_wait_rendering(obj);
2263         if (ret != 0)
2264                 return (ret);
2265
2266         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2267
2268         return (0);
2269 }
2270
2271 /**
2272  * Moves a single object to the CPU read, and possibly write domain.
2273  *
2274  * This function returns when the move is complete, including waiting on
2275  * flushes to occur.
2276  */
2277 int
2278 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2279 {
2280         uint32_t old_write_domain, old_read_domains;
2281         int ret;
2282
2283         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2284                 return 0;
2285
2286         ret = i915_gem_object_flush_gpu_write_domain(obj);
2287         if (ret != 0)
2288                 return (ret);
2289
2290         ret = i915_gem_object_wait_rendering(obj);
2291         if (ret)
2292                 return ret;
2293
2294         i915_gem_object_flush_gtt_write_domain(obj);
2295
2296         old_write_domain = obj->base.write_domain;
2297         old_read_domains = obj->base.read_domains;
2298
2299         /* Flush the CPU cache if it's still invalid. */
2300         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2301                 i915_gem_clflush_object(obj);
2302
2303                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2304         }
2305
2306         /* It should now be out of any other write domains, and we can update
2307          * the domain values for our changes.
2308          */
2309         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2310
2311         /* If we're writing through the CPU, then the GPU read domains will
2312          * need to be invalidated at next use.
2313          */
2314         if (write) {
2315                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2316                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2317         }
2318
2319         return 0;
2320 }
2321
2322 /* Throttle our rendering by waiting until the ring has completed our requests
2323  * emitted over 20 msec ago.
2324  *
2325  * Note that if we were to use the current jiffies each time around the loop,
2326  * we wouldn't escape the function with any frames outstanding if the time to
2327  * render a frame was over 20ms.
2328  *
2329  * This should get us reasonable parallelism between CPU and GPU but also
2330  * relatively low latency when blocking on a particular request to finish.
2331  */
2332 static int
2333 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2334 {
2335         struct drm_i915_private *dev_priv = dev->dev_private;
2336         struct drm_i915_file_private *file_priv = file->driver_priv;
2337         unsigned long recent_enough = ticks - (20 * hz / 1000);
2338         struct drm_i915_gem_request *request;
2339         struct intel_ring_buffer *ring = NULL;
2340         u32 seqno = 0;
2341         int ret;
2342
2343         if (atomic_read(&dev_priv->mm.wedged))
2344                 return -EIO;
2345
2346         spin_lock(&file_priv->mm.lock);
2347         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2348                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2349                         break;
2350
2351                 ring = request->ring;
2352                 seqno = request->seqno;
2353         }
2354         spin_unlock(&file_priv->mm.lock);
2355
2356         if (seqno == 0)
2357                 return 0;
2358
2359         ret = __wait_seqno(ring, seqno, true, NULL);
2360
2361         if (ret == 0)
2362                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2363
2364         return ret;
2365 }
2366
2367 int
2368 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
2369      bool map_and_fenceable)
2370 {
2371         struct drm_device *dev;
2372         struct drm_i915_private *dev_priv;
2373         int ret;
2374
2375         dev = obj->base.dev;
2376         dev_priv = dev->dev_private;
2377
2378         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
2379             ("Max pin count"));
2380
2381         if (obj->gtt_space != NULL) {
2382                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
2383                     (map_and_fenceable && !obj->map_and_fenceable)) {
2384                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
2385                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2386                              " obj->map_and_fenceable=%d\n",
2387                              obj->gtt_offset, alignment,
2388                              map_and_fenceable,
2389                              obj->map_and_fenceable);
2390                         ret = i915_gem_object_unbind(obj);
2391                         if (ret != 0)
2392                                 return (ret);
2393                 }
2394         }
2395
2396         if (obj->gtt_space == NULL) {
2397                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
2398                     map_and_fenceable);
2399                 if (ret)
2400                         return (ret);
2401         }
2402
2403         if (obj->pin_count++ == 0 && !obj->active)
2404                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2405         obj->pin_mappable |= map_and_fenceable;
2406
2407 #if 1
2408         KIB_NOTYET();
2409 #else
2410         WARN_ON(i915_verify_lists(dev));
2411 #endif
2412         return (0);
2413 }
2414
2415 void
2416 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
2417 {
2418         struct drm_device *dev;
2419         drm_i915_private_t *dev_priv;
2420
2421         dev = obj->base.dev;
2422         dev_priv = dev->dev_private;
2423
2424 #if 1
2425         KIB_NOTYET();
2426 #else
2427         WARN_ON(i915_verify_lists(dev));
2428 #endif
2429
2430         KASSERT(obj->pin_count != 0, ("zero pin count"));
2431         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
2432
2433         if (--obj->pin_count == 0) {
2434                 if (!obj->active)
2435                         list_move_tail(&obj->mm_list,
2436                             &dev_priv->mm.inactive_list);
2437                 obj->pin_mappable = false;
2438         }
2439 #if 1
2440         KIB_NOTYET();
2441 #else
2442         WARN_ON(i915_verify_lists(dev));
2443 #endif
2444 }
2445
2446 int
2447 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2448     struct drm_file *file)
2449 {
2450         struct drm_i915_gem_pin *args;
2451         struct drm_i915_gem_object *obj;
2452         struct drm_gem_object *gobj;
2453         int ret;
2454
2455         args = data;
2456
2457         ret = i915_mutex_lock_interruptible(dev);
2458         if (ret != 0)
2459                 return ret;
2460
2461         gobj = drm_gem_object_lookup(dev, file, args->handle);
2462         if (gobj == NULL) {
2463                 ret = -ENOENT;
2464                 goto unlock;
2465         }
2466         obj = to_intel_bo(gobj);
2467
2468         if (obj->madv != I915_MADV_WILLNEED) {
2469                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2470                 ret = -EINVAL;
2471                 goto out;
2472         }
2473
2474         if (obj->pin_filp != NULL && obj->pin_filp != file) {
2475                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2476                     args->handle);
2477                 ret = -EINVAL;
2478                 goto out;
2479         }
2480
2481         obj->user_pin_count++;
2482         obj->pin_filp = file;
2483         if (obj->user_pin_count == 1) {
2484                 ret = i915_gem_object_pin(obj, args->alignment, true);
2485                 if (ret != 0)
2486                         goto out;
2487         }
2488
2489         /* XXX - flush the CPU caches for pinned objects
2490          * as the X server doesn't manage domains yet
2491          */
2492         i915_gem_object_flush_cpu_write_domain(obj);
2493         args->offset = obj->gtt_offset;
2494 out:
2495         drm_gem_object_unreference(&obj->base);
2496 unlock:
2497         DRM_UNLOCK(dev);
2498         return (ret);
2499 }
2500
2501 int
2502 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2503     struct drm_file *file)
2504 {
2505         struct drm_i915_gem_pin *args;
2506         struct drm_i915_gem_object *obj;
2507         int ret;
2508
2509         args = data;
2510         ret = i915_mutex_lock_interruptible(dev);
2511         if (ret != 0)
2512                 return (ret);
2513
2514         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2515         if (&obj->base == NULL) {
2516                 ret = -ENOENT;
2517                 goto unlock;
2518         }
2519
2520         if (obj->pin_filp != file) {
2521                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2522                     args->handle);
2523                 ret = -EINVAL;
2524                 goto out;
2525         }
2526         obj->user_pin_count--;
2527         if (obj->user_pin_count == 0) {
2528                 obj->pin_filp = NULL;
2529                 i915_gem_object_unpin(obj);
2530         }
2531
2532 out:
2533         drm_gem_object_unreference(&obj->base);
2534 unlock:
2535         DRM_UNLOCK(dev);
2536         return (ret);
2537 }
2538
2539 int
2540 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2541                     struct drm_file *file)
2542 {
2543         struct drm_i915_gem_busy *args;
2544         struct drm_i915_gem_object *obj;
2545         struct drm_i915_gem_request *request;
2546         int ret;
2547
2548         args = data;
2549
2550         ret = i915_mutex_lock_interruptible(dev);
2551         if (ret != 0)
2552                 return ret;
2553
2554         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2555         if (&obj->base == NULL) {
2556                 ret = -ENOENT;
2557                 goto unlock;
2558         }
2559
2560         args->busy = obj->active;
2561         if (args->busy) {
2562                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2563                         ret = i915_gem_flush_ring(obj->ring,
2564                             0, obj->base.write_domain);
2565                 } else if (obj->ring->outstanding_lazy_request ==
2566                     obj->last_rendering_seqno) {
2567                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
2568                             M_WAITOK | M_ZERO);
2569                         ret = i915_add_request(obj->ring, NULL, request);
2570                         if (ret != 0)
2571                                 drm_free(request, DRM_I915_GEM);
2572                 }
2573
2574                 i915_gem_retire_requests_ring(obj->ring);
2575                 args->busy = obj->active;
2576         }
2577
2578         drm_gem_object_unreference(&obj->base);
2579 unlock:
2580         DRM_UNLOCK(dev);
2581         return (ret);
2582 }
2583
2584 int
2585 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2586     struct drm_file *file_priv)
2587 {
2588
2589         return (i915_gem_ring_throttle(dev, file_priv));
2590 }
2591
2592 int
2593 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
2594                        struct drm_file *file_priv)
2595 {
2596         struct drm_i915_gem_madvise *args = data;
2597         struct drm_i915_gem_object *obj;
2598         int ret;
2599
2600         switch (args->madv) {
2601         case I915_MADV_DONTNEED:
2602         case I915_MADV_WILLNEED:
2603             break;
2604         default:
2605             return -EINVAL;
2606         }
2607
2608         ret = i915_mutex_lock_interruptible(dev);
2609         if (ret)
2610                 return ret;
2611
2612         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
2613         if (&obj->base == NULL) {
2614                 ret = -ENOENT;
2615                 goto unlock;
2616         }
2617
2618         if (obj->pin_count) {
2619                 ret = -EINVAL;
2620                 goto out;
2621         }
2622
2623         if (obj->madv != __I915_MADV_PURGED)
2624                 obj->madv = args->madv;
2625
2626         /* if the object is no longer attached, discard its backing storage */
2627         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
2628                 i915_gem_object_truncate(obj);
2629
2630         args->retained = obj->madv != __I915_MADV_PURGED;
2631
2632 out:
2633         drm_gem_object_unreference(&obj->base);
2634 unlock:
2635         DRM_UNLOCK(dev);
2636         return ret;
2637 }
2638
2639 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
2640                                                   size_t size)
2641 {
2642         struct drm_i915_private *dev_priv;
2643         struct drm_i915_gem_object *obj;
2644
2645         dev_priv = dev->dev_private;
2646
2647         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
2648
2649         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
2650                 drm_free(obj, DRM_I915_GEM);
2651                 return (NULL);
2652         }
2653
2654         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2655         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2656
2657         if (HAS_LLC(dev))
2658                 obj->cache_level = I915_CACHE_LLC;
2659         else
2660                 obj->cache_level = I915_CACHE_NONE;
2661         obj->base.driver_private = NULL;
2662         obj->fence_reg = I915_FENCE_REG_NONE;
2663         INIT_LIST_HEAD(&obj->mm_list);
2664         INIT_LIST_HEAD(&obj->gtt_list);
2665         INIT_LIST_HEAD(&obj->ring_list);
2666         INIT_LIST_HEAD(&obj->exec_list);
2667         INIT_LIST_HEAD(&obj->gpu_write_list);
2668         obj->madv = I915_MADV_WILLNEED;
2669         /* Avoid an unnecessary call to unbind on the first bind. */
2670         obj->map_and_fenceable = true;
2671
2672         i915_gem_info_add_obj(dev_priv, size);
2673
2674         return (obj);
2675 }
2676
2677 int i915_gem_init_object(struct drm_gem_object *obj)
2678 {
2679
2680         kprintf("i915_gem_init_object called\n");
2681         return (0);
2682 }
2683
2684 static void
2685 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
2686 {
2687         struct drm_device *dev;
2688         drm_i915_private_t *dev_priv;
2689         int ret;
2690
2691         dev = obj->base.dev;
2692         dev_priv = dev->dev_private;
2693
2694         ret = i915_gem_object_unbind(obj);
2695         if (ret == -ERESTART) {
2696                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
2697                 return;
2698         }
2699
2700         drm_gem_free_mmap_offset(&obj->base);
2701         drm_gem_object_release(&obj->base);
2702         i915_gem_info_remove_obj(dev_priv, obj->base.size);
2703
2704         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
2705         drm_free(obj->bit_17, DRM_I915_GEM);
2706         drm_free(obj, DRM_I915_GEM);
2707 }
2708
2709 void
2710 i915_gem_free_object(struct drm_gem_object *gem_obj)
2711 {
2712         struct drm_i915_gem_object *obj;
2713         struct drm_device *dev;
2714
2715         obj = to_intel_bo(gem_obj);
2716         dev = obj->base.dev;
2717
2718         while (obj->pin_count > 0)
2719                 i915_gem_object_unpin(obj);
2720
2721         if (obj->phys_obj != NULL)
2722                 i915_gem_detach_phys_object(dev, obj);
2723
2724         i915_gem_free_object_tail(obj);
2725 }
2726
2727 int
2728 i915_gem_do_init(struct drm_device *dev, unsigned long start,
2729     unsigned long mappable_end, unsigned long end)
2730 {
2731         drm_i915_private_t *dev_priv;
2732         unsigned long mappable;
2733         int error;
2734
2735         dev_priv = dev->dev_private;
2736         mappable = min(end, mappable_end) - start;
2737
2738         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
2739
2740         dev_priv->mm.gtt_start = start;
2741         dev_priv->mm.gtt_mappable_end = mappable_end;
2742         dev_priv->mm.gtt_end = end;
2743         dev_priv->mm.gtt_total = end - start;
2744         dev_priv->mm.mappable_gtt_total = mappable;
2745
2746         /* Take over this portion of the GTT */
2747         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
2748         device_printf(dev->dev,
2749             "taking over the fictitious range 0x%lx-0x%lx\n",
2750             dev->agp->base + start, dev->agp->base + start + mappable);
2751         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
2752             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2753         return (error);
2754 }
2755
2756 int
2757 i915_gem_idle(struct drm_device *dev)
2758 {
2759         drm_i915_private_t *dev_priv;
2760         int ret;
2761
2762         dev_priv = dev->dev_private;
2763         if (dev_priv->mm.suspended)
2764                 return (0);
2765
2766         ret = i915_gpu_idle(dev);
2767         if (ret != 0)
2768                 return (ret);
2769
2770         /* Under UMS, be paranoid and evict. */
2771         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
2772                 ret = i915_gem_evict_inactive(dev, false);
2773                 if (ret != 0)
2774                         return ret;
2775         }
2776
2777         i915_gem_reset_fences(dev);
2778
2779         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2780          * We need to replace this with a semaphore, or something.
2781          * And not confound mm.suspended!
2782          */
2783         dev_priv->mm.suspended = 1;
2784         del_timer_sync(&dev_priv->hangcheck_timer);
2785
2786         i915_kernel_lost_context(dev);
2787         i915_gem_cleanup_ringbuffer(dev);
2788
2789         /* Cancel the retire work handler, which should be idle now. */
2790         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2791
2792         return (ret);
2793 }
2794
2795 void i915_gem_l3_remap(struct drm_device *dev)
2796 {
2797         drm_i915_private_t *dev_priv = dev->dev_private;
2798         u32 misccpctl;
2799         int i;
2800
2801         if (!HAS_L3_GPU_CACHE(dev))
2802                 return;
2803
2804         if (!dev_priv->l3_parity.remap_info)
2805                 return;
2806
2807         misccpctl = I915_READ(GEN7_MISCCPCTL);
2808         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
2809         POSTING_READ(GEN7_MISCCPCTL);
2810
2811         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
2812                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
2813                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
2814                         DRM_DEBUG("0x%x was already programmed to %x\n",
2815                                   GEN7_L3LOG_BASE + i, remap);
2816                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
2817                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
2818                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
2819         }
2820
2821         /* Make sure all the writes land before disabling dop clock gating */
2822         POSTING_READ(GEN7_L3LOG_BASE);
2823
2824         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
2825 }
2826
2827 void
2828 i915_gem_init_swizzling(struct drm_device *dev)
2829 {
2830         drm_i915_private_t *dev_priv;
2831
2832         dev_priv = dev->dev_private;
2833
2834         if (INTEL_INFO(dev)->gen < 5 ||
2835             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
2836                 return;
2837
2838         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
2839                                  DISP_TILE_SURFACE_SWIZZLING);
2840
2841         if (IS_GEN5(dev))
2842                 return;
2843
2844         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
2845         if (IS_GEN6(dev))
2846                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
2847         else
2848                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
2849 }
2850
2851 static bool
2852 intel_enable_blt(struct drm_device *dev)
2853 {
2854         int revision;
2855
2856         if (!HAS_BLT(dev))
2857                 return false;
2858
2859         /* The blitter was dysfunctional on early prototypes */
2860         revision = pci_read_config(dev->dev, PCIR_REVID, 1);
2861         if (IS_GEN6(dev) && revision < 8) {
2862                 DRM_INFO("BLT not supported on this pre-production hardware;"
2863                          " graphics performance will be degraded.\n");
2864                 return false;
2865         }
2866
2867         return true;
2868 }
2869
2870 int
2871 i915_gem_init_hw(struct drm_device *dev)
2872 {
2873         drm_i915_private_t *dev_priv = dev->dev_private;
2874         int ret;
2875
2876         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
2877                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
2878
2879         i915_gem_l3_remap(dev);
2880
2881         i915_gem_init_swizzling(dev);
2882
2883         ret = intel_init_render_ring_buffer(dev);
2884         if (ret)
2885                 return ret;
2886
2887         if (HAS_BSD(dev)) {
2888                 ret = intel_init_bsd_ring_buffer(dev);
2889                 if (ret)
2890                         goto cleanup_render_ring;
2891         }
2892
2893         if (intel_enable_blt(dev)) {
2894                 ret = intel_init_blt_ring_buffer(dev);
2895                 if (ret)
2896                         goto cleanup_bsd_ring;
2897         }
2898
2899         dev_priv->next_seqno = 1;
2900
2901         /*
2902          * XXX: There was some w/a described somewhere suggesting loading
2903          * contexts before PPGTT.
2904          */
2905 #if 0   /* XXX: HW context support */
2906         i915_gem_context_init(dev);
2907 #endif
2908         i915_gem_init_ppgtt(dev);
2909
2910         return 0;
2911
2912 cleanup_bsd_ring:
2913         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
2914 cleanup_render_ring:
2915         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
2916         return ret;
2917 }
2918
2919 void
2920 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2921 {
2922         drm_i915_private_t *dev_priv;
2923         int i;
2924
2925         dev_priv = dev->dev_private;
2926         for (i = 0; i < I915_NUM_RINGS; i++)
2927                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
2928 }
2929
2930 int
2931 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2932                        struct drm_file *file_priv)
2933 {
2934         drm_i915_private_t *dev_priv = dev->dev_private;
2935         int ret;
2936
2937         if (drm_core_check_feature(dev, DRIVER_MODESET))
2938                 return 0;
2939
2940         if (atomic_read(&dev_priv->mm.wedged)) {
2941                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2942                 atomic_set(&dev_priv->mm.wedged, 0);
2943         }
2944
2945         DRM_LOCK(dev);
2946         dev_priv->mm.suspended = 0;
2947
2948         ret = i915_gem_init_hw(dev);
2949         if (ret != 0) {
2950                 DRM_UNLOCK(dev);
2951                 return ret;
2952         }
2953
2954         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
2955         DRM_UNLOCK(dev);
2956
2957         ret = drm_irq_install(dev);
2958         if (ret)
2959                 goto cleanup_ringbuffer;
2960
2961         return 0;
2962
2963 cleanup_ringbuffer:
2964         DRM_LOCK(dev);
2965         i915_gem_cleanup_ringbuffer(dev);
2966         dev_priv->mm.suspended = 1;
2967         DRM_UNLOCK(dev);
2968
2969         return ret;
2970 }
2971
2972 int
2973 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2974     struct drm_file *file_priv)
2975 {
2976
2977         if (drm_core_check_feature(dev, DRIVER_MODESET))
2978                 return 0;
2979
2980         drm_irq_uninstall(dev);
2981         return (i915_gem_idle(dev));
2982 }
2983
2984 void
2985 i915_gem_lastclose(struct drm_device *dev)
2986 {
2987         int ret;
2988
2989         if (drm_core_check_feature(dev, DRIVER_MODESET))
2990                 return;
2991
2992         ret = i915_gem_idle(dev);
2993         if (ret != 0)
2994                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2995 }
2996
2997 static void
2998 init_ring_lists(struct intel_ring_buffer *ring)
2999 {
3000
3001         INIT_LIST_HEAD(&ring->active_list);
3002         INIT_LIST_HEAD(&ring->request_list);
3003         INIT_LIST_HEAD(&ring->gpu_write_list);
3004 }
3005
3006 void
3007 i915_gem_load(struct drm_device *dev)
3008 {
3009         int i;
3010         drm_i915_private_t *dev_priv = dev->dev_private;
3011
3012         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3013         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3014         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3015         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3016         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3017         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3018         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3019         for (i = 0; i < I915_NUM_RINGS; i++)
3020                 init_ring_lists(&dev_priv->ring[i]);
3021         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3022                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3023         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3024                           i915_gem_retire_work_handler);
3025         init_completion(&dev_priv->error_completion);
3026
3027         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3028         if (IS_GEN3(dev)) {
3029                 I915_WRITE(MI_ARB_STATE,
3030                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3031         }
3032
3033         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3034
3035         /* Old X drivers will take 0-2 for front, back, depth buffers */
3036         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3037                 dev_priv->fence_reg_start = 3;
3038
3039         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3040                 dev_priv->num_fence_regs = 16;
3041         else
3042                 dev_priv->num_fence_regs = 8;
3043
3044         /* Initialize fence registers to zero */
3045         i915_gem_reset_fences(dev);
3046
3047         i915_gem_detect_bit_6_swizzle(dev);
3048
3049         dev_priv->mm.interruptible = true;
3050
3051         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
3052             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
3053 }
3054
3055 static int
3056 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
3057 {
3058         drm_i915_private_t *dev_priv;
3059         struct drm_i915_gem_phys_object *phys_obj;
3060         int ret;
3061
3062         dev_priv = dev->dev_private;
3063         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
3064                 return (0);
3065
3066         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3067             M_WAITOK | M_ZERO);
3068
3069         phys_obj->id = id;
3070
3071         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3072         if (phys_obj->handle == NULL) {
3073                 ret = -ENOMEM;
3074                 goto free_obj;
3075         }
3076         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3077             size / PAGE_SIZE, PAT_WRITE_COMBINING);
3078
3079         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3080
3081         return (0);
3082
3083 free_obj:
3084         drm_free(phys_obj, DRM_I915_GEM);
3085         return (ret);
3086 }
3087
3088 static void
3089 i915_gem_free_phys_object(struct drm_device *dev, int id)
3090 {
3091         drm_i915_private_t *dev_priv;
3092         struct drm_i915_gem_phys_object *phys_obj;
3093
3094         dev_priv = dev->dev_private;
3095         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3096                 return;
3097
3098         phys_obj = dev_priv->mm.phys_objs[id - 1];
3099         if (phys_obj->cur_obj != NULL)
3100                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3101
3102         drm_pci_free(dev, phys_obj->handle);
3103         drm_free(phys_obj, DRM_I915_GEM);
3104         dev_priv->mm.phys_objs[id - 1] = NULL;
3105 }
3106
3107 void
3108 i915_gem_free_all_phys_object(struct drm_device *dev)
3109 {
3110         int i;
3111
3112         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3113                 i915_gem_free_phys_object(dev, i);
3114 }
3115
3116 void
3117 i915_gem_detach_phys_object(struct drm_device *dev,
3118     struct drm_i915_gem_object *obj)
3119 {
3120         vm_page_t m;
3121         struct sf_buf *sf;
3122         char *vaddr, *dst;
3123         int i, page_count;
3124
3125         if (obj->phys_obj == NULL)
3126                 return;
3127         vaddr = obj->phys_obj->handle->vaddr;
3128
3129         page_count = obj->base.size / PAGE_SIZE;
3130         VM_OBJECT_LOCK(obj->base.vm_obj);
3131         for (i = 0; i < page_count; i++) {
3132                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3133                 if (m == NULL)
3134                         continue; /* XXX */
3135
3136                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3137                 sf = sf_buf_alloc(m);
3138                 if (sf != NULL) {
3139                         dst = (char *)sf_buf_kva(sf);
3140                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3141                         sf_buf_free(sf);
3142                 }
3143                 drm_clflush_pages(&m, 1);
3144
3145                 VM_OBJECT_LOCK(obj->base.vm_obj);
3146                 vm_page_reference(m);
3147                 vm_page_dirty(m);
3148                 vm_page_busy_wait(m, FALSE, "i915gem");
3149                 vm_page_unwire(m, 0);
3150                 vm_page_wakeup(m);
3151                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3152         }
3153         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3154         intel_gtt_chipset_flush();
3155
3156         obj->phys_obj->cur_obj = NULL;
3157         obj->phys_obj = NULL;
3158 }
3159
3160 int
3161 i915_gem_attach_phys_object(struct drm_device *dev,
3162                             struct drm_i915_gem_object *obj,
3163                             int id,
3164                             int align)
3165 {
3166         drm_i915_private_t *dev_priv;
3167         vm_page_t m;
3168         struct sf_buf *sf;
3169         char *dst, *src;
3170         int i, page_count, ret;
3171
3172         if (id > I915_MAX_PHYS_OBJECT)
3173                 return (-EINVAL);
3174
3175         if (obj->phys_obj != NULL) {
3176                 if (obj->phys_obj->id == id)
3177                         return (0);
3178                 i915_gem_detach_phys_object(dev, obj);
3179         }
3180
3181         dev_priv = dev->dev_private;
3182         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3183                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3184                 if (ret != 0) {
3185                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3186                                   id, obj->base.size);
3187                         return (ret);
3188                 }
3189         }
3190
3191         /* bind to the object */
3192         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3193         obj->phys_obj->cur_obj = obj;
3194
3195         page_count = obj->base.size / PAGE_SIZE;
3196
3197         VM_OBJECT_LOCK(obj->base.vm_obj);
3198         ret = 0;
3199         for (i = 0; i < page_count; i++) {
3200                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3201                 if (m == NULL) {
3202                         ret = -EIO;
3203                         break;
3204                 }
3205                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3206                 sf = sf_buf_alloc(m);
3207                 src = (char *)sf_buf_kva(sf);
3208                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3209                 memcpy(dst, src, PAGE_SIZE);
3210                 sf_buf_free(sf);
3211
3212                 VM_OBJECT_LOCK(obj->base.vm_obj);
3213
3214                 vm_page_reference(m);
3215                 vm_page_busy_wait(m, FALSE, "i915gem");
3216                 vm_page_unwire(m, 0);
3217                 vm_page_wakeup(m);
3218                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3219         }
3220         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3221
3222         return (0);
3223 }
3224
3225 static int
3226 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3227     uint64_t data_ptr, uint64_t offset, uint64_t size,
3228     struct drm_file *file_priv)
3229 {
3230         char *user_data, *vaddr;
3231         int ret;
3232
3233         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3234         user_data = (char *)(uintptr_t)data_ptr;
3235
3236         if (copyin_nofault(user_data, vaddr, size) != 0) {
3237                 /* The physical object once assigned is fixed for the lifetime
3238                  * of the obj, so we can safely drop the lock and continue
3239                  * to access vaddr.
3240                  */
3241                 DRM_UNLOCK(dev);
3242                 ret = -copyin(user_data, vaddr, size);
3243                 DRM_LOCK(dev);
3244                 if (ret != 0)
3245                         return (ret);
3246         }
3247
3248         intel_gtt_chipset_flush();
3249         return (0);
3250 }
3251
3252 void
3253 i915_gem_release(struct drm_device *dev, struct drm_file *file)
3254 {
3255         struct drm_i915_file_private *file_priv;
3256         struct drm_i915_gem_request *request;
3257
3258         file_priv = file->driver_priv;
3259
3260         /* Clean up our request list when the client is going away, so that
3261          * later retire_requests won't dereference our soon-to-be-gone
3262          * file_priv.
3263          */
3264         spin_lock(&file_priv->mm.lock);
3265         while (!list_empty(&file_priv->mm.request_list)) {
3266                 request = list_first_entry(&file_priv->mm.request_list,
3267                                            struct drm_i915_gem_request,
3268                                            client_list);
3269                 list_del(&request->client_list);
3270                 request->file_priv = NULL;
3271         }
3272         spin_unlock(&file_priv->mm.lock);
3273 }
3274
3275 static int
3276 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
3277     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
3278     struct drm_file *file)
3279 {
3280         vm_object_t vm_obj;
3281         vm_page_t m;
3282         struct sf_buf *sf;
3283         vm_offset_t mkva;
3284         vm_pindex_t obj_pi;
3285         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
3286
3287         if (obj->gtt_offset != 0 && rw == UIO_READ)
3288                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
3289         else
3290                 do_bit17_swizzling = 0;
3291
3292         obj->dirty = 1;
3293         vm_obj = obj->base.vm_obj;
3294         ret = 0;
3295
3296         VM_OBJECT_LOCK(vm_obj);
3297         vm_object_pip_add(vm_obj, 1);
3298         while (size > 0) {
3299                 obj_pi = OFF_TO_IDX(offset);
3300                 obj_po = offset & PAGE_MASK;
3301
3302                 m = i915_gem_wire_page(vm_obj, obj_pi);
3303                 VM_OBJECT_UNLOCK(vm_obj);
3304
3305                 sf = sf_buf_alloc(m);
3306                 mkva = sf_buf_kva(sf);
3307                 length = min(size, PAGE_SIZE - obj_po);
3308                 while (length > 0) {
3309                         if (do_bit17_swizzling &&
3310                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
3311                                 cnt = roundup2(obj_po + 1, 64);
3312                                 cnt = min(cnt - obj_po, length);
3313                                 swizzled_po = obj_po ^ 64;
3314                         } else {
3315                                 cnt = length;
3316                                 swizzled_po = obj_po;
3317                         }
3318                         if (rw == UIO_READ)
3319                                 ret = -copyout_nofault(
3320                                     (char *)mkva + swizzled_po,
3321                                     (void *)(uintptr_t)data_ptr, cnt);
3322                         else
3323                                 ret = -copyin_nofault(
3324                                     (void *)(uintptr_t)data_ptr,
3325                                     (char *)mkva + swizzled_po, cnt);
3326                         if (ret != 0)
3327                                 break;
3328                         data_ptr += cnt;
3329                         size -= cnt;
3330                         length -= cnt;
3331                         offset += cnt;
3332                         obj_po += cnt;
3333                 }
3334                 sf_buf_free(sf);
3335                 VM_OBJECT_LOCK(vm_obj);
3336                 if (rw == UIO_WRITE)
3337                         vm_page_dirty(m);
3338                 vm_page_reference(m);
3339                 vm_page_busy_wait(m, FALSE, "i915gem");
3340                 vm_page_unwire(m, 1);
3341                 vm_page_wakeup(m);
3342                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3343
3344                 if (ret != 0)
3345                         break;
3346         }
3347         vm_object_pip_wakeup(vm_obj);
3348         VM_OBJECT_UNLOCK(vm_obj);
3349
3350         return (ret);
3351 }
3352
3353 static int
3354 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
3355     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
3356 {
3357         vm_offset_t mkva;
3358         int ret;
3359
3360         /*
3361          * Pass the unaligned physical address and size to pmap_mapdev_attr()
3362          * so it can properly calculate whether an extra page needs to be
3363          * mapped or not to cover the requested range.  The function will
3364          * add the page offset into the returned mkva for us.
3365          */
3366         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
3367             offset, size, PAT_WRITE_COMBINING);
3368         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
3369         pmap_unmapdev(mkva, size);
3370         return (ret);
3371 }
3372
3373 static int
3374 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
3375     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
3376 {
3377         struct drm_i915_gem_object *obj;
3378         vm_page_t *ma;
3379         vm_offset_t start, end;
3380         int npages, ret;
3381
3382         if (size == 0)
3383                 return (0);
3384         start = trunc_page(data_ptr);
3385         end = round_page(data_ptr + size);
3386         npages = howmany(end - start, PAGE_SIZE);
3387         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
3388             M_ZERO);
3389         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
3390             (vm_offset_t)data_ptr, size,
3391             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
3392         if (npages == -1) {
3393                 ret = -EFAULT;
3394                 goto free_ma;
3395         }
3396
3397         ret = i915_mutex_lock_interruptible(dev);
3398         if (ret != 0)
3399                 goto unlocked;
3400
3401         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
3402         if (&obj->base == NULL) {
3403                 ret = -ENOENT;
3404                 goto unlock;
3405         }
3406         if (offset > obj->base.size || size > obj->base.size - offset) {
3407                 ret = -EINVAL;
3408                 goto out;
3409         }
3410
3411         if (rw == UIO_READ) {
3412                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
3413                     offset, size);
3414                 if (ret != 0)
3415                         goto out;
3416                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3417                     UIO_READ, file);
3418         } else {
3419                 if (obj->phys_obj) {
3420                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
3421                             size, file);
3422                 } else if (obj->gtt_space &&
3423                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3424                         ret = i915_gem_object_pin(obj, 0, true);
3425                         if (ret != 0)
3426                                 goto out;
3427                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
3428                         if (ret != 0)
3429                                 goto out_unpin;
3430                         ret = i915_gem_object_put_fence(obj);
3431                         if (ret != 0)
3432                                 goto out_unpin;
3433                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
3434                             offset, file);
3435 out_unpin:
3436                         i915_gem_object_unpin(obj);
3437                 } else {
3438                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
3439                         if (ret != 0)
3440                                 goto out;
3441                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3442                             UIO_WRITE, file);
3443                 }
3444         }
3445 out:
3446         drm_gem_object_unreference(&obj->base);
3447 unlock:
3448         DRM_UNLOCK(dev);
3449 unlocked:
3450         vm_page_unhold_pages(ma, npages);
3451 free_ma:
3452         drm_free(ma, DRM_I915_GEM);
3453         return (ret);
3454 }
3455
3456 static int
3457 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
3458     vm_ooffset_t foff, struct ucred *cred, u_short *color)
3459 {
3460
3461         *color = 0; /* XXXKIB */
3462         return (0);
3463 }
3464
3465 int i915_intr_pf;
3466
3467 static int
3468 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
3469     vm_page_t *mres)
3470 {
3471         struct drm_gem_object *gem_obj;
3472         struct drm_i915_gem_object *obj;
3473         struct drm_device *dev;
3474         drm_i915_private_t *dev_priv;
3475         vm_page_t m, oldm;
3476         int cause, ret;
3477         bool write;
3478
3479         gem_obj = vm_obj->handle;
3480         obj = to_intel_bo(gem_obj);
3481         dev = obj->base.dev;
3482         dev_priv = dev->dev_private;
3483 #if 0
3484         write = (prot & VM_PROT_WRITE) != 0;
3485 #else
3486         write = true;
3487 #endif
3488         vm_object_pip_add(vm_obj, 1);
3489
3490         /*
3491          * Remove the placeholder page inserted by vm_fault() from the
3492          * object before dropping the object lock. If
3493          * i915_gem_release_mmap() is active in parallel on this gem
3494          * object, then it owns the drm device sx and might find the
3495          * placeholder already. Then, since the page is busy,
3496          * i915_gem_release_mmap() sleeps waiting for the busy state
3497          * of the page cleared. We will be not able to acquire drm
3498          * device lock until i915_gem_release_mmap() is able to make a
3499          * progress.
3500          */
3501         if (*mres != NULL) {
3502                 oldm = *mres;
3503                 vm_page_remove(oldm);
3504                 *mres = NULL;
3505         } else
3506                 oldm = NULL;
3507 retry:
3508         VM_OBJECT_UNLOCK(vm_obj);
3509 unlocked_vmobj:
3510         cause = ret = 0;
3511         m = NULL;
3512
3513         if (i915_intr_pf) {
3514                 ret = i915_mutex_lock_interruptible(dev);
3515                 if (ret != 0) {
3516                         cause = 10;
3517                         goto out;
3518                 }
3519         } else
3520                 DRM_LOCK(dev);
3521
3522         /*
3523          * Since the object lock was dropped, other thread might have
3524          * faulted on the same GTT address and instantiated the
3525          * mapping for the page.  Recheck.
3526          */
3527         VM_OBJECT_LOCK(vm_obj);
3528         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
3529         if (m != NULL) {
3530                 if ((m->flags & PG_BUSY) != 0) {
3531                         DRM_UNLOCK(dev);
3532 #if 0 /* XXX */
3533                         vm_page_sleep(m, "915pee");
3534 #endif
3535                         goto retry;
3536                 }
3537                 goto have_page;
3538         } else
3539                 VM_OBJECT_UNLOCK(vm_obj);
3540
3541         /* Now bind it into the GTT if needed */
3542         if (!obj->map_and_fenceable) {
3543                 ret = i915_gem_object_unbind(obj);
3544                 if (ret != 0) {
3545                         cause = 20;
3546                         goto unlock;
3547                 }
3548         }
3549         if (!obj->gtt_space) {
3550                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
3551                 if (ret != 0) {
3552                         cause = 30;
3553                         goto unlock;
3554                 }
3555
3556                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
3557                 if (ret != 0) {
3558                         cause = 40;
3559                         goto unlock;
3560                 }
3561         }
3562
3563         if (obj->tiling_mode == I915_TILING_NONE)
3564                 ret = i915_gem_object_put_fence(obj);
3565         else
3566                 ret = i915_gem_object_get_fence(obj, NULL);
3567         if (ret != 0) {
3568                 cause = 50;
3569                 goto unlock;
3570         }
3571
3572         if (i915_gem_object_is_inactive(obj))
3573                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3574
3575         obj->fault_mappable = true;
3576         VM_OBJECT_LOCK(vm_obj);
3577         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
3578             offset);
3579         if (m == NULL) {
3580                 cause = 60;
3581                 ret = -EFAULT;
3582                 goto unlock;
3583         }
3584         KASSERT((m->flags & PG_FICTITIOUS) != 0,
3585             ("not fictitious %p", m));
3586         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
3587
3588         if ((m->flags & PG_BUSY) != 0) {
3589                 DRM_UNLOCK(dev);
3590 #if 0 /* XXX */
3591                 vm_page_sleep(m, "915pbs");
3592 #endif
3593                 goto retry;
3594         }
3595         m->valid = VM_PAGE_BITS_ALL;
3596         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
3597 have_page:
3598         *mres = m;
3599         vm_page_busy_try(m, false);
3600
3601         DRM_UNLOCK(dev);
3602         if (oldm != NULL) {
3603                 vm_page_free(oldm);
3604         }
3605         vm_object_pip_wakeup(vm_obj);
3606         return (VM_PAGER_OK);
3607
3608 unlock:
3609         DRM_UNLOCK(dev);
3610 out:
3611         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
3612         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
3613                 goto unlocked_vmobj;
3614         }
3615         VM_OBJECT_LOCK(vm_obj);
3616         vm_object_pip_wakeup(vm_obj);
3617         return (VM_PAGER_ERROR);
3618 }
3619
3620 static void
3621 i915_gem_pager_dtor(void *handle)
3622 {
3623         struct drm_gem_object *obj;
3624         struct drm_device *dev;
3625
3626         obj = handle;
3627         dev = obj->dev;
3628
3629         DRM_LOCK(dev);
3630         drm_gem_free_mmap_offset(obj);
3631         i915_gem_release_mmap(to_intel_bo(obj));
3632         drm_gem_object_unreference(obj);
3633         DRM_UNLOCK(dev);
3634 }
3635
3636 struct cdev_pager_ops i915_gem_pager_ops = {
3637         .cdev_pg_fault  = i915_gem_pager_fault,
3638         .cdev_pg_ctor   = i915_gem_pager_ctor,
3639         .cdev_pg_dtor   = i915_gem_pager_dtor
3640 };
3641
3642 static int
3643 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3644     uint64_t offset, uint64_t size)
3645 {
3646         uint32_t old_read_domains;
3647         int i, ret;
3648
3649         if (offset == 0 && size == obj->base.size)
3650                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
3651
3652         ret = i915_gem_object_flush_gpu_write_domain(obj);
3653         if (ret != 0)
3654                 return (ret);
3655         ret = i915_gem_object_wait_rendering(obj);
3656         if (ret != 0)
3657                 return (ret);
3658
3659         i915_gem_object_flush_gtt_write_domain(obj);
3660
3661         if (obj->page_cpu_valid == NULL &&
3662             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3663                 return (0);
3664
3665         if (obj->page_cpu_valid == NULL) {
3666                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
3667                     DRM_I915_GEM, M_WAITOK | M_ZERO);
3668         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3669                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3670
3671         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3672              i++) {
3673                 if (obj->page_cpu_valid[i])
3674                         continue;
3675                 drm_clflush_pages(obj->pages + i, 1);
3676                 obj->page_cpu_valid[i] = 1;
3677         }
3678
3679         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
3680             ("In gpu write domain"));
3681
3682         old_read_domains = obj->base.read_domains;
3683         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3684
3685         return (0);
3686 }
3687
3688 #define GEM_PARANOID_CHECK_GTT 0
3689 #if GEM_PARANOID_CHECK_GTT
3690 static void
3691 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
3692     int page_count)
3693 {
3694         struct drm_i915_private *dev_priv;
3695         vm_paddr_t pa;
3696         unsigned long start, end;
3697         u_int i;
3698         int j;
3699
3700         dev_priv = dev->dev_private;
3701         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
3702         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
3703         for (i = start; i < end; i++) {
3704                 pa = intel_gtt_read_pte_paddr(i);
3705                 for (j = 0; j < page_count; j++) {
3706                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
3707                                 panic("Page %p in GTT pte index %d pte %x",
3708                                     ma[i], i, intel_gtt_read_pte(i));
3709                         }
3710                 }
3711         }
3712 }
3713 #endif
3714
3715 static void
3716 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
3717     uint32_t flush_domains)
3718 {
3719         struct drm_i915_gem_object *obj, *next;
3720         uint32_t old_write_domain;
3721
3722         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
3723             gpu_write_list) {
3724                 if (obj->base.write_domain & flush_domains) {
3725                         old_write_domain = obj->base.write_domain;
3726                         obj->base.write_domain = 0;
3727                         list_del_init(&obj->gpu_write_list);
3728                         i915_gem_object_move_to_active(obj, ring,
3729                             i915_gem_next_request_seqno(ring));
3730                 }
3731         }
3732 }
3733
3734 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3735
3736 static vm_page_t
3737 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3738 {
3739         vm_page_t m;
3740         int rv;
3741
3742         VM_OBJECT_LOCK_ASSERT_OWNED(object);
3743         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
3744         if (m->valid != VM_PAGE_BITS_ALL) {
3745                 if (vm_pager_has_page(object, pindex)) {
3746                         rv = vm_pager_get_page(object, &m, 1);
3747                         m = vm_page_lookup(object, pindex);
3748                         if (m == NULL)
3749                                 return (NULL);
3750                         if (rv != VM_PAGER_OK) {
3751                                 vm_page_free(m);
3752                                 return (NULL);
3753                         }
3754                 } else {
3755                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
3756                         m->valid = VM_PAGE_BITS_ALL;
3757                         m->dirty = 0;
3758                 }
3759         }
3760         vm_page_wire(m);
3761         vm_page_wakeup(m);
3762         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3763         return (m);
3764 }
3765
3766 int
3767 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
3768     uint32_t flush_domains)
3769 {
3770         int ret;
3771
3772         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
3773                 return 0;
3774
3775         ret = ring->flush(ring, invalidate_domains, flush_domains);
3776         if (ret)
3777                 return ret;
3778
3779         if (flush_domains & I915_GEM_GPU_DOMAINS)
3780                 i915_gem_process_flushing_list(ring, flush_domains);
3781         return 0;
3782 }
3783
3784 u32
3785 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
3786 {
3787         if (ring->outstanding_lazy_request == 0)
3788                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
3789
3790         return ring->outstanding_lazy_request;
3791 }
3792
3793 static void
3794 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3795 {
3796         drm_i915_private_t *dev_priv = dev->dev_private;
3797         uint32_t fence_reg = reg - dev_priv->fence_regs;
3798
3799         switch (INTEL_INFO(dev)->gen) {
3800         case 7:
3801         case 6:
3802                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3803                 break;
3804         case 5:
3805         case 4:
3806                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3807                 break;
3808         case 3:
3809                 if (fence_reg >= 8)
3810                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3811                 else
3812         case 2:
3813                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3814
3815                 I915_WRITE(fence_reg, 0);
3816                 break;
3817         }
3818
3819         list_del_init(&reg->lru_list);
3820         reg->obj = NULL;
3821         reg->setup_seqno = 0;
3822         reg->pin_count = 0;
3823 }
3824
3825 static int
3826 i915_gpu_is_active(struct drm_device *dev)
3827 {
3828         drm_i915_private_t *dev_priv;
3829
3830         dev_priv = dev->dev_private;
3831         return (!list_empty(&dev_priv->mm.flushing_list) ||
3832             !list_empty(&dev_priv->mm.active_list));
3833 }
3834
3835 static void
3836 i915_gem_lowmem(void *arg)
3837 {
3838         struct drm_device *dev;
3839         struct drm_i915_private *dev_priv;
3840         struct drm_i915_gem_object *obj, *next;
3841         int cnt, cnt_fail, cnt_total;
3842
3843         dev = arg;
3844         dev_priv = dev->dev_private;
3845
3846         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3847                 return;
3848
3849 rescan:
3850         /* first scan for clean buffers */
3851         i915_gem_retire_requests(dev);
3852
3853         cnt_total = cnt_fail = cnt = 0;
3854
3855         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3856             mm_list) {
3857                 if (i915_gem_object_is_purgeable(obj)) {
3858                         if (i915_gem_object_unbind(obj) != 0)
3859                                 cnt_total++;
3860                 } else
3861                         cnt_total++;
3862         }
3863
3864         /* second pass, evict/count anything still on the inactive list */
3865         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3866             mm_list) {
3867                 if (i915_gem_object_unbind(obj) == 0)
3868                         cnt++;
3869                 else
3870                         cnt_fail++;
3871         }
3872
3873         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3874                 /*
3875                  * We are desperate for pages, so as a last resort, wait
3876                  * for the GPU to finish and discard whatever we can.
3877                  * This has a dramatic impact to reduce the number of
3878                  * OOM-killer events whilst running the GPU aggressively.
3879                  */
3880                 if (i915_gpu_idle(dev) == 0)
3881                         goto rescan;
3882         }
3883         DRM_UNLOCK(dev);
3884 }
3885
3886 void
3887 i915_gem_unload(struct drm_device *dev)
3888 {
3889         struct drm_i915_private *dev_priv;
3890
3891         dev_priv = dev->dev_private;
3892         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3893 }