sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64 #include <linux/completion.h>
  65
  66 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  67 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  68 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  69     unsigned alignment, bool map_and_fenceable);
  70
  71 static int i915_gem_phys_pwrite(struct drm_device *dev,
  72     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  73     uint64_t size, struct drm_file *file_priv);
  74
  75 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  76     int tiling_mode);
  77 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  78     uint32_t size, int tiling_mode);
  79 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  80     int flags);
  81 static int i915_gem_object_set_cpu_read_domain_range(
  82     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  83 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  84 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  85 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  86 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  87 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  88 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  89 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  90     uint32_t flush_domains);
  91 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  92     struct drm_i915_fence_reg *reg);
  93 static void i915_gem_reset_fences(struct drm_device *dev);
  94 static void i915_gem_lowmem(void *arg);
  95
  96 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
  97     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file);
  98
  99 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
 100 long i915_gem_wired_pages_cnt;
 101
 102 /* some bookkeeping */
 103 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 104                                   size_t size)
 105 {
 106
 107         dev_priv->mm.object_count++;
 108         dev_priv->mm.object_memory += size;
 109 }
 110
 111 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 112                                      size_t size)
 113 {
 114
 115         dev_priv->mm.object_count--;
 116         dev_priv->mm.object_memory -= size;
 117 }
 118
 119 static int
 120 i915_gem_wait_for_error(struct drm_device *dev)
 121 {
 122         struct drm_i915_private *dev_priv = dev->dev_private;
 123         struct completion *x = &dev_priv->error_completion;
 124         int ret;
 125
 126         if (!atomic_read(&dev_priv->mm.wedged))
 127                 return 0;
 128
 129         /*
 130          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 131          * userspace. If it takes that long something really bad is going on and
 132          * we should simply try to bail out and fail as gracefully as possible.
 133          */
 134         ret = wait_for_completion_interruptible_timeout(x, 10*hz);
 135         if (ret == 0) {
 136                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 137                 return -EIO;
 138         } else if (ret < 0) {
 139                 return ret;
 140         }
 141
 142         if (atomic_read(&dev_priv->mm.wedged)) {
 143                 /* GPU is hung, bump the completion count to account for
 144                  * the token we just consumed so that we never hit zero and
 145                  * end up waiting upon a subsequent completion event that
 146                  * will never happen.
 147                  */
 148                 spin_lock(&x->wait.lock);
 149                 x->done++;
 150                 spin_unlock(&x->wait.lock);
 151         }
 152         return 0;
 153 }
 154
 155 int i915_mutex_lock_interruptible(struct drm_device *dev)
 156 {
 157         int ret;
 158
 159         ret = i915_gem_wait_for_error(dev);
 160         if (ret != 0)
 161                 return (ret);
 162
 163         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 164         if (ret)
 165                 return -EINTR;
 166
 167 #if 0
 168         WARN_ON(i915_verify_lists(dev));
 169 #endif
 170         return 0;
 171 }
 172
 173 static inline bool
 174 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 175 {
 176         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
 177 }
 178
 179 int
 180 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 181     struct drm_file *file)
 182 {
 183         struct drm_i915_gem_init *args;
 184         drm_i915_private_t *dev_priv;
 185
 186         dev_priv = dev->dev_private;
 187         args = data;
 188
 189         if (args->gtt_start >= args->gtt_end ||
 190             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 191                 return (-EINVAL);
 192
 193         /*
 194          * XXXKIB. The second-time initialization should be guarded
 195          * against.
 196          */
 197         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 198         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 199         lockmgr(&dev->dev_lock, LK_RELEASE);
 200
 201         return 0;
 202 }
 203
 204 int
 205 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 206                             struct drm_file *file)
 207 {
 208         struct drm_i915_private *dev_priv;
 209         struct drm_i915_gem_get_aperture *args;
 210         struct drm_i915_gem_object *obj;
 211         size_t pinned;
 212
 213         dev_priv = dev->dev_private;
 214         args = data;
 215
 216         if (!(dev->driver->driver_features & DRIVER_GEM))
 217                 return (-ENODEV);
 218
 219         pinned = 0;
 220         DRM_LOCK(dev);
 221         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 222                 pinned += obj->gtt_space->size;
 223         DRM_UNLOCK(dev);
 224
 225         args->aper_size = dev_priv->mm.gtt_total;
 226         args->aper_available_size = args->aper_size - pinned;
 227
 228         return (0);
 229 }
 230
 231 static int
 232 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 233     uint32_t *handle_p)
 234 {
 235         struct drm_i915_gem_object *obj;
 236         uint32_t handle;
 237         int ret;
 238
 239         size = roundup(size, PAGE_SIZE);
 240         if (size == 0)
 241                 return (-EINVAL);
 242
 243         obj = i915_gem_alloc_object(dev, size);
 244         if (obj == NULL)
 245                 return (-ENOMEM);
 246
 247         handle = 0;
 248         ret = drm_gem_handle_create(file, &obj->base, &handle);
 249         if (ret != 0) {
 250                 drm_gem_object_release(&obj->base);
 251                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 252                 drm_free(obj, DRM_I915_GEM);
 253                 return (-ret);
 254         }
 255
 256         /* drop reference from allocate - handle holds it now */
 257         drm_gem_object_unreference(&obj->base);
 258         *handle_p = handle;
 259         return (0);
 260 }
 261
 262 int
 263 i915_gem_dumb_create(struct drm_file *file,
 264                      struct drm_device *dev,
 265                      struct drm_mode_create_dumb *args)
 266 {
 267
 268         /* have to work out size/pitch and return them */
 269         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 270         args->size = args->pitch * args->height;
 271         return (i915_gem_create(file, dev, args->size, &args->handle));
 272 }
 273
 274 int i915_gem_dumb_destroy(struct drm_file *file,
 275                           struct drm_device *dev,
 276                           uint32_t handle)
 277 {
 278
 279         return (drm_gem_handle_delete(file, handle));
 280 }
 281
 282 /**
 283  * Creates a new mm object and returns a handle to it.
 284  */
 285 int
 286 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 287                       struct drm_file *file)
 288 {
 289         struct drm_i915_gem_create *args = data;
 290
 291         return (i915_gem_create(file, dev, args->size, &args->handle));
 292 }
 293
 294 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 295 {
 296         drm_i915_private_t *dev_priv;
 297
 298         dev_priv = obj->base.dev->dev_private;
 299         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 300             obj->tiling_mode != I915_TILING_NONE);
 301 }
 302
 303 /**
 304  * Reads data from the object referenced by handle.
 305  *
 306  * On error, the contents of *data are undefined.
 307  */
 308 int
 309 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 310                      struct drm_file *file)
 311 {
 312         struct drm_i915_gem_pread *args;
 313
 314         args = data;
 315         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 316             args->offset, UIO_READ, file));
 317 }
 318
 319 /**
 320  * Writes data to the object referenced by handle.
 321  *
 322  * On error, the contents of the buffer that were to be modified are undefined.
 323  */
 324 int
 325 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 326                       struct drm_file *file)
 327 {
 328         struct drm_i915_gem_pwrite *args;
 329
 330         args = data;
 331         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 332             args->offset, UIO_WRITE, file));
 333 }
 334
 335 int
 336 i915_gem_check_wedge(struct drm_i915_private *dev_priv,
 337                      bool interruptible)
 338 {
 339         if (atomic_read(&dev_priv->mm.wedged)) {
 340                 struct completion *x = &dev_priv->error_completion;
 341                 bool recovery_complete;
 342
 343                 /* Give the error handler a chance to run. */
 344                 spin_lock(&x->wait.lock);
 345                 recovery_complete = x->done > 0;
 346                 spin_unlock(&x->wait.lock);
 347
 348                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 349                  * -EIO unconditionally for these. */
 350                 if (!interruptible)
 351                         return -EIO;
 352
 353                 /* Recovery complete, but still wedged means reset failure. */
 354                 if (recovery_complete)
 355                         return -EIO;
 356
 357                 return -EAGAIN;
 358         }
 359
 360         return 0;
 361 }
 362
 363 /**
 364  * __wait_seqno - wait until execution of seqno has finished
 365  * @ring: the ring expected to report seqno
 366  * @seqno: duh!
 367  * @interruptible: do an interruptible wait (normally yes)
 368  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 369  *
 370  * Returns 0 if the seqno was found within the alloted time. Else returns the
 371  * errno with remaining time filled in timeout argument.
 372  */
 373 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
 374                         bool interruptible, struct timespec *timeout)
 375 {
 376         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 377         int ret = 0;
 378
 379         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
 380                 return 0;
 381
 382         if (WARN_ON(!ring->irq_get(ring)))
 383                 return -ENODEV;
 384
 385         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
 386
 387                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
 388                 if (ring->irq_get(ring)) {
 389                         int flags = dev_priv->mm.interruptible ? PCATCH : 0;
 390                         while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)
 391                             && !atomic_read(&dev_priv->mm.wedged) &&
 392                             ret == 0) {
 393                                 ret = -lksleep(ring, &ring->irq_lock, flags,
 394                                     "915gwr", 1*hz);
 395                         }
 396                         ring->irq_put(ring);
 397                         lockmgr(&ring->irq_lock, LK_RELEASE);
 398                 } else {
 399                         lockmgr(&ring->irq_lock, LK_RELEASE);
 400                         if (_intel_wait_for(ring->dev,
 401                             i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
 402                             atomic_read(&dev_priv->mm.wedged), 3000,
 403                             0, "i915wrq") != 0)
 404                                 ret = -EBUSY;
 405                 }
 406
 407         }
 408
 409         ring->irq_put(ring);
 410
 411         return ret;
 412 }
 413
 414 /**
 415  * Waits for a sequence number to be signaled, and cleans up the
 416  * request and object lists appropriately for that event.
 417  */
 418 int
 419 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
 420 {
 421         struct drm_device *dev = ring->dev;
 422         struct drm_i915_private *dev_priv = dev->dev_private;
 423         bool interruptible = dev_priv->mm.interruptible;
 424         struct drm_i915_gem_request *request;
 425         int ret;
 426
 427         DRM_LOCK_ASSERT(dev);
 428         BUG_ON(seqno == 0);
 429
 430         ret = i915_gem_check_wedge(dev_priv, interruptible);
 431         if (ret)
 432                 return ret;
 433
 434         if (seqno == ring->outstanding_lazy_request) {
 435                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
 436                     M_WAITOK | M_ZERO);
 437                 if (request == NULL)
 438                         return (-ENOMEM);
 439
 440                 ret = i915_add_request(ring, NULL, request);
 441                 if (ret != 0) {
 442                         drm_free(request, DRM_I915_GEM);
 443                         return (ret);
 444                 }
 445
 446                 seqno = request->seqno;
 447         }
 448
 449         return __wait_seqno(ring, seqno, interruptible, NULL);
 450 }
 451
 452 /**
 453  * Ensures that all rendering to the object has completed and the object is
 454  * safe to unbind from the GTT or access from the CPU.
 455  */
 456 int
 457 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 458 {
 459         u32 seqno;
 460         int ret;
 461
 462
 463         seqno = obj->last_rendering_seqno;
 464         if (seqno == 0)
 465                 return 0;
 466
 467         if (obj->active) {
 468                 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
 469                 if (ret != 0)
 470                         return (ret);
 471         }
 472
 473         /* Manually manage the write flush as we may have not yet
 474          * retired the buffer.
 475          */
 476         if (obj->last_rendering_seqno &&
 477             i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
 478                 obj->last_rendering_seqno = 0;
 479                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
 480         }
 481
 482         return 0;
 483 }
 484
 485 /**
 486  * Called when user space prepares to use an object with the CPU, either
 487  * through the mmap ioctl's mapping or a GTT mapping.
 488  */
 489 int
 490 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 491                           struct drm_file *file)
 492 {
 493         struct drm_i915_gem_set_domain *args;
 494         struct drm_i915_gem_object *obj;
 495         uint32_t read_domains;
 496         uint32_t write_domain;
 497         int ret;
 498
 499         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 500                 return (-ENODEV);
 501
 502         args = data;
 503         read_domains = args->read_domains;
 504         write_domain = args->write_domain;
 505
 506         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
 507             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
 508             (write_domain != 0 && read_domains != write_domain))
 509                 return (-EINVAL);
 510
 511         ret = i915_mutex_lock_interruptible(dev);
 512         if (ret != 0)
 513                 return (ret);
 514
 515         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 516         if (&obj->base == NULL) {
 517                 ret = -ENOENT;
 518                 goto unlock;
 519         }
 520
 521         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
 522                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 523                 if (ret == -EINVAL)
 524                         ret = 0;
 525         } else
 526                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 527
 528         drm_gem_object_unreference(&obj->base);
 529 unlock:
 530         DRM_UNLOCK(dev);
 531         return (ret);
 532 }
 533
 534 /**
 535  * Called when user space has done writes to this buffer
 536  */
 537 int
 538 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 539                          struct drm_file *file)
 540 {
 541         struct drm_i915_gem_sw_finish *args;
 542         struct drm_i915_gem_object *obj;
 543         int ret;
 544
 545         args = data;
 546         ret = 0;
 547         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 548                 return (ENODEV);
 549         ret = i915_mutex_lock_interruptible(dev);
 550         if (ret != 0)
 551                 return (ret);
 552         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 553         if (&obj->base == NULL) {
 554                 ret = -ENOENT;
 555                 goto unlock;
 556         }
 557         if (obj->pin_count != 0)
 558                 i915_gem_object_flush_cpu_write_domain(obj);
 559         drm_gem_object_unreference(&obj->base);
 560 unlock:
 561         DRM_UNLOCK(dev);
 562         return (ret);
 563 }
 564
 565 /**
 566  * Maps the contents of an object, returning the address it is mapped
 567  * into.
 568  *
 569  * While the mapping holds a reference on the contents of the object, it doesn't
 570  * imply a ref on the object itself.
 571  */
 572 int
 573 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 574                     struct drm_file *file)
 575 {
 576         struct drm_i915_gem_mmap *args;
 577         struct drm_gem_object *obj;
 578         struct proc *p;
 579         vm_map_t map;
 580         vm_offset_t addr;
 581         vm_size_t size;
 582         int error, rv;
 583
 584         args = data;
 585
 586         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 587                 return (-ENODEV);
 588
 589         obj = drm_gem_object_lookup(dev, file, args->handle);
 590         if (obj == NULL)
 591                 return (-ENOENT);
 592         error = 0;
 593         if (args->size == 0)
 594                 goto out;
 595         p = curproc;
 596         map = &p->p_vmspace->vm_map;
 597         size = round_page(args->size);
 598         PROC_LOCK(p);
 599         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
 600                 PROC_UNLOCK(p);
 601                 error = ENOMEM;
 602                 goto out;
 603         }
 604         PROC_UNLOCK(p);
 605
 606         addr = 0;
 607         vm_object_hold(obj->vm_obj);
 608         vm_object_reference_locked(obj->vm_obj);
 609         vm_object_drop(obj->vm_obj);
 610         DRM_UNLOCK(dev);
 611         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
 612             PAGE_SIZE, /* align */
 613             TRUE, /* fitit */
 614             VM_MAPTYPE_NORMAL, /* maptype */
 615             VM_PROT_READ | VM_PROT_WRITE, /* prot */
 616             VM_PROT_READ | VM_PROT_WRITE, /* max */
 617             MAP_SHARED /* cow */);
 618         if (rv != KERN_SUCCESS) {
 619                 vm_object_deallocate(obj->vm_obj);
 620                 error = -vm_mmap_to_errno(rv);
 621         } else {
 622                 args->addr_ptr = (uint64_t)addr;
 623         }
 624         DRM_LOCK(dev);
 625 out:
 626         drm_gem_object_unreference(obj);
 627         return (error);
 628 }
 629
 630 /**
 631  * i915_gem_release_mmap - remove physical page mappings
 632  * @obj: obj in question
 633  *
 634  * Preserve the reservation of the mmapping with the DRM core code, but
 635  * relinquish ownership of the pages back to the system.
 636  *
 637  * It is vital that we remove the page mapping if we have mapped a tiled
 638  * object through the GTT and then lose the fence register due to
 639  * resource pressure. Similarly if the object has been moved out of the
 640  * aperture, than pages mapped into userspace must be revoked. Removing the
 641  * mapping will then trigger a page fault on the next user access, allowing
 642  * fixup by i915_gem_fault().
 643  */
 644 void
 645 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 646 {
 647         vm_object_t devobj;
 648         vm_page_t m;
 649         int i, page_count;
 650
 651         if (!obj->fault_mappable)
 652                 return;
 653
 654         devobj = cdev_pager_lookup(obj);
 655         if (devobj != NULL) {
 656                 page_count = OFF_TO_IDX(obj->base.size);
 657
 658                 VM_OBJECT_LOCK(devobj);
 659                 for (i = 0; i < page_count; i++) {
 660                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
 661                         if (m == NULL)
 662                                 continue;
 663                         cdev_pager_free_page(devobj, m);
 664                 }
 665                 VM_OBJECT_UNLOCK(devobj);
 666                 vm_object_deallocate(devobj);
 667         }
 668
 669         obj->fault_mappable = false;
 670 }
 671
 672 static uint32_t
 673 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 674 {
 675         uint32_t gtt_size;
 676
 677         if (INTEL_INFO(dev)->gen >= 4 ||
 678             tiling_mode == I915_TILING_NONE)
 679                 return (size);
 680
 681         /* Previous chips need a power-of-two fence region when tiling */
 682         if (INTEL_INFO(dev)->gen == 3)
 683                 gtt_size = 1024*1024;
 684         else
 685                 gtt_size = 512*1024;
 686
 687         while (gtt_size < size)
 688                 gtt_size <<= 1;
 689
 690         return (gtt_size);
 691 }
 692
 693 /**
 694  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 695  * @obj: object to check
 696  *
 697  * Return the required GTT alignment for an object, taking into account
 698  * potential fence register mapping.
 699  */
 700 static uint32_t
 701 i915_gem_get_gtt_alignment(struct drm_device *dev,
 702                            uint32_t size,
 703                            int tiling_mode)
 704 {
 705
 706         /*
 707          * Minimum alignment is 4k (GTT page size), but might be greater
 708          * if a fence register is needed for the object.
 709          */
 710         if (INTEL_INFO(dev)->gen >= 4 ||
 711             tiling_mode == I915_TILING_NONE)
 712                 return (4096);
 713
 714         /*
 715          * Previous chips need to be aligned to the size of the smallest
 716          * fence register that can contain the object.
 717          */
 718         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 719 }
 720
 721 /**
 722  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
 723  *                                       unfenced object
 724  * @dev: the device
 725  * @size: size of the object
 726  * @tiling_mode: tiling mode of the object
 727  *
 728  * Return the required GTT alignment for an object, only taking into account
 729  * unfenced tiled surface requirements.
 730  */
 731 uint32_t
 732 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 733                                     uint32_t size,
 734                                     int tiling_mode)
 735 {
 736
 737         if (tiling_mode == I915_TILING_NONE)
 738                 return (4096);
 739
 740         /*
 741          * Minimum alignment is 4k (GTT page size) for sane hw.
 742          */
 743         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
 744                 return (4096);
 745
 746         /*
 747          * Previous hardware however needs to be aligned to a power-of-two
 748          * tile height. The simplest method for determining this is to reuse
 749          * the power-of-tile object size.
 750          */
 751         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 752 }
 753
 754 int
 755 i915_gem_mmap_gtt(struct drm_file *file,
 756                   struct drm_device *dev,
 757                   uint32_t handle,
 758                   uint64_t *offset)
 759 {
 760         struct drm_i915_private *dev_priv;
 761         struct drm_i915_gem_object *obj;
 762         int ret;
 763
 764         if (!(dev->driver->driver_features & DRIVER_GEM))
 765                 return (-ENODEV);
 766
 767         dev_priv = dev->dev_private;
 768
 769         ret = i915_mutex_lock_interruptible(dev);
 770         if (ret != 0)
 771                 return (ret);
 772
 773         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 774         if (&obj->base == NULL) {
 775                 ret = -ENOENT;
 776                 goto unlock;
 777         }
 778
 779         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
 780                 ret = -E2BIG;
 781                 goto out;
 782         }
 783
 784         if (obj->madv != I915_MADV_WILLNEED) {
 785                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
 786                 ret = -EINVAL;
 787                 goto out;
 788         }
 789
 790         ret = drm_gem_create_mmap_offset(&obj->base);
 791         if (ret != 0)
 792                 goto out;
 793
 794         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
 795             DRM_GEM_MAPPING_KEY;
 796 out:
 797         drm_gem_object_unreference(&obj->base);
 798 unlock:
 799         DRM_UNLOCK(dev);
 800         return (ret);
 801 }
 802
 803 /**
 804  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 805  * @dev: DRM device
 806  * @data: GTT mapping ioctl data
 807  * @file: GEM object info
 808  *
 809  * Simply returns the fake offset to userspace so it can mmap it.
 810  * The mmap call will end up in drm_gem_mmap(), which will set things
 811  * up so we can get faults in the handler above.
 812  *
 813  * The fault handler will take care of binding the object into the GTT
 814  * (since it may have been evicted to make room for something), allocating
 815  * a fence register, and mapping the appropriate aperture address into
 816  * userspace.
 817  */
 818 int
 819 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 820                         struct drm_file *file)
 821 {
 822         struct drm_i915_private *dev_priv;
 823         struct drm_i915_gem_mmap_gtt *args;
 824
 825         dev_priv = dev->dev_private;
 826         args = data;
 827
 828         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
 829 }
 830
 831 /* Immediately discard the backing storage */
 832 static void
 833 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 834 {
 835         vm_object_t vm_obj;
 836
 837         vm_obj = obj->base.vm_obj;
 838         VM_OBJECT_LOCK(vm_obj);
 839         vm_object_page_remove(vm_obj, 0, 0, false);
 840         VM_OBJECT_UNLOCK(vm_obj);
 841         obj->madv = __I915_MADV_PURGED;
 842 }
 843
 844 static inline int
 845 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 846 {
 847         return obj->madv == I915_MADV_DONTNEED;
 848 }
 849
 850 static inline void vm_page_reference(vm_page_t m)
 851 {
 852         vm_page_flag_set(m, PG_REFERENCED);
 853 }
 854
 855 static void
 856 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 857 {
 858         vm_page_t m;
 859         int page_count, i;
 860
 861         BUG_ON(obj->madv == __I915_MADV_PURGED);
 862
 863         if (obj->tiling_mode != I915_TILING_NONE)
 864                 i915_gem_object_save_bit_17_swizzle(obj);
 865         if (obj->madv == I915_MADV_DONTNEED)
 866                 obj->dirty = 0;
 867         page_count = obj->base.size / PAGE_SIZE;
 868         VM_OBJECT_LOCK(obj->base.vm_obj);
 869 #if GEM_PARANOID_CHECK_GTT
 870         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
 871 #endif
 872         for (i = 0; i < page_count; i++) {
 873                 m = obj->pages[i];
 874                 if (obj->dirty)
 875                         vm_page_dirty(m);
 876                 if (obj->madv == I915_MADV_WILLNEED)
 877                         vm_page_reference(m);
 878                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
 879                 vm_page_unwire(obj->pages[i], 1);
 880                 vm_page_wakeup(obj->pages[i]);
 881                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 882         }
 883         VM_OBJECT_UNLOCK(obj->base.vm_obj);
 884         obj->dirty = 0;
 885         drm_free(obj->pages, DRM_I915_GEM);
 886         obj->pages = NULL;
 887 }
 888
 889 static int
 890 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 891     int flags)
 892 {
 893         struct drm_device *dev;
 894         vm_object_t vm_obj;
 895         vm_page_t m;
 896         int page_count, i, j;
 897
 898         dev = obj->base.dev;
 899         KASSERT(obj->pages == NULL, ("Obj already has pages"));
 900         page_count = obj->base.size / PAGE_SIZE;
 901         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 902             M_WAITOK);
 903         vm_obj = obj->base.vm_obj;
 904         VM_OBJECT_LOCK(vm_obj);
 905         for (i = 0; i < page_count; i++) {
 906                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
 907                         goto failed;
 908         }
 909         VM_OBJECT_UNLOCK(vm_obj);
 910         if (i915_gem_object_needs_bit17_swizzle(obj))
 911                 i915_gem_object_do_bit_17_swizzle(obj);
 912         return (0);
 913
 914 failed:
 915         for (j = 0; j < i; j++) {
 916                 m = obj->pages[j];
 917                 vm_page_busy_wait(m, FALSE, "i915gem");
 918                 vm_page_unwire(m, 0);
 919                 vm_page_wakeup(m);
 920                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 921         }
 922         VM_OBJECT_UNLOCK(vm_obj);
 923         drm_free(obj->pages, DRM_I915_GEM);
 924         obj->pages = NULL;
 925         return (-EIO);
 926 }
 927
 928 void
 929 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 930     struct intel_ring_buffer *ring, uint32_t seqno)
 931 {
 932         struct drm_device *dev = obj->base.dev;
 933         struct drm_i915_private *dev_priv = dev->dev_private;
 934         struct drm_i915_fence_reg *reg;
 935
 936         obj->ring = ring;
 937         KASSERT(ring != NULL, ("NULL ring"));
 938
 939         /* Add a reference if we're newly entering the active list. */
 940         if (!obj->active) {
 941                 drm_gem_object_reference(&obj->base);
 942                 obj->active = 1;
 943         }
 944
 945         /* Move from whatever list we were on to the tail of execution. */
 946         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 947         list_move_tail(&obj->ring_list, &ring->active_list);
 948
 949         obj->last_rendering_seqno = seqno;
 950         if (obj->fenced_gpu_access) {
 951                 obj->last_fenced_seqno = seqno;
 952                 obj->last_fenced_ring = ring;
 953
 954                 /* Bump MRU to take account of the delayed flush */
 955                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
 956                         reg = &dev_priv->fence_regs[obj->fence_reg];
 957                         list_move_tail(&reg->lru_list,
 958                                        &dev_priv->mm.fence_list);
 959                 }
 960         }
 961 }
 962
 963 static void
 964 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 965 {
 966         list_del_init(&obj->ring_list);
 967         obj->last_rendering_seqno = 0;
 968         obj->last_fenced_seqno = 0;
 969 }
 970
 971 static void
 972 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 973 {
 974         struct drm_device *dev = obj->base.dev;
 975         struct drm_i915_private *dev_priv = dev->dev_private;
 976
 977         if (obj->pin_count != 0)
 978                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
 979         else
 980                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 981
 982         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
 983         KASSERT(obj->active, ("Object not active"));
 984         obj->ring = NULL;
 985         obj->last_fenced_ring = NULL;
 986
 987         i915_gem_object_move_off_active(obj);
 988         obj->fenced_gpu_access = false;
 989
 990         obj->active = 0;
 991         obj->pending_gpu_write = false;
 992         drm_gem_object_unreference(&obj->base);
 993
 994 #if 1
 995         KIB_NOTYET();
 996 #else
 997         WARN_ON(i915_verify_lists(dev));
 998 #endif
 999 }
1000
1001 static u32
1002 i915_gem_get_seqno(struct drm_device *dev)
1003 {
1004         drm_i915_private_t *dev_priv = dev->dev_private;
1005         u32 seqno = dev_priv->next_seqno;
1006
1007         /* reserve 0 for non-seqno */
1008         if (++dev_priv->next_seqno == 0)
1009                 dev_priv->next_seqno = 1;
1010
1011         return seqno;
1012 }
1013
1014 int
1015 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
1016      struct drm_i915_gem_request *request)
1017 {
1018         drm_i915_private_t *dev_priv;
1019         struct drm_i915_file_private *file_priv;
1020         uint32_t seqno;
1021         u32 request_ring_position;
1022         int was_empty;
1023         int ret;
1024
1025         KASSERT(request != NULL, ("NULL request in add"));
1026         DRM_LOCK_ASSERT(ring->dev);
1027         dev_priv = ring->dev->dev_private;
1028
1029         seqno = i915_gem_next_request_seqno(ring);
1030         request_ring_position = intel_ring_get_tail(ring);
1031
1032         ret = ring->add_request(ring, &seqno);
1033         if (ret != 0)
1034             return ret;
1035
1036         request->seqno = seqno;
1037         request->ring = ring;
1038         request->tail = request_ring_position;
1039         request->emitted_jiffies = ticks;
1040         was_empty = list_empty(&ring->request_list);
1041         list_add_tail(&request->list, &ring->request_list);
1042
1043         if (file != NULL) {
1044                 file_priv = file->driver_priv;
1045
1046                 spin_lock(&file_priv->mm.lock);
1047                 request->file_priv = file_priv;
1048                 list_add_tail(&request->client_list,
1049                     &file_priv->mm.request_list);
1050                 spin_unlock(&file_priv->mm.lock);
1051         }
1052
1053         ring->outstanding_lazy_request = 0;
1054
1055         if (!dev_priv->mm.suspended) {
1056                 if (i915_enable_hangcheck) {
1057                         mod_timer(&dev_priv->hangcheck_timer,
1058                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
1059                 }
1060                 if (was_empty) {
1061                         queue_delayed_work(dev_priv->wq,
1062                                            &dev_priv->mm.retire_work,
1063                                            round_jiffies_up_relative(hz));
1064                         intel_mark_busy(dev_priv->dev);
1065                 }
1066         }
1067         return (0);
1068 }
1069
1070 static inline void
1071 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1072 {
1073         struct drm_i915_file_private *file_priv = request->file_priv;
1074
1075         if (!file_priv)
1076                 return;
1077
1078         DRM_LOCK_ASSERT(request->ring->dev);
1079
1080         spin_lock(&file_priv->mm.lock);
1081         if (request->file_priv != NULL) {
1082                 list_del(&request->client_list);
1083                 request->file_priv = NULL;
1084         }
1085         spin_unlock(&file_priv->mm.lock);
1086 }
1087
1088 static void
1089 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1090     struct intel_ring_buffer *ring)
1091 {
1092
1093         if (ring->dev != NULL)
1094                 DRM_LOCK_ASSERT(ring->dev);
1095
1096         while (!list_empty(&ring->request_list)) {
1097                 struct drm_i915_gem_request *request;
1098
1099                 request = list_first_entry(&ring->request_list,
1100                     struct drm_i915_gem_request, list);
1101
1102                 list_del(&request->list);
1103                 i915_gem_request_remove_from_client(request);
1104                 drm_free(request, DRM_I915_GEM);
1105         }
1106
1107         while (!list_empty(&ring->active_list)) {
1108                 struct drm_i915_gem_object *obj;
1109
1110                 obj = list_first_entry(&ring->active_list,
1111                     struct drm_i915_gem_object, ring_list);
1112
1113                 obj->base.write_domain = 0;
1114                 list_del_init(&obj->gpu_write_list);
1115                 i915_gem_object_move_to_inactive(obj);
1116         }
1117 }
1118
1119 static void
1120 i915_gem_reset_fences(struct drm_device *dev)
1121 {
1122         struct drm_i915_private *dev_priv = dev->dev_private;
1123         int i;
1124
1125         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1126                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1127                 struct drm_i915_gem_object *obj = reg->obj;
1128
1129                 if (!obj)
1130                         continue;
1131
1132                 if (obj->tiling_mode)
1133                         i915_gem_release_mmap(obj);
1134
1135                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1136                 reg->obj->fenced_gpu_access = false;
1137                 reg->obj->last_fenced_seqno = 0;
1138                 reg->obj->last_fenced_ring = NULL;
1139                 i915_gem_clear_fence_reg(dev, reg);
1140         }
1141 }
1142
1143 void i915_gem_reset(struct drm_device *dev)
1144 {
1145         struct drm_i915_private *dev_priv = dev->dev_private;
1146         struct drm_i915_gem_object *obj;
1147         int i;
1148
1149         for (i = 0; i < I915_NUM_RINGS; i++)
1150                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1151
1152         /* Remove anything from the flushing lists. The GPU cache is likely
1153          * to be lost on reset along with the data, so simply move the
1154          * lost bo to the inactive list.
1155          */
1156         while (!list_empty(&dev_priv->mm.flushing_list)) {
1157                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1158                                       struct drm_i915_gem_object,
1159                                       mm_list);
1160
1161                 obj->base.write_domain = 0;
1162                 list_del_init(&obj->gpu_write_list);
1163                 i915_gem_object_move_to_inactive(obj);
1164         }
1165
1166         /* Move everything out of the GPU domains to ensure we do any
1167          * necessary invalidation upon reuse.
1168          */
1169         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
1170                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1171         }
1172
1173         /* The fence registers are invalidated so clear them out */
1174         i915_gem_reset_fences(dev);
1175 }
1176
1177 static void
1178 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1179 {
1180         struct drm_device *dev = obj->base.dev;
1181         drm_i915_private_t *dev_priv = dev->dev_private;
1182
1183         KASSERT(obj->active, ("Object not active"));
1184         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1185
1186         i915_gem_object_move_off_active(obj);
1187 }
1188
1189 /**
1190  * This function clears the request list as sequence numbers are passed.
1191  */
1192 void
1193 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1194 {
1195         uint32_t seqno;
1196
1197         if (list_empty(&ring->request_list))
1198                 return;
1199
1200         seqno = ring->get_seqno(ring, true);
1201
1202         while (!list_empty(&ring->request_list)) {
1203                 struct drm_i915_gem_request *request;
1204
1205                 request = list_first_entry(&ring->request_list,
1206                                            struct drm_i915_gem_request,
1207                                            list);
1208
1209                 if (!i915_seqno_passed(seqno, request->seqno))
1210                         break;
1211
1212                 /* We know the GPU must have read the request to have
1213                  * sent us the seqno + interrupt, so use the position
1214                  * of tail of the request to update the last known position
1215                  * of the GPU head.
1216                  */
1217                 ring->last_retired_head = request->tail;
1218
1219                 list_del(&request->list);
1220                 i915_gem_request_remove_from_client(request);
1221                 drm_free(request, DRM_I915_GEM);
1222         }
1223
1224         /* Move any buffers on the active list that are no longer referenced
1225          * by the ringbuffer to the flushing/inactive lists as appropriate.
1226          */
1227         while (!list_empty(&ring->active_list)) {
1228                 struct drm_i915_gem_object *obj;
1229
1230                 obj = list_first_entry(&ring->active_list,
1231                                       struct drm_i915_gem_object,
1232                                       ring_list);
1233
1234                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1235                         break;
1236
1237                 if (obj->base.write_domain != 0)
1238                         i915_gem_object_move_to_flushing(obj);
1239                 else
1240                         i915_gem_object_move_to_inactive(obj);
1241         }
1242
1243         if (unlikely(ring->trace_irq_seqno &&
1244                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1245                 ring->irq_put(ring);
1246                 ring->trace_irq_seqno = 0;
1247         }
1248
1249 }
1250
1251 static void
1252 i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
1253
1254 void
1255 i915_gem_retire_requests(struct drm_device *dev)
1256 {
1257         drm_i915_private_t *dev_priv = dev->dev_private;
1258         struct drm_i915_gem_object *obj, *next;
1259         int i;
1260
1261         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1262                 list_for_each_entry_safe(obj, next,
1263                     &dev_priv->mm.deferred_free_list, mm_list)
1264                         i915_gem_free_object_tail(obj);
1265         }
1266
1267         for (i = 0; i < I915_NUM_RINGS; i++)
1268                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1269 }
1270
1271 static void
1272 i915_gem_retire_work_handler(struct work_struct *work)
1273 {
1274         drm_i915_private_t *dev_priv;
1275         struct drm_device *dev;
1276         struct intel_ring_buffer *ring;
1277         bool idle;
1278         int i;
1279
1280         dev_priv = container_of(work, drm_i915_private_t,
1281                                 mm.retire_work.work);
1282         dev = dev_priv->dev;
1283
1284         /* Come back later if the device is busy... */
1285         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
1286                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1287                                    round_jiffies_up_relative(hz));
1288                 return;
1289         }
1290
1291         i915_gem_retire_requests(dev);
1292
1293         /* Send a periodic flush down the ring so we don't hold onto GEM
1294          * objects indefinitely.
1295          */
1296         idle = true;
1297         for_each_ring(ring, dev_priv, i) {
1298                 if (ring->gpu_caches_dirty)
1299                         i915_add_request(ring, NULL, NULL);
1300
1301                 idle &= list_empty(&ring->request_list);
1302         }
1303
1304         if (!dev_priv->mm.suspended && !idle)
1305                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1306                                    round_jiffies_up_relative(hz));
1307         if (idle)
1308                 intel_mark_idle(dev);
1309
1310         DRM_UNLOCK(dev);
1311 }
1312
1313 /**
1314  * i915_gem_object_sync - sync an object to a ring.
1315  *
1316  * @obj: object which may be in use on another ring.
1317  * @to: ring we wish to use the object on. May be NULL.
1318  *
1319  * This code is meant to abstract object synchronization with the GPU.
1320  * Calling with NULL implies synchronizing the object with the CPU
1321  * rather than a particular GPU ring.
1322  *
1323  * Returns 0 if successful, else propagates up the lower layer error.
1324  */
1325 int
1326 i915_gem_object_sync(struct drm_i915_gem_object *obj,
1327                      struct intel_ring_buffer *to)
1328 {
1329         struct intel_ring_buffer *from = obj->ring;
1330         u32 seqno;
1331         int ret, idx;
1332
1333         if (from == NULL || to == from)
1334                 return 0;
1335
1336         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
1337                 return i915_gem_object_wait_rendering(obj);
1338
1339         idx = intel_ring_sync_index(from, to);
1340
1341         seqno = obj->last_rendering_seqno;
1342         if (seqno <= from->sync_seqno[idx])
1343                 return 0;
1344
1345         if (seqno == from->outstanding_lazy_request) {
1346                 struct drm_i915_gem_request *request;
1347
1348                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
1349                     M_WAITOK | M_ZERO);
1350                 if (request == NULL)
1351                         return -ENOMEM;
1352
1353                 ret = i915_add_request(from, NULL, request);
1354                 if (ret) {
1355                         kfree(request, DRM_I915_GEM);
1356                         return ret;
1357                 }
1358
1359                 seqno = request->seqno;
1360         }
1361
1362         from->sync_seqno[idx] = seqno;
1363
1364         return to->sync_to(to, from, seqno - 1);
1365 }
1366
1367 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1368 {
1369         u32 old_write_domain, old_read_domains;
1370
1371         /* Act a barrier for all accesses through the GTT */
1372         cpu_mfence();
1373
1374         /* Force a pagefault for domain tracking on next user access */
1375         i915_gem_release_mmap(obj);
1376
1377         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1378                 return;
1379
1380         old_read_domains = obj->base.read_domains;
1381         old_write_domain = obj->base.write_domain;
1382
1383         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1384         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1385
1386 }
1387
1388 int
1389 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1390 {
1391         drm_i915_private_t *dev_priv;
1392         int ret;
1393
1394         dev_priv = obj->base.dev->dev_private;
1395         ret = 0;
1396         if (obj->gtt_space == NULL)
1397                 return (0);
1398         if (obj->pin_count != 0) {
1399                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1400                 return (-EINVAL);
1401         }
1402
1403         ret = i915_gem_object_finish_gpu(obj);
1404         if (ret == -ERESTART || ret == -EINTR)
1405                 return (ret);
1406
1407         i915_gem_object_finish_gtt(obj);
1408
1409         if (ret == 0)
1410                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1411         if (ret == -ERESTART || ret == -EINTR)
1412                 return (ret);
1413         if (ret != 0) {
1414                 i915_gem_clflush_object(obj);
1415                 obj->base.read_domains = obj->base.write_domain =
1416                     I915_GEM_DOMAIN_CPU;
1417         }
1418
1419         ret = i915_gem_object_put_fence(obj);
1420         if (ret == -ERESTART)
1421                 return (ret);
1422
1423         i915_gem_gtt_unbind_object(obj);
1424         if (obj->has_aliasing_ppgtt_mapping) {
1425                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
1426                 obj->has_aliasing_ppgtt_mapping = 0;
1427         }
1428         i915_gem_object_put_pages_gtt(obj);
1429
1430         list_del_init(&obj->gtt_list);
1431         list_del_init(&obj->mm_list);
1432         obj->map_and_fenceable = true;
1433
1434         drm_mm_put_block(obj->gtt_space);
1435         obj->gtt_space = NULL;
1436         obj->gtt_offset = 0;
1437
1438         if (i915_gem_object_is_purgeable(obj))
1439                 i915_gem_object_truncate(obj);
1440
1441         return (ret);
1442 }
1443
1444 int i915_gpu_idle(struct drm_device *dev)
1445 {
1446         drm_i915_private_t *dev_priv = dev->dev_private;
1447         struct intel_ring_buffer *ring;
1448         int ret, i;
1449
1450         /* Flush everything onto the inactive list. */
1451         for_each_ring(ring, dev_priv, i) {
1452                 ret = intel_ring_idle(ring);
1453                 if (ret)
1454                         return ret;
1455         }
1456
1457         return 0;
1458 }
1459
1460 static int
1461 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
1462     struct intel_ring_buffer *pipelined)
1463 {
1464         struct drm_device *dev = obj->base.dev;
1465         drm_i915_private_t *dev_priv = dev->dev_private;
1466         u32 size = obj->gtt_space->size;
1467         int regnum = obj->fence_reg;
1468         uint64_t val;
1469
1470         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1471                          0xfffff000) << 32;
1472         val |= obj->gtt_offset & 0xfffff000;
1473         val |= (uint64_t)((obj->stride / 128) - 1) <<
1474                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
1475
1476         if (obj->tiling_mode == I915_TILING_Y)
1477                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1478         val |= I965_FENCE_REG_VALID;
1479
1480         if (pipelined) {
1481                 int ret = intel_ring_begin(pipelined, 6);
1482                 if (ret)
1483                         return ret;
1484
1485                 intel_ring_emit(pipelined, MI_NOOP);
1486                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1487                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
1488                 intel_ring_emit(pipelined, (u32)val);
1489                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
1490                 intel_ring_emit(pipelined, (u32)(val >> 32));
1491                 intel_ring_advance(pipelined);
1492         } else
1493                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
1494
1495         return 0;
1496 }
1497
1498 static int
1499 i965_write_fence_reg(struct drm_i915_gem_object *obj,
1500     struct intel_ring_buffer *pipelined)
1501 {
1502         struct drm_device *dev = obj->base.dev;
1503         drm_i915_private_t *dev_priv = dev->dev_private;
1504         u32 size = obj->gtt_space->size;
1505         int regnum = obj->fence_reg;
1506         uint64_t val;
1507
1508         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1509                     0xfffff000) << 32;
1510         val |= obj->gtt_offset & 0xfffff000;
1511         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1512         if (obj->tiling_mode == I915_TILING_Y)
1513                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1514         val |= I965_FENCE_REG_VALID;
1515
1516         if (pipelined) {
1517                 int ret = intel_ring_begin(pipelined, 6);
1518                 if (ret)
1519                         return ret;
1520
1521                 intel_ring_emit(pipelined, MI_NOOP);
1522                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1523                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
1524                 intel_ring_emit(pipelined, (u32)val);
1525                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
1526                 intel_ring_emit(pipelined, (u32)(val >> 32));
1527                 intel_ring_advance(pipelined);
1528         } else
1529                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
1530
1531         return 0;
1532 }
1533
1534 static int
1535 i915_write_fence_reg(struct drm_i915_gem_object *obj,
1536     struct intel_ring_buffer *pipelined)
1537 {
1538         struct drm_device *dev = obj->base.dev;
1539         drm_i915_private_t *dev_priv = dev->dev_private;
1540         u32 size = obj->gtt_space->size;
1541         u32 fence_reg, val, pitch_val;
1542         int tile_width;
1543
1544         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
1545             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1546                 kprintf(
1547 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1548                  obj->gtt_offset, obj->map_and_fenceable, size);
1549                 return -EINVAL;
1550         }
1551
1552         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1553                 tile_width = 128;
1554         else
1555                 tile_width = 512;
1556
1557         /* Note: pitch better be a power of two tile widths */
1558         pitch_val = obj->stride / tile_width;
1559         pitch_val = ffs(pitch_val) - 1;
1560
1561         val = obj->gtt_offset;
1562         if (obj->tiling_mode == I915_TILING_Y)
1563                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1564         val |= I915_FENCE_SIZE_BITS(size);
1565         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1566         val |= I830_FENCE_REG_VALID;
1567
1568         fence_reg = obj->fence_reg;
1569         if (fence_reg < 8)
1570                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
1571         else
1572                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
1573
1574         if (pipelined) {
1575                 int ret = intel_ring_begin(pipelined, 4);
1576                 if (ret)
1577                         return ret;
1578
1579                 intel_ring_emit(pipelined, MI_NOOP);
1580                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1581                 intel_ring_emit(pipelined, fence_reg);
1582                 intel_ring_emit(pipelined, val);
1583                 intel_ring_advance(pipelined);
1584         } else
1585                 I915_WRITE(fence_reg, val);
1586
1587         return 0;
1588 }
1589
1590 static int
1591 i830_write_fence_reg(struct drm_i915_gem_object *obj,
1592     struct intel_ring_buffer *pipelined)
1593 {
1594         struct drm_device *dev = obj->base.dev;
1595         drm_i915_private_t *dev_priv = dev->dev_private;
1596         u32 size = obj->gtt_space->size;
1597         int regnum = obj->fence_reg;
1598         uint32_t val;
1599         uint32_t pitch_val;
1600
1601         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
1602             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1603                 kprintf(
1604 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1605                     obj->gtt_offset, size);
1606                 return -EINVAL;
1607         }
1608
1609         pitch_val = obj->stride / 128;
1610         pitch_val = ffs(pitch_val) - 1;
1611
1612         val = obj->gtt_offset;
1613         if (obj->tiling_mode == I915_TILING_Y)
1614                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1615         val |= I830_FENCE_SIZE_BITS(size);
1616         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1617         val |= I830_FENCE_REG_VALID;
1618
1619         if (pipelined) {
1620                 int ret = intel_ring_begin(pipelined, 4);
1621                 if (ret)
1622                         return ret;
1623
1624                 intel_ring_emit(pipelined, MI_NOOP);
1625                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1626                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
1627                 intel_ring_emit(pipelined, val);
1628                 intel_ring_advance(pipelined);
1629         } else
1630                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
1631
1632         return 0;
1633 }
1634
1635 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
1636 {
1637         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
1638 }
1639
1640 static int
1641 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
1642     struct intel_ring_buffer *pipelined)
1643 {
1644         int ret;
1645
1646         if (obj->fenced_gpu_access) {
1647                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1648                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
1649                             obj->base.write_domain);
1650                         if (ret)
1651                                 return ret;
1652                 }
1653
1654                 obj->fenced_gpu_access = false;
1655         }
1656
1657         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
1658                 if (!ring_passed_seqno(obj->last_fenced_ring,
1659                                        obj->last_fenced_seqno)) {
1660                         ret = i915_wait_seqno(obj->last_fenced_ring,
1661                                                 obj->last_fenced_seqno);
1662                         if (ret)
1663                                 return ret;
1664                 }
1665
1666                 obj->last_fenced_seqno = 0;
1667                 obj->last_fenced_ring = NULL;
1668         }
1669
1670         /* Ensure that all CPU reads are completed before installing a fence
1671          * and all writes before removing the fence.
1672          */
1673         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
1674                 cpu_mfence();
1675
1676         return 0;
1677 }
1678
1679 int
1680 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
1681 {
1682         int ret;
1683
1684         if (obj->tiling_mode)
1685                 i915_gem_release_mmap(obj);
1686
1687         ret = i915_gem_object_flush_fence(obj, NULL);
1688         if (ret)
1689                 return ret;
1690
1691         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1692                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1693
1694                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
1695                         kprintf("%s: pin_count %d\n", __func__,
1696                             dev_priv->fence_regs[obj->fence_reg].pin_count);
1697                 i915_gem_clear_fence_reg(obj->base.dev,
1698                                          &dev_priv->fence_regs[obj->fence_reg]);
1699
1700                 obj->fence_reg = I915_FENCE_REG_NONE;
1701         }
1702
1703         return 0;
1704 }
1705
1706 static struct drm_i915_fence_reg *
1707 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
1708 {
1709         struct drm_i915_private *dev_priv = dev->dev_private;
1710         struct drm_i915_fence_reg *reg, *first, *avail;
1711         int i;
1712
1713         /* First try to find a free reg */
1714         avail = NULL;
1715         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
1716                 reg = &dev_priv->fence_regs[i];
1717                 if (!reg->obj)
1718                         return reg;
1719
1720                 if (!reg->pin_count)
1721                         avail = reg;
1722         }
1723
1724         if (avail == NULL)
1725                 return NULL;
1726
1727         /* None available, try to steal one or wait for a user to finish */
1728         avail = first = NULL;
1729         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1730                 if (reg->pin_count)
1731                         continue;
1732
1733                 if (first == NULL)
1734                         first = reg;
1735
1736                 if (!pipelined ||
1737                     !reg->obj->last_fenced_ring ||
1738                     reg->obj->last_fenced_ring == pipelined) {
1739                         avail = reg;
1740                         break;
1741                 }
1742         }
1743
1744         if (avail == NULL)
1745                 avail = first;
1746
1747         return avail;
1748 }
1749
1750 int
1751 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1752     struct intel_ring_buffer *pipelined)
1753 {
1754         struct drm_device *dev = obj->base.dev;
1755         struct drm_i915_private *dev_priv = dev->dev_private;
1756         struct drm_i915_fence_reg *reg;
1757         int ret;
1758
1759         pipelined = NULL;
1760         ret = 0;
1761
1762         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1763                 reg = &dev_priv->fence_regs[obj->fence_reg];
1764                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1765
1766                 if (obj->tiling_changed) {
1767                         ret = i915_gem_object_flush_fence(obj, pipelined);
1768                         if (ret)
1769                                 return ret;
1770
1771                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1772                                 pipelined = NULL;
1773
1774                         if (pipelined) {
1775                                 reg->setup_seqno =
1776                                         i915_gem_next_request_seqno(pipelined);
1777                                 obj->last_fenced_seqno = reg->setup_seqno;
1778                                 obj->last_fenced_ring = pipelined;
1779                         }
1780
1781                         goto update;
1782                 }
1783
1784                 if (!pipelined) {
1785                         if (reg->setup_seqno) {
1786                                 if (!ring_passed_seqno(obj->last_fenced_ring,
1787                                     reg->setup_seqno)) {
1788                                         ret = i915_wait_seqno(
1789                                             obj->last_fenced_ring,
1790                                             reg->setup_seqno);
1791                                         if (ret)
1792                                                 return ret;
1793                                 }
1794
1795                                 reg->setup_seqno = 0;
1796                         }
1797                 } else if (obj->last_fenced_ring &&
1798                            obj->last_fenced_ring != pipelined) {
1799                         ret = i915_gem_object_flush_fence(obj, pipelined);
1800                         if (ret)
1801                                 return ret;
1802                 }
1803
1804                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1805                         pipelined = NULL;
1806                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
1807
1808                 if (obj->tiling_changed) {
1809                         if (pipelined) {
1810                                 reg->setup_seqno =
1811                                         i915_gem_next_request_seqno(pipelined);
1812                                 obj->last_fenced_seqno = reg->setup_seqno;
1813                                 obj->last_fenced_ring = pipelined;
1814                         }
1815                         goto update;
1816                 }
1817
1818                 return 0;
1819         }
1820
1821         reg = i915_find_fence_reg(dev, pipelined);
1822         if (reg == NULL)
1823                 return -EDEADLK;
1824
1825         ret = i915_gem_object_flush_fence(obj, pipelined);
1826         if (ret)
1827                 return ret;
1828
1829         if (reg->obj) {
1830                 struct drm_i915_gem_object *old = reg->obj;
1831
1832                 drm_gem_object_reference(&old->base);
1833
1834                 if (old->tiling_mode)
1835                         i915_gem_release_mmap(old);
1836
1837                 ret = i915_gem_object_flush_fence(old, pipelined);
1838                 if (ret) {
1839                         drm_gem_object_unreference(&old->base);
1840                         return ret;
1841                 }
1842
1843                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
1844                         pipelined = NULL;
1845
1846                 old->fence_reg = I915_FENCE_REG_NONE;
1847                 old->last_fenced_ring = pipelined;
1848                 old->last_fenced_seqno =
1849                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1850
1851                 drm_gem_object_unreference(&old->base);
1852         } else if (obj->last_fenced_seqno == 0)
1853                 pipelined = NULL;
1854
1855         reg->obj = obj;
1856         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1857         obj->fence_reg = reg - dev_priv->fence_regs;
1858         obj->last_fenced_ring = pipelined;
1859
1860         reg->setup_seqno =
1861                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1862         obj->last_fenced_seqno = reg->setup_seqno;
1863
1864 update:
1865         obj->tiling_changed = false;
1866         switch (INTEL_INFO(dev)->gen) {
1867         case 7:
1868         case 6:
1869                 ret = sandybridge_write_fence_reg(obj, pipelined);
1870                 break;
1871         case 5:
1872         case 4:
1873                 ret = i965_write_fence_reg(obj, pipelined);
1874                 break;
1875         case 3:
1876                 ret = i915_write_fence_reg(obj, pipelined);
1877                 break;
1878         case 2:
1879                 ret = i830_write_fence_reg(obj, pipelined);
1880                 break;
1881         }
1882
1883         return ret;
1884 }
1885
1886 static int
1887 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1888     unsigned alignment, bool map_and_fenceable)
1889 {
1890         struct drm_device *dev;
1891         struct drm_i915_private *dev_priv;
1892         struct drm_mm_node *free_space;
1893         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1894         bool mappable, fenceable;
1895         int ret;
1896
1897         dev = obj->base.dev;
1898         dev_priv = dev->dev_private;
1899
1900         if (obj->madv != I915_MADV_WILLNEED) {
1901                 DRM_ERROR("Attempting to bind a purgeable object\n");
1902                 return (-EINVAL);
1903         }
1904
1905         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1906             obj->tiling_mode);
1907         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1908             obj->tiling_mode);
1909         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1910             obj->base.size, obj->tiling_mode);
1911         if (alignment == 0)
1912                 alignment = map_and_fenceable ? fence_alignment :
1913                     unfenced_alignment;
1914         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1915                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1916                 return (-EINVAL);
1917         }
1918
1919         size = map_and_fenceable ? fence_size : obj->base.size;
1920
1921         /* If the object is bigger than the entire aperture, reject it early
1922          * before evicting everything in a vain attempt to find space.
1923          */
1924         if (obj->base.size > (map_and_fenceable ?
1925             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1926                 DRM_ERROR(
1927 "Attempting to bind an object larger than the aperture\n");
1928                 return (-E2BIG);
1929         }
1930
1931  search_free:
1932         if (map_and_fenceable)
1933                 free_space = drm_mm_search_free_in_range(
1934                     &dev_priv->mm.gtt_space, size, alignment, 0,
1935                     dev_priv->mm.gtt_mappable_end, 0);
1936         else
1937                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1938                     size, alignment, 0);
1939         if (free_space != NULL) {
1940                 int color = 0;
1941                 if (map_and_fenceable)
1942                         obj->gtt_space = drm_mm_get_block_range_generic(
1943                             free_space, size, alignment, color, 0,
1944                             dev_priv->mm.gtt_mappable_end, 1);
1945                 else
1946                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1947                             size, alignment, color, 1);
1948         }
1949         if (obj->gtt_space == NULL) {
1950                 ret = i915_gem_evict_something(dev, size, alignment,
1951                     map_and_fenceable);
1952                 if (ret != 0)
1953                         return (ret);
1954                 goto search_free;
1955         }
1956
1957         /*
1958          * NOTE: i915_gem_object_get_pages_gtt() cannot
1959          *       return ENOMEM, since we used VM_ALLOC_RETRY.
1960          */
1961         ret = i915_gem_object_get_pages_gtt(obj, 0);
1962         if (ret != 0) {
1963                 drm_mm_put_block(obj->gtt_space);
1964                 obj->gtt_space = NULL;
1965                 return (ret);
1966         }
1967
1968         i915_gem_gtt_bind_object(obj, obj->cache_level);
1969         if (ret != 0) {
1970                 i915_gem_object_put_pages_gtt(obj);
1971                 drm_mm_put_block(obj->gtt_space);
1972                 obj->gtt_space = NULL;
1973                 if (i915_gem_evict_everything(dev, false))
1974                         return (ret);
1975                 goto search_free;
1976         }
1977
1978         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1979         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1980
1981         obj->gtt_offset = obj->gtt_space->start;
1982
1983         fenceable =
1984                 obj->gtt_space->size == fence_size &&
1985                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1986
1987         mappable =
1988                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1989         obj->map_and_fenceable = mappable && fenceable;
1990
1991         return (0);
1992 }
1993
1994 void
1995 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1996 {
1997
1998         /* If we don't have a page list set up, then we're not pinned
1999          * to GPU, and we can ignore the cache flush because it'll happen
2000          * again at bind time.
2001          */
2002         if (obj->pages == NULL)
2003                 return;
2004
2005         /* If the GPU is snooping the contents of the CPU cache,
2006          * we do not need to manually clear the CPU cache lines.  However,
2007          * the caches are only snooped when the render cache is
2008          * flushed/invalidated.  As we always have to emit invalidations
2009          * and flushes when moving into and out of the RENDER domain, correct
2010          * snooping behaviour occurs naturally as the result of our domain
2011          * tracking.
2012          */
2013         if (obj->cache_level != I915_CACHE_NONE)
2014                 return;
2015
2016         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2017 }
2018
2019 /** Flushes the GTT write domain for the object if it's dirty. */
2020 static void
2021 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2022 {
2023         uint32_t old_write_domain;
2024
2025         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2026                 return;
2027
2028         /* No actual flushing is required for the GTT write domain.  Writes
2029          * to it immediately go to main memory as far as we know, so there's
2030          * no chipset flush.  It also doesn't land in render cache.
2031          *
2032          * However, we do have to enforce the order so that all writes through
2033          * the GTT land before any writes to the device, such as updates to
2034          * the GATT itself.
2035          */
2036         cpu_sfence();
2037
2038         old_write_domain = obj->base.write_domain;
2039         obj->base.write_domain = 0;
2040 }
2041
2042 /** Flushes the CPU write domain for the object if it's dirty. */
2043 static void
2044 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2045 {
2046         uint32_t old_write_domain;
2047
2048         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2049                 return;
2050
2051         i915_gem_clflush_object(obj);
2052         intel_gtt_chipset_flush();
2053         old_write_domain = obj->base.write_domain;
2054         obj->base.write_domain = 0;
2055 }
2056
2057 static int
2058 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2059 {
2060
2061         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2062                 return (0);
2063         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
2064 }
2065
2066 /**
2067  * Moves a single object to the GTT read, and possibly write domain.
2068  *
2069  * This function returns when the move is complete, including waiting on
2070  * flushes to occur.
2071  */
2072 int
2073 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2074 {
2075         uint32_t old_write_domain, old_read_domains;
2076         int ret;
2077
2078         if (obj->gtt_space == NULL)
2079                 return (-EINVAL);
2080
2081         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2082                 return 0;
2083
2084         ret = i915_gem_object_flush_gpu_write_domain(obj);
2085         if (ret != 0)
2086                 return (ret);
2087
2088         if (obj->pending_gpu_write || write) {
2089                 ret = i915_gem_object_wait_rendering(obj);
2090                 if (ret != 0)
2091                         return (ret);
2092         }
2093
2094         i915_gem_object_flush_cpu_write_domain(obj);
2095
2096         old_write_domain = obj->base.write_domain;
2097         old_read_domains = obj->base.read_domains;
2098
2099         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2100             ("In GTT write domain"));
2101         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2102         if (write) {
2103                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2104                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2105                 obj->dirty = 1;
2106         }
2107
2108         return (0);
2109 }
2110
2111 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2112                                     enum i915_cache_level cache_level)
2113 {
2114         struct drm_device *dev = obj->base.dev;
2115         drm_i915_private_t *dev_priv = dev->dev_private;
2116         int ret;
2117
2118         if (obj->cache_level == cache_level)
2119                 return 0;
2120
2121         if (obj->pin_count) {
2122                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2123                 return -EBUSY;
2124         }
2125
2126         if (obj->gtt_space) {
2127                 ret = i915_gem_object_finish_gpu(obj);
2128                 if (ret != 0)
2129                         return (ret);
2130
2131                 i915_gem_object_finish_gtt(obj);
2132
2133                 /* Before SandyBridge, you could not use tiling or fence
2134                  * registers with snooped memory, so relinquish any fences
2135                  * currently pointing to our region in the aperture.
2136                  */
2137                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2138                         ret = i915_gem_object_put_fence(obj);
2139                         if (ret)
2140                                 return ret;
2141                 }
2142
2143                 if (obj->has_global_gtt_mapping)
2144                         i915_gem_gtt_bind_object(obj, cache_level);
2145                 if (obj->has_aliasing_ppgtt_mapping)
2146                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2147                                                obj, cache_level);
2148         }
2149
2150         if (cache_level == I915_CACHE_NONE) {
2151                 u32 old_read_domains, old_write_domain;
2152
2153                 /* If we're coming from LLC cached, then we haven't
2154                  * actually been tracking whether the data is in the
2155                  * CPU cache or not, since we only allow one bit set
2156                  * in obj->write_domain and have been skipping the clflushes.
2157                  * Just set it to the CPU cache for now.
2158                  */
2159                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
2160                     ("obj %p in CPU write domain", obj));
2161                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
2162                     ("obj %p in CPU read domain", obj));
2163
2164                 old_read_domains = obj->base.read_domains;
2165                 old_write_domain = obj->base.write_domain;
2166
2167                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2168                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2169
2170         }
2171
2172         obj->cache_level = cache_level;
2173         return 0;
2174 }
2175
2176 /*
2177  * Prepare buffer for display plane (scanout, cursors, etc).
2178  * Can be called from an uninterruptible phase (modesetting) and allows
2179  * any flushes to be pipelined (for pageflips).
2180  */
2181 int
2182 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2183                                      u32 alignment,
2184                                      struct intel_ring_buffer *pipelined)
2185 {
2186         u32 old_read_domains, old_write_domain;
2187         int ret;
2188
2189         ret = i915_gem_object_flush_gpu_write_domain(obj);
2190         if (ret != 0)
2191                 return (ret);
2192
2193         if (pipelined != obj->ring) {
2194                 ret = i915_gem_object_sync(obj, pipelined);
2195                 if (ret)
2196                         return ret;
2197         }
2198
2199         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2200         if (ret != 0)
2201                 return (ret);
2202
2203         ret = i915_gem_object_pin(obj, alignment, true);
2204         if (ret != 0)
2205                 return (ret);
2206
2207         i915_gem_object_flush_cpu_write_domain(obj);
2208
2209         old_write_domain = obj->base.write_domain;
2210         old_read_domains = obj->base.read_domains;
2211
2212         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2213             ("obj %p in GTT write domain", obj));
2214         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2215
2216         return (0);
2217 }
2218
2219 int
2220 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2221 {
2222         int ret;
2223
2224         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2225                 return (0);
2226
2227         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2228                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2229                 if (ret != 0)
2230                         return (ret);
2231         }
2232
2233         ret = i915_gem_object_wait_rendering(obj);
2234         if (ret != 0)
2235                 return (ret);
2236
2237         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2238
2239         return (0);
2240 }
2241
2242 /**
2243  * Moves a single object to the CPU read, and possibly write domain.
2244  *
2245  * This function returns when the move is complete, including waiting on
2246  * flushes to occur.
2247  */
2248 int
2249 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2250 {
2251         uint32_t old_write_domain, old_read_domains;
2252         int ret;
2253
2254         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2255                 return 0;
2256
2257         ret = i915_gem_object_flush_gpu_write_domain(obj);
2258         if (ret != 0)
2259                 return (ret);
2260
2261         ret = i915_gem_object_wait_rendering(obj);
2262         if (ret)
2263                 return ret;
2264
2265         i915_gem_object_flush_gtt_write_domain(obj);
2266
2267         old_write_domain = obj->base.write_domain;
2268         old_read_domains = obj->base.read_domains;
2269
2270         /* Flush the CPU cache if it's still invalid. */
2271         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2272                 i915_gem_clflush_object(obj);
2273
2274                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2275         }
2276
2277         /* It should now be out of any other write domains, and we can update
2278          * the domain values for our changes.
2279          */
2280         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2281
2282         /* If we're writing through the CPU, then the GPU read domains will
2283          * need to be invalidated at next use.
2284          */
2285         if (write) {
2286                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2287                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2288         }
2289
2290         return 0;
2291 }
2292
2293 /* Throttle our rendering by waiting until the ring has completed our requests
2294  * emitted over 20 msec ago.
2295  *
2296  * Note that if we were to use the current jiffies each time around the loop,
2297  * we wouldn't escape the function with any frames outstanding if the time to
2298  * render a frame was over 20ms.
2299  *
2300  * This should get us reasonable parallelism between CPU and GPU but also
2301  * relatively low latency when blocking on a particular request to finish.
2302  */
2303 static int
2304 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2305 {
2306         struct drm_i915_private *dev_priv = dev->dev_private;
2307         struct drm_i915_file_private *file_priv = file->driver_priv;
2308         unsigned long recent_enough = ticks - (20 * hz / 1000);
2309         struct drm_i915_gem_request *request;
2310         struct intel_ring_buffer *ring = NULL;
2311         u32 seqno = 0;
2312         int ret;
2313
2314         if (atomic_read(&dev_priv->mm.wedged))
2315                 return -EIO;
2316
2317         spin_lock(&file_priv->mm.lock);
2318         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2319                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2320                         break;
2321
2322                 ring = request->ring;
2323                 seqno = request->seqno;
2324         }
2325         spin_unlock(&file_priv->mm.lock);
2326
2327         if (seqno == 0)
2328                 return 0;
2329
2330         ret = __wait_seqno(ring, seqno, true, NULL);
2331
2332         if (ret == 0)
2333                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2334
2335         return ret;
2336 }
2337
2338 int
2339 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
2340      bool map_and_fenceable)
2341 {
2342         struct drm_device *dev;
2343         struct drm_i915_private *dev_priv;
2344         int ret;
2345
2346         dev = obj->base.dev;
2347         dev_priv = dev->dev_private;
2348
2349         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
2350             ("Max pin count"));
2351
2352         if (obj->gtt_space != NULL) {
2353                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
2354                     (map_and_fenceable && !obj->map_and_fenceable)) {
2355                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
2356                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2357                              " obj->map_and_fenceable=%d\n",
2358                              obj->gtt_offset, alignment,
2359                              map_and_fenceable,
2360                              obj->map_and_fenceable);
2361                         ret = i915_gem_object_unbind(obj);
2362                         if (ret != 0)
2363                                 return (ret);
2364                 }
2365         }
2366
2367         if (obj->gtt_space == NULL) {
2368                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
2369                     map_and_fenceable);
2370                 if (ret)
2371                         return (ret);
2372         }
2373
2374         if (obj->pin_count++ == 0 && !obj->active)
2375                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2376         obj->pin_mappable |= map_and_fenceable;
2377
2378 #if 1
2379         KIB_NOTYET();
2380 #else
2381         WARN_ON(i915_verify_lists(dev));
2382 #endif
2383         return (0);
2384 }
2385
2386 void
2387 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
2388 {
2389         struct drm_device *dev;
2390         drm_i915_private_t *dev_priv;
2391
2392         dev = obj->base.dev;
2393         dev_priv = dev->dev_private;
2394
2395 #if 1
2396         KIB_NOTYET();
2397 #else
2398         WARN_ON(i915_verify_lists(dev));
2399 #endif
2400
2401         KASSERT(obj->pin_count != 0, ("zero pin count"));
2402         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
2403
2404         if (--obj->pin_count == 0) {
2405                 if (!obj->active)
2406                         list_move_tail(&obj->mm_list,
2407                             &dev_priv->mm.inactive_list);
2408                 obj->pin_mappable = false;
2409         }
2410 #if 1
2411         KIB_NOTYET();
2412 #else
2413         WARN_ON(i915_verify_lists(dev));
2414 #endif
2415 }
2416
2417 int
2418 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2419     struct drm_file *file)
2420 {
2421         struct drm_i915_gem_pin *args;
2422         struct drm_i915_gem_object *obj;
2423         struct drm_gem_object *gobj;
2424         int ret;
2425
2426         args = data;
2427
2428         ret = i915_mutex_lock_interruptible(dev);
2429         if (ret != 0)
2430                 return ret;
2431
2432         gobj = drm_gem_object_lookup(dev, file, args->handle);
2433         if (gobj == NULL) {
2434                 ret = -ENOENT;
2435                 goto unlock;
2436         }
2437         obj = to_intel_bo(gobj);
2438
2439         if (obj->madv != I915_MADV_WILLNEED) {
2440                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2441                 ret = -EINVAL;
2442                 goto out;
2443         }
2444
2445         if (obj->pin_filp != NULL && obj->pin_filp != file) {
2446                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2447                     args->handle);
2448                 ret = -EINVAL;
2449                 goto out;
2450         }
2451
2452         obj->user_pin_count++;
2453         obj->pin_filp = file;
2454         if (obj->user_pin_count == 1) {
2455                 ret = i915_gem_object_pin(obj, args->alignment, true);
2456                 if (ret != 0)
2457                         goto out;
2458         }
2459
2460         /* XXX - flush the CPU caches for pinned objects
2461          * as the X server doesn't manage domains yet
2462          */
2463         i915_gem_object_flush_cpu_write_domain(obj);
2464         args->offset = obj->gtt_offset;
2465 out:
2466         drm_gem_object_unreference(&obj->base);
2467 unlock:
2468         DRM_UNLOCK(dev);
2469         return (ret);
2470 }
2471
2472 int
2473 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2474     struct drm_file *file)
2475 {
2476         struct drm_i915_gem_pin *args;
2477         struct drm_i915_gem_object *obj;
2478         int ret;
2479
2480         args = data;
2481         ret = i915_mutex_lock_interruptible(dev);
2482         if (ret != 0)
2483                 return (ret);
2484
2485         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2486         if (&obj->base == NULL) {
2487                 ret = -ENOENT;
2488                 goto unlock;
2489         }
2490
2491         if (obj->pin_filp != file) {
2492                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2493                     args->handle);
2494                 ret = -EINVAL;
2495                 goto out;
2496         }
2497         obj->user_pin_count--;
2498         if (obj->user_pin_count == 0) {
2499                 obj->pin_filp = NULL;
2500                 i915_gem_object_unpin(obj);
2501         }
2502
2503 out:
2504         drm_gem_object_unreference(&obj->base);
2505 unlock:
2506         DRM_UNLOCK(dev);
2507         return (ret);
2508 }
2509
2510 int
2511 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2512                     struct drm_file *file)
2513 {
2514         struct drm_i915_gem_busy *args;
2515         struct drm_i915_gem_object *obj;
2516         struct drm_i915_gem_request *request;
2517         int ret;
2518
2519         args = data;
2520
2521         ret = i915_mutex_lock_interruptible(dev);
2522         if (ret != 0)
2523                 return ret;
2524
2525         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2526         if (&obj->base == NULL) {
2527                 ret = -ENOENT;
2528                 goto unlock;
2529         }
2530
2531         args->busy = obj->active;
2532         if (args->busy) {
2533                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2534                         ret = i915_gem_flush_ring(obj->ring,
2535                             0, obj->base.write_domain);
2536                 } else if (obj->ring->outstanding_lazy_request ==
2537                     obj->last_rendering_seqno) {
2538                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
2539                             M_WAITOK | M_ZERO);
2540                         ret = i915_add_request(obj->ring, NULL, request);
2541                         if (ret != 0)
2542                                 drm_free(request, DRM_I915_GEM);
2543                 }
2544
2545                 i915_gem_retire_requests_ring(obj->ring);
2546                 args->busy = obj->active;
2547         }
2548
2549         drm_gem_object_unreference(&obj->base);
2550 unlock:
2551         DRM_UNLOCK(dev);
2552         return (ret);
2553 }
2554
2555 int
2556 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2557     struct drm_file *file_priv)
2558 {
2559
2560         return (i915_gem_ring_throttle(dev, file_priv));
2561 }
2562
2563 int
2564 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
2565                        struct drm_file *file_priv)
2566 {
2567         struct drm_i915_gem_madvise *args = data;
2568         struct drm_i915_gem_object *obj;
2569         int ret;
2570
2571         switch (args->madv) {
2572         case I915_MADV_DONTNEED:
2573         case I915_MADV_WILLNEED:
2574             break;
2575         default:
2576             return -EINVAL;
2577         }
2578
2579         ret = i915_mutex_lock_interruptible(dev);
2580         if (ret)
2581                 return ret;
2582
2583         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
2584         if (&obj->base == NULL) {
2585                 ret = -ENOENT;
2586                 goto unlock;
2587         }
2588
2589         if (obj->pin_count) {
2590                 ret = -EINVAL;
2591                 goto out;
2592         }
2593
2594         if (obj->madv != __I915_MADV_PURGED)
2595                 obj->madv = args->madv;
2596
2597         /* if the object is no longer attached, discard its backing storage */
2598         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
2599                 i915_gem_object_truncate(obj);
2600
2601         args->retained = obj->madv != __I915_MADV_PURGED;
2602
2603 out:
2604         drm_gem_object_unreference(&obj->base);
2605 unlock:
2606         DRM_UNLOCK(dev);
2607         return ret;
2608 }
2609
2610 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
2611                                                   size_t size)
2612 {
2613         struct drm_i915_private *dev_priv;
2614         struct drm_i915_gem_object *obj;
2615
2616         dev_priv = dev->dev_private;
2617
2618         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
2619
2620         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
2621                 drm_free(obj, DRM_I915_GEM);
2622                 return (NULL);
2623         }
2624
2625         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2626         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2627
2628         if (HAS_LLC(dev))
2629                 obj->cache_level = I915_CACHE_LLC;
2630         else
2631                 obj->cache_level = I915_CACHE_NONE;
2632         obj->base.driver_private = NULL;
2633         obj->fence_reg = I915_FENCE_REG_NONE;
2634         INIT_LIST_HEAD(&obj->mm_list);
2635         INIT_LIST_HEAD(&obj->gtt_list);
2636         INIT_LIST_HEAD(&obj->ring_list);
2637         INIT_LIST_HEAD(&obj->exec_list);
2638         INIT_LIST_HEAD(&obj->gpu_write_list);
2639         obj->madv = I915_MADV_WILLNEED;
2640         /* Avoid an unnecessary call to unbind on the first bind. */
2641         obj->map_and_fenceable = true;
2642
2643         i915_gem_info_add_obj(dev_priv, size);
2644
2645         return (obj);
2646 }
2647
2648 int i915_gem_init_object(struct drm_gem_object *obj)
2649 {
2650
2651         kprintf("i915_gem_init_object called\n");
2652         return (0);
2653 }
2654
2655 static void
2656 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
2657 {
2658         struct drm_device *dev;
2659         drm_i915_private_t *dev_priv;
2660         int ret;
2661
2662         dev = obj->base.dev;
2663         dev_priv = dev->dev_private;
2664
2665         ret = i915_gem_object_unbind(obj);
2666         if (ret == -ERESTART) {
2667                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
2668                 return;
2669         }
2670
2671         drm_gem_free_mmap_offset(&obj->base);
2672         drm_gem_object_release(&obj->base);
2673         i915_gem_info_remove_obj(dev_priv, obj->base.size);
2674
2675         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
2676         drm_free(obj->bit_17, DRM_I915_GEM);
2677         drm_free(obj, DRM_I915_GEM);
2678 }
2679
2680 void
2681 i915_gem_free_object(struct drm_gem_object *gem_obj)
2682 {
2683         struct drm_i915_gem_object *obj;
2684         struct drm_device *dev;
2685
2686         obj = to_intel_bo(gem_obj);
2687         dev = obj->base.dev;
2688
2689         while (obj->pin_count > 0)
2690                 i915_gem_object_unpin(obj);
2691
2692         if (obj->phys_obj != NULL)
2693                 i915_gem_detach_phys_object(dev, obj);
2694
2695         i915_gem_free_object_tail(obj);
2696 }
2697
2698 int
2699 i915_gem_do_init(struct drm_device *dev, unsigned long start,
2700     unsigned long mappable_end, unsigned long end)
2701 {
2702         drm_i915_private_t *dev_priv;
2703         unsigned long mappable;
2704         int error;
2705
2706         dev_priv = dev->dev_private;
2707         mappable = min(end, mappable_end) - start;
2708
2709         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
2710
2711         dev_priv->mm.gtt_start = start;
2712         dev_priv->mm.gtt_mappable_end = mappable_end;
2713         dev_priv->mm.gtt_end = end;
2714         dev_priv->mm.gtt_total = end - start;
2715         dev_priv->mm.mappable_gtt_total = mappable;
2716
2717         /* Take over this portion of the GTT */
2718         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
2719         device_printf(dev->dev,
2720             "taking over the fictitious range 0x%lx-0x%lx\n",
2721             dev->agp->base + start, dev->agp->base + start + mappable);
2722         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
2723             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2724         return (error);
2725 }
2726
2727 int
2728 i915_gem_idle(struct drm_device *dev)
2729 {
2730         drm_i915_private_t *dev_priv;
2731         int ret;
2732
2733         dev_priv = dev->dev_private;
2734         if (dev_priv->mm.suspended)
2735                 return (0);
2736
2737         ret = i915_gpu_idle(dev);
2738         if (ret != 0)
2739                 return (ret);
2740
2741         /* Under UMS, be paranoid and evict. */
2742         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
2743                 ret = i915_gem_evict_inactive(dev, false);
2744                 if (ret != 0)
2745                         return ret;
2746         }
2747
2748         i915_gem_reset_fences(dev);
2749
2750         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2751          * We need to replace this with a semaphore, or something.
2752          * And not confound mm.suspended!
2753          */
2754         dev_priv->mm.suspended = 1;
2755         del_timer_sync(&dev_priv->hangcheck_timer);
2756
2757         i915_kernel_lost_context(dev);
2758         i915_gem_cleanup_ringbuffer(dev);
2759
2760         /* Cancel the retire work handler, which should be idle now. */
2761         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2762
2763         return (ret);
2764 }
2765
2766 void i915_gem_l3_remap(struct drm_device *dev)
2767 {
2768         drm_i915_private_t *dev_priv = dev->dev_private;
2769         u32 misccpctl;
2770         int i;
2771
2772         if (!HAS_L3_GPU_CACHE(dev))
2773                 return;
2774
2775         if (!dev_priv->l3_parity.remap_info)
2776                 return;
2777
2778         misccpctl = I915_READ(GEN7_MISCCPCTL);
2779         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
2780         POSTING_READ(GEN7_MISCCPCTL);
2781
2782         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
2783                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
2784                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
2785                         DRM_DEBUG("0x%x was already programmed to %x\n",
2786                                   GEN7_L3LOG_BASE + i, remap);
2787                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
2788                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
2789                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
2790         }
2791
2792         /* Make sure all the writes land before disabling dop clock gating */
2793         POSTING_READ(GEN7_L3LOG_BASE);
2794
2795         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
2796 }
2797
2798 void
2799 i915_gem_init_swizzling(struct drm_device *dev)
2800 {
2801         drm_i915_private_t *dev_priv;
2802
2803         dev_priv = dev->dev_private;
2804
2805         if (INTEL_INFO(dev)->gen < 5 ||
2806             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
2807                 return;
2808
2809         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
2810                                  DISP_TILE_SURFACE_SWIZZLING);
2811
2812         if (IS_GEN5(dev))
2813                 return;
2814
2815         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
2816         if (IS_GEN6(dev))
2817                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
2818         else
2819                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
2820 }
2821
2822 static bool
2823 intel_enable_blt(struct drm_device *dev)
2824 {
2825         int revision;
2826
2827         if (!HAS_BLT(dev))
2828                 return false;
2829
2830         /* The blitter was dysfunctional on early prototypes */
2831         revision = pci_read_config(dev->dev, PCIR_REVID, 1);
2832         if (IS_GEN6(dev) && revision < 8) {
2833                 DRM_INFO("BLT not supported on this pre-production hardware;"
2834                          " graphics performance will be degraded.\n");
2835                 return false;
2836         }
2837
2838         return true;
2839 }
2840
2841 int
2842 i915_gem_init_hw(struct drm_device *dev)
2843 {
2844         drm_i915_private_t *dev_priv = dev->dev_private;
2845         int ret;
2846
2847         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
2848                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
2849
2850         i915_gem_l3_remap(dev);
2851
2852         i915_gem_init_swizzling(dev);
2853
2854         ret = intel_init_render_ring_buffer(dev);
2855         if (ret)
2856                 return ret;
2857
2858         if (HAS_BSD(dev)) {
2859                 ret = intel_init_bsd_ring_buffer(dev);
2860                 if (ret)
2861                         goto cleanup_render_ring;
2862         }
2863
2864         if (intel_enable_blt(dev)) {
2865                 ret = intel_init_blt_ring_buffer(dev);
2866                 if (ret)
2867                         goto cleanup_bsd_ring;
2868         }
2869
2870         dev_priv->next_seqno = 1;
2871
2872         /*
2873          * XXX: There was some w/a described somewhere suggesting loading
2874          * contexts before PPGTT.
2875          */
2876 #if 0   /* XXX: HW context support */
2877         i915_gem_context_init(dev);
2878 #endif
2879         i915_gem_init_ppgtt(dev);
2880
2881         return 0;
2882
2883 cleanup_bsd_ring:
2884         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
2885 cleanup_render_ring:
2886         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
2887         return ret;
2888 }
2889
2890 void
2891 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2892 {
2893         drm_i915_private_t *dev_priv;
2894         int i;
2895
2896         dev_priv = dev->dev_private;
2897         for (i = 0; i < I915_NUM_RINGS; i++)
2898                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
2899 }
2900
2901 int
2902 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2903                        struct drm_file *file_priv)
2904 {
2905         drm_i915_private_t *dev_priv = dev->dev_private;
2906         int ret;
2907
2908         if (drm_core_check_feature(dev, DRIVER_MODESET))
2909                 return 0;
2910
2911         if (atomic_read(&dev_priv->mm.wedged)) {
2912                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2913                 atomic_set(&dev_priv->mm.wedged, 0);
2914         }
2915
2916         DRM_LOCK(dev);
2917         dev_priv->mm.suspended = 0;
2918
2919         ret = i915_gem_init_hw(dev);
2920         if (ret != 0) {
2921                 DRM_UNLOCK(dev);
2922                 return ret;
2923         }
2924
2925         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
2926         DRM_UNLOCK(dev);
2927
2928         ret = drm_irq_install(dev);
2929         if (ret)
2930                 goto cleanup_ringbuffer;
2931
2932         return 0;
2933
2934 cleanup_ringbuffer:
2935         DRM_LOCK(dev);
2936         i915_gem_cleanup_ringbuffer(dev);
2937         dev_priv->mm.suspended = 1;
2938         DRM_UNLOCK(dev);
2939
2940         return ret;
2941 }
2942
2943 int
2944 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2945     struct drm_file *file_priv)
2946 {
2947
2948         if (drm_core_check_feature(dev, DRIVER_MODESET))
2949                 return 0;
2950
2951         drm_irq_uninstall(dev);
2952         return (i915_gem_idle(dev));
2953 }
2954
2955 void
2956 i915_gem_lastclose(struct drm_device *dev)
2957 {
2958         int ret;
2959
2960         if (drm_core_check_feature(dev, DRIVER_MODESET))
2961                 return;
2962
2963         ret = i915_gem_idle(dev);
2964         if (ret != 0)
2965                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2966 }
2967
2968 static void
2969 init_ring_lists(struct intel_ring_buffer *ring)
2970 {
2971
2972         INIT_LIST_HEAD(&ring->active_list);
2973         INIT_LIST_HEAD(&ring->request_list);
2974         INIT_LIST_HEAD(&ring->gpu_write_list);
2975 }
2976
2977 void
2978 i915_gem_load(struct drm_device *dev)
2979 {
2980         int i;
2981         drm_i915_private_t *dev_priv = dev->dev_private;
2982
2983         INIT_LIST_HEAD(&dev_priv->mm.active_list);
2984         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2985         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2986         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
2987         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2988         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
2989         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
2990         for (i = 0; i < I915_NUM_RINGS; i++)
2991                 init_ring_lists(&dev_priv->ring[i]);
2992         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2993                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2994         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2995                           i915_gem_retire_work_handler);
2996         init_completion(&dev_priv->error_completion);
2997
2998         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
2999         if (IS_GEN3(dev)) {
3000                 I915_WRITE(MI_ARB_STATE,
3001                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3002         }
3003
3004         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3005
3006         /* Old X drivers will take 0-2 for front, back, depth buffers */
3007         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3008                 dev_priv->fence_reg_start = 3;
3009
3010         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3011                 dev_priv->num_fence_regs = 16;
3012         else
3013                 dev_priv->num_fence_regs = 8;
3014
3015         /* Initialize fence registers to zero */
3016         i915_gem_reset_fences(dev);
3017
3018         i915_gem_detect_bit_6_swizzle(dev);
3019
3020         dev_priv->mm.interruptible = true;
3021
3022         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
3023             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
3024 }
3025
3026 static int
3027 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
3028 {
3029         drm_i915_private_t *dev_priv;
3030         struct drm_i915_gem_phys_object *phys_obj;
3031         int ret;
3032
3033         dev_priv = dev->dev_private;
3034         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
3035                 return (0);
3036
3037         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3038             M_WAITOK | M_ZERO);
3039
3040         phys_obj->id = id;
3041
3042         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3043         if (phys_obj->handle == NULL) {
3044                 ret = -ENOMEM;
3045                 goto free_obj;
3046         }
3047         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3048             size / PAGE_SIZE, PAT_WRITE_COMBINING);
3049
3050         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3051
3052         return (0);
3053
3054 free_obj:
3055         drm_free(phys_obj, DRM_I915_GEM);
3056         return (ret);
3057 }
3058
3059 static void
3060 i915_gem_free_phys_object(struct drm_device *dev, int id)
3061 {
3062         drm_i915_private_t *dev_priv;
3063         struct drm_i915_gem_phys_object *phys_obj;
3064
3065         dev_priv = dev->dev_private;
3066         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3067                 return;
3068
3069         phys_obj = dev_priv->mm.phys_objs[id - 1];
3070         if (phys_obj->cur_obj != NULL)
3071                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3072
3073         drm_pci_free(dev, phys_obj->handle);
3074         drm_free(phys_obj, DRM_I915_GEM);
3075         dev_priv->mm.phys_objs[id - 1] = NULL;
3076 }
3077
3078 void
3079 i915_gem_free_all_phys_object(struct drm_device *dev)
3080 {
3081         int i;
3082
3083         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3084                 i915_gem_free_phys_object(dev, i);
3085 }
3086
3087 void
3088 i915_gem_detach_phys_object(struct drm_device *dev,
3089     struct drm_i915_gem_object *obj)
3090 {
3091         vm_page_t m;
3092         struct sf_buf *sf;
3093         char *vaddr, *dst;
3094         int i, page_count;
3095
3096         if (obj->phys_obj == NULL)
3097                 return;
3098         vaddr = obj->phys_obj->handle->vaddr;
3099
3100         page_count = obj->base.size / PAGE_SIZE;
3101         VM_OBJECT_LOCK(obj->base.vm_obj);
3102         for (i = 0; i < page_count; i++) {
3103                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3104                 if (m == NULL)
3105                         continue; /* XXX */
3106
3107                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3108                 sf = sf_buf_alloc(m);
3109                 if (sf != NULL) {
3110                         dst = (char *)sf_buf_kva(sf);
3111                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3112                         sf_buf_free(sf);
3113                 }
3114                 drm_clflush_pages(&m, 1);
3115
3116                 VM_OBJECT_LOCK(obj->base.vm_obj);
3117                 vm_page_reference(m);
3118                 vm_page_dirty(m);
3119                 vm_page_busy_wait(m, FALSE, "i915gem");
3120                 vm_page_unwire(m, 0);
3121                 vm_page_wakeup(m);
3122                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3123         }
3124         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3125         intel_gtt_chipset_flush();
3126
3127         obj->phys_obj->cur_obj = NULL;
3128         obj->phys_obj = NULL;
3129 }
3130
3131 int
3132 i915_gem_attach_phys_object(struct drm_device *dev,
3133                             struct drm_i915_gem_object *obj,
3134                             int id,
3135                             int align)
3136 {
3137         drm_i915_private_t *dev_priv;
3138         vm_page_t m;
3139         struct sf_buf *sf;
3140         char *dst, *src;
3141         int i, page_count, ret;
3142
3143         if (id > I915_MAX_PHYS_OBJECT)
3144                 return (-EINVAL);
3145
3146         if (obj->phys_obj != NULL) {
3147                 if (obj->phys_obj->id == id)
3148                         return (0);
3149                 i915_gem_detach_phys_object(dev, obj);
3150         }
3151
3152         dev_priv = dev->dev_private;
3153         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3154                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3155                 if (ret != 0) {
3156                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3157                                   id, obj->base.size);
3158                         return (ret);
3159                 }
3160         }
3161
3162         /* bind to the object */
3163         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3164         obj->phys_obj->cur_obj = obj;
3165
3166         page_count = obj->base.size / PAGE_SIZE;
3167
3168         VM_OBJECT_LOCK(obj->base.vm_obj);
3169         ret = 0;
3170         for (i = 0; i < page_count; i++) {
3171                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3172                 if (m == NULL) {
3173                         ret = -EIO;
3174                         break;
3175                 }
3176                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3177                 sf = sf_buf_alloc(m);
3178                 src = (char *)sf_buf_kva(sf);
3179                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3180                 memcpy(dst, src, PAGE_SIZE);
3181                 sf_buf_free(sf);
3182
3183                 VM_OBJECT_LOCK(obj->base.vm_obj);
3184
3185                 vm_page_reference(m);
3186                 vm_page_busy_wait(m, FALSE, "i915gem");
3187                 vm_page_unwire(m, 0);
3188                 vm_page_wakeup(m);
3189                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3190         }
3191         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3192
3193         return (0);
3194 }
3195
3196 static int
3197 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3198     uint64_t data_ptr, uint64_t offset, uint64_t size,
3199     struct drm_file *file_priv)
3200 {
3201         char *user_data, *vaddr;
3202         int ret;
3203
3204         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3205         user_data = (char *)(uintptr_t)data_ptr;
3206
3207         if (copyin_nofault(user_data, vaddr, size) != 0) {
3208                 /* The physical object once assigned is fixed for the lifetime
3209                  * of the obj, so we can safely drop the lock and continue
3210                  * to access vaddr.
3211                  */
3212                 DRM_UNLOCK(dev);
3213                 ret = -copyin(user_data, vaddr, size);
3214                 DRM_LOCK(dev);
3215                 if (ret != 0)
3216                         return (ret);
3217         }
3218
3219         intel_gtt_chipset_flush();
3220         return (0);
3221 }
3222
3223 void
3224 i915_gem_release(struct drm_device *dev, struct drm_file *file)
3225 {
3226         struct drm_i915_file_private *file_priv;
3227         struct drm_i915_gem_request *request;
3228
3229         file_priv = file->driver_priv;
3230
3231         /* Clean up our request list when the client is going away, so that
3232          * later retire_requests won't dereference our soon-to-be-gone
3233          * file_priv.
3234          */
3235         spin_lock(&file_priv->mm.lock);
3236         while (!list_empty(&file_priv->mm.request_list)) {
3237                 request = list_first_entry(&file_priv->mm.request_list,
3238                                            struct drm_i915_gem_request,
3239                                            client_list);
3240                 list_del(&request->client_list);
3241                 request->file_priv = NULL;
3242         }
3243         spin_unlock(&file_priv->mm.lock);
3244 }
3245
3246 static int
3247 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
3248     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
3249     struct drm_file *file)
3250 {
3251         vm_object_t vm_obj;
3252         vm_page_t m;
3253         struct sf_buf *sf;
3254         vm_offset_t mkva;
3255         vm_pindex_t obj_pi;
3256         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
3257
3258         if (obj->gtt_offset != 0 && rw == UIO_READ)
3259                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
3260         else
3261                 do_bit17_swizzling = 0;
3262
3263         obj->dirty = 1;
3264         vm_obj = obj->base.vm_obj;
3265         ret = 0;
3266
3267         VM_OBJECT_LOCK(vm_obj);
3268         vm_object_pip_add(vm_obj, 1);
3269         while (size > 0) {
3270                 obj_pi = OFF_TO_IDX(offset);
3271                 obj_po = offset & PAGE_MASK;
3272
3273                 m = i915_gem_wire_page(vm_obj, obj_pi);
3274                 VM_OBJECT_UNLOCK(vm_obj);
3275
3276                 sf = sf_buf_alloc(m);
3277                 mkva = sf_buf_kva(sf);
3278                 length = min(size, PAGE_SIZE - obj_po);
3279                 while (length > 0) {
3280                         if (do_bit17_swizzling &&
3281                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
3282                                 cnt = roundup2(obj_po + 1, 64);
3283                                 cnt = min(cnt - obj_po, length);
3284                                 swizzled_po = obj_po ^ 64;
3285                         } else {
3286                                 cnt = length;
3287                                 swizzled_po = obj_po;
3288                         }
3289                         if (rw == UIO_READ)
3290                                 ret = -copyout_nofault(
3291                                     (char *)mkva + swizzled_po,
3292                                     (void *)(uintptr_t)data_ptr, cnt);
3293                         else
3294                                 ret = -copyin_nofault(
3295                                     (void *)(uintptr_t)data_ptr,
3296                                     (char *)mkva + swizzled_po, cnt);
3297                         if (ret != 0)
3298                                 break;
3299                         data_ptr += cnt;
3300                         size -= cnt;
3301                         length -= cnt;
3302                         offset += cnt;
3303                         obj_po += cnt;
3304                 }
3305                 sf_buf_free(sf);
3306                 VM_OBJECT_LOCK(vm_obj);
3307                 if (rw == UIO_WRITE)
3308                         vm_page_dirty(m);
3309                 vm_page_reference(m);
3310                 vm_page_busy_wait(m, FALSE, "i915gem");
3311                 vm_page_unwire(m, 1);
3312                 vm_page_wakeup(m);
3313                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3314
3315                 if (ret != 0)
3316                         break;
3317         }
3318         vm_object_pip_wakeup(vm_obj);
3319         VM_OBJECT_UNLOCK(vm_obj);
3320
3321         return (ret);
3322 }
3323
3324 static int
3325 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
3326     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
3327 {
3328         vm_offset_t mkva;
3329         int ret;
3330
3331         /*
3332          * Pass the unaligned physical address and size to pmap_mapdev_attr()
3333          * so it can properly calculate whether an extra page needs to be
3334          * mapped or not to cover the requested range.  The function will
3335          * add the page offset into the returned mkva for us.
3336          */
3337         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
3338             offset, size, PAT_WRITE_COMBINING);
3339         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
3340         pmap_unmapdev(mkva, size);
3341         return (ret);
3342 }
3343
3344 static int
3345 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
3346     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
3347 {
3348         struct drm_i915_gem_object *obj;
3349         vm_page_t *ma;
3350         vm_offset_t start, end;
3351         int npages, ret;
3352
3353         if (size == 0)
3354                 return (0);
3355         start = trunc_page(data_ptr);
3356         end = round_page(data_ptr + size);
3357         npages = howmany(end - start, PAGE_SIZE);
3358         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
3359             M_ZERO);
3360         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
3361             (vm_offset_t)data_ptr, size,
3362             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
3363         if (npages == -1) {
3364                 ret = -EFAULT;
3365                 goto free_ma;
3366         }
3367
3368         ret = i915_mutex_lock_interruptible(dev);
3369         if (ret != 0)
3370                 goto unlocked;
3371
3372         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
3373         if (&obj->base == NULL) {
3374                 ret = -ENOENT;
3375                 goto unlock;
3376         }
3377         if (offset > obj->base.size || size > obj->base.size - offset) {
3378                 ret = -EINVAL;
3379                 goto out;
3380         }
3381
3382         if (rw == UIO_READ) {
3383                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
3384                     offset, size);
3385                 if (ret != 0)
3386                         goto out;
3387                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3388                     UIO_READ, file);
3389         } else {
3390                 if (obj->phys_obj) {
3391                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
3392                             size, file);
3393                 } else if (obj->gtt_space &&
3394                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3395                         ret = i915_gem_object_pin(obj, 0, true);
3396                         if (ret != 0)
3397                                 goto out;
3398                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
3399                         if (ret != 0)
3400                                 goto out_unpin;
3401                         ret = i915_gem_object_put_fence(obj);
3402                         if (ret != 0)
3403                                 goto out_unpin;
3404                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
3405                             offset, file);
3406 out_unpin:
3407                         i915_gem_object_unpin(obj);
3408                 } else {
3409                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
3410                         if (ret != 0)
3411                                 goto out;
3412                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3413                             UIO_WRITE, file);
3414                 }
3415         }
3416 out:
3417         drm_gem_object_unreference(&obj->base);
3418 unlock:
3419         DRM_UNLOCK(dev);
3420 unlocked:
3421         vm_page_unhold_pages(ma, npages);
3422 free_ma:
3423         drm_free(ma, DRM_I915_GEM);
3424         return (ret);
3425 }
3426
3427 static int
3428 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
3429     vm_ooffset_t foff, struct ucred *cred, u_short *color)
3430 {
3431
3432         *color = 0; /* XXXKIB */
3433         return (0);
3434 }
3435
3436 int i915_intr_pf;
3437
3438 static int
3439 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
3440     vm_page_t *mres)
3441 {
3442         struct drm_gem_object *gem_obj;
3443         struct drm_i915_gem_object *obj;
3444         struct drm_device *dev;
3445         drm_i915_private_t *dev_priv;
3446         vm_page_t m, oldm;
3447         int cause, ret;
3448         bool write;
3449
3450         gem_obj = vm_obj->handle;
3451         obj = to_intel_bo(gem_obj);
3452         dev = obj->base.dev;
3453         dev_priv = dev->dev_private;
3454 #if 0
3455         write = (prot & VM_PROT_WRITE) != 0;
3456 #else
3457         write = true;
3458 #endif
3459         vm_object_pip_add(vm_obj, 1);
3460
3461         /*
3462          * Remove the placeholder page inserted by vm_fault() from the
3463          * object before dropping the object lock. If
3464          * i915_gem_release_mmap() is active in parallel on this gem
3465          * object, then it owns the drm device sx and might find the
3466          * placeholder already. Then, since the page is busy,
3467          * i915_gem_release_mmap() sleeps waiting for the busy state
3468          * of the page cleared. We will be not able to acquire drm
3469          * device lock until i915_gem_release_mmap() is able to make a
3470          * progress.
3471          */
3472         if (*mres != NULL) {
3473                 oldm = *mres;
3474                 vm_page_remove(oldm);
3475                 *mres = NULL;
3476         } else
3477                 oldm = NULL;
3478 retry:
3479         VM_OBJECT_UNLOCK(vm_obj);
3480 unlocked_vmobj:
3481         cause = ret = 0;
3482         m = NULL;
3483
3484         if (i915_intr_pf) {
3485                 ret = i915_mutex_lock_interruptible(dev);
3486                 if (ret != 0) {
3487                         cause = 10;
3488                         goto out;
3489                 }
3490         } else
3491                 DRM_LOCK(dev);
3492
3493         /*
3494          * Since the object lock was dropped, other thread might have
3495          * faulted on the same GTT address and instantiated the
3496          * mapping for the page.  Recheck.
3497          */
3498         VM_OBJECT_LOCK(vm_obj);
3499         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
3500         if (m != NULL) {
3501                 if ((m->flags & PG_BUSY) != 0) {
3502                         DRM_UNLOCK(dev);
3503 #if 0 /* XXX */
3504                         vm_page_sleep(m, "915pee");
3505 #endif
3506                         goto retry;
3507                 }
3508                 goto have_page;
3509         } else
3510                 VM_OBJECT_UNLOCK(vm_obj);
3511
3512         /* Now bind it into the GTT if needed */
3513         if (!obj->map_and_fenceable) {
3514                 ret = i915_gem_object_unbind(obj);
3515                 if (ret != 0) {
3516                         cause = 20;
3517                         goto unlock;
3518                 }
3519         }
3520         if (!obj->gtt_space) {
3521                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
3522                 if (ret != 0) {
3523                         cause = 30;
3524                         goto unlock;
3525                 }
3526
3527                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
3528                 if (ret != 0) {
3529                         cause = 40;
3530                         goto unlock;
3531                 }
3532         }
3533
3534         if (obj->tiling_mode == I915_TILING_NONE)
3535                 ret = i915_gem_object_put_fence(obj);
3536         else
3537                 ret = i915_gem_object_get_fence(obj, NULL);
3538         if (ret != 0) {
3539                 cause = 50;
3540                 goto unlock;
3541         }
3542
3543         if (i915_gem_object_is_inactive(obj))
3544                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3545
3546         obj->fault_mappable = true;
3547         VM_OBJECT_LOCK(vm_obj);
3548         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
3549             offset);
3550         if (m == NULL) {
3551                 cause = 60;
3552                 ret = -EFAULT;
3553                 goto unlock;
3554         }
3555         KASSERT((m->flags & PG_FICTITIOUS) != 0,
3556             ("not fictitious %p", m));
3557         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
3558
3559         if ((m->flags & PG_BUSY) != 0) {
3560                 DRM_UNLOCK(dev);
3561 #if 0 /* XXX */
3562                 vm_page_sleep(m, "915pbs");
3563 #endif
3564                 goto retry;
3565         }
3566         m->valid = VM_PAGE_BITS_ALL;
3567         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
3568 have_page:
3569         *mres = m;
3570         vm_page_busy_try(m, false);
3571
3572         DRM_UNLOCK(dev);
3573         if (oldm != NULL) {
3574                 vm_page_free(oldm);
3575         }
3576         vm_object_pip_wakeup(vm_obj);
3577         return (VM_PAGER_OK);
3578
3579 unlock:
3580         DRM_UNLOCK(dev);
3581 out:
3582         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
3583         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
3584                 goto unlocked_vmobj;
3585         }
3586         VM_OBJECT_LOCK(vm_obj);
3587         vm_object_pip_wakeup(vm_obj);
3588         return (VM_PAGER_ERROR);
3589 }
3590
3591 static void
3592 i915_gem_pager_dtor(void *handle)
3593 {
3594         struct drm_gem_object *obj;
3595         struct drm_device *dev;
3596
3597         obj = handle;
3598         dev = obj->dev;
3599
3600         DRM_LOCK(dev);
3601         drm_gem_free_mmap_offset(obj);
3602         i915_gem_release_mmap(to_intel_bo(obj));
3603         drm_gem_object_unreference(obj);
3604         DRM_UNLOCK(dev);
3605 }
3606
3607 struct cdev_pager_ops i915_gem_pager_ops = {
3608         .cdev_pg_fault  = i915_gem_pager_fault,
3609         .cdev_pg_ctor   = i915_gem_pager_ctor,
3610         .cdev_pg_dtor   = i915_gem_pager_dtor
3611 };
3612
3613 static int
3614 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3615     uint64_t offset, uint64_t size)
3616 {
3617         uint32_t old_read_domains;
3618         int i, ret;
3619
3620         if (offset == 0 && size == obj->base.size)
3621                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
3622
3623         ret = i915_gem_object_flush_gpu_write_domain(obj);
3624         if (ret != 0)
3625                 return (ret);
3626         ret = i915_gem_object_wait_rendering(obj);
3627         if (ret != 0)
3628                 return (ret);
3629
3630         i915_gem_object_flush_gtt_write_domain(obj);
3631
3632         if (obj->page_cpu_valid == NULL &&
3633             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3634                 return (0);
3635
3636         if (obj->page_cpu_valid == NULL) {
3637                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
3638                     DRM_I915_GEM, M_WAITOK | M_ZERO);
3639         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3640                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3641
3642         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3643              i++) {
3644                 if (obj->page_cpu_valid[i])
3645                         continue;
3646                 drm_clflush_pages(obj->pages + i, 1);
3647                 obj->page_cpu_valid[i] = 1;
3648         }
3649
3650         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
3651             ("In gpu write domain"));
3652
3653         old_read_domains = obj->base.read_domains;
3654         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3655
3656         return (0);
3657 }
3658
3659 #define GEM_PARANOID_CHECK_GTT 0
3660 #if GEM_PARANOID_CHECK_GTT
3661 static void
3662 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
3663     int page_count)
3664 {
3665         struct drm_i915_private *dev_priv;
3666         vm_paddr_t pa;
3667         unsigned long start, end;
3668         u_int i;
3669         int j;
3670
3671         dev_priv = dev->dev_private;
3672         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
3673         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
3674         for (i = start; i < end; i++) {
3675                 pa = intel_gtt_read_pte_paddr(i);
3676                 for (j = 0; j < page_count; j++) {
3677                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
3678                                 panic("Page %p in GTT pte index %d pte %x",
3679                                     ma[i], i, intel_gtt_read_pte(i));
3680                         }
3681                 }
3682         }
3683 }
3684 #endif
3685
3686 static void
3687 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
3688     uint32_t flush_domains)
3689 {
3690         struct drm_i915_gem_object *obj, *next;
3691         uint32_t old_write_domain;
3692
3693         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
3694             gpu_write_list) {
3695                 if (obj->base.write_domain & flush_domains) {
3696                         old_write_domain = obj->base.write_domain;
3697                         obj->base.write_domain = 0;
3698                         list_del_init(&obj->gpu_write_list);
3699                         i915_gem_object_move_to_active(obj, ring,
3700                             i915_gem_next_request_seqno(ring));
3701                 }
3702         }
3703 }
3704
3705 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3706
3707 static vm_page_t
3708 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3709 {
3710         vm_page_t m;
3711         int rv;
3712
3713         VM_OBJECT_LOCK_ASSERT_OWNED(object);
3714         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
3715         if (m->valid != VM_PAGE_BITS_ALL) {
3716                 if (vm_pager_has_page(object, pindex)) {
3717                         rv = vm_pager_get_page(object, &m, 1);
3718                         m = vm_page_lookup(object, pindex);
3719                         if (m == NULL)
3720                                 return (NULL);
3721                         if (rv != VM_PAGER_OK) {
3722                                 vm_page_free(m);
3723                                 return (NULL);
3724                         }
3725                 } else {
3726                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
3727                         m->valid = VM_PAGE_BITS_ALL;
3728                         m->dirty = 0;
3729                 }
3730         }
3731         vm_page_wire(m);
3732         vm_page_wakeup(m);
3733         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3734         return (m);
3735 }
3736
3737 int
3738 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
3739     uint32_t flush_domains)
3740 {
3741         int ret;
3742
3743         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
3744                 return 0;
3745
3746         ret = ring->flush(ring, invalidate_domains, flush_domains);
3747         if (ret)
3748                 return ret;
3749
3750         if (flush_domains & I915_GEM_GPU_DOMAINS)
3751                 i915_gem_process_flushing_list(ring, flush_domains);
3752         return 0;
3753 }
3754
3755 u32
3756 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
3757 {
3758         if (ring->outstanding_lazy_request == 0)
3759                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
3760
3761         return ring->outstanding_lazy_request;
3762 }
3763
3764 static void
3765 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3766 {
3767         drm_i915_private_t *dev_priv = dev->dev_private;
3768         uint32_t fence_reg = reg - dev_priv->fence_regs;
3769
3770         switch (INTEL_INFO(dev)->gen) {
3771         case 7:
3772         case 6:
3773                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3774                 break;
3775         case 5:
3776         case 4:
3777                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3778                 break;
3779         case 3:
3780                 if (fence_reg >= 8)
3781                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3782                 else
3783         case 2:
3784                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3785
3786                 I915_WRITE(fence_reg, 0);
3787                 break;
3788         }
3789
3790         list_del_init(&reg->lru_list);
3791         reg->obj = NULL;
3792         reg->setup_seqno = 0;
3793         reg->pin_count = 0;
3794 }
3795
3796 static int
3797 i915_gpu_is_active(struct drm_device *dev)
3798 {
3799         drm_i915_private_t *dev_priv;
3800
3801         dev_priv = dev->dev_private;
3802         return (!list_empty(&dev_priv->mm.flushing_list) ||
3803             !list_empty(&dev_priv->mm.active_list));
3804 }
3805
3806 static void
3807 i915_gem_lowmem(void *arg)
3808 {
3809         struct drm_device *dev;
3810         struct drm_i915_private *dev_priv;
3811         struct drm_i915_gem_object *obj, *next;
3812         int cnt, cnt_fail, cnt_total;
3813
3814         dev = arg;
3815         dev_priv = dev->dev_private;
3816
3817         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3818                 return;
3819
3820 rescan:
3821         /* first scan for clean buffers */
3822         i915_gem_retire_requests(dev);
3823
3824         cnt_total = cnt_fail = cnt = 0;
3825
3826         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3827             mm_list) {
3828                 if (i915_gem_object_is_purgeable(obj)) {
3829                         if (i915_gem_object_unbind(obj) != 0)
3830                                 cnt_total++;
3831                 } else
3832                         cnt_total++;
3833         }
3834
3835         /* second pass, evict/count anything still on the inactive list */
3836         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3837             mm_list) {
3838                 if (i915_gem_object_unbind(obj) == 0)
3839                         cnt++;
3840                 else
3841                         cnt_fail++;
3842         }
3843
3844         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3845                 /*
3846                  * We are desperate for pages, so as a last resort, wait
3847                  * for the GPU to finish and discard whatever we can.
3848                  * This has a dramatic impact to reduce the number of
3849                  * OOM-killer events whilst running the GPU aggressively.
3850                  */
3851                 if (i915_gpu_idle(dev) == 0)
3852                         goto rescan;
3853         }
3854         DRM_UNLOCK(dev);
3855 }
3856
3857 void
3858 i915_gem_unload(struct drm_device *dev)
3859 {
3860         struct drm_i915_private *dev_priv;
3861
3862         dev_priv = dev->dev_private;
3863         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3864 }