sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64 #include <linux/completion.h>
  65
  66 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  67 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  68 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  69     unsigned alignment, bool map_and_fenceable);
  70
  71 static int i915_gem_phys_pwrite(struct drm_device *dev,
  72     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  73     uint64_t size, struct drm_file *file_priv);
  74
  75 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  76     int tiling_mode);
  77 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  78     uint32_t size, int tiling_mode);
  79 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  80     int flags);
  81 static int i915_gem_object_set_cpu_read_domain_range(
  82     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  83 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  84 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  85 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  86 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  87 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  88 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  89 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  90     uint32_t flush_domains);
  91 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  92     struct drm_i915_fence_reg *reg);
  93 static void i915_gem_reset_fences(struct drm_device *dev);
  94 static void i915_gem_lowmem(void *arg);
  95
  96 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
  97     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file);
  98
  99 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
 100 long i915_gem_wired_pages_cnt;
 101
 102 /* some bookkeeping */
 103 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 104                                   size_t size)
 105 {
 106
 107         dev_priv->mm.object_count++;
 108         dev_priv->mm.object_memory += size;
 109 }
 110
 111 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 112                                      size_t size)
 113 {
 114
 115         dev_priv->mm.object_count--;
 116         dev_priv->mm.object_memory -= size;
 117 }
 118
 119 static int
 120 i915_gem_wait_for_error(struct drm_device *dev)
 121 {
 122         struct drm_i915_private *dev_priv = dev->dev_private;
 123         struct completion *x = &dev_priv->error_completion;
 124         int ret;
 125
 126         if (!atomic_read(&dev_priv->mm.wedged))
 127                 return 0;
 128
 129         /*
 130          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 131          * userspace. If it takes that long something really bad is going on and
 132          * we should simply try to bail out and fail as gracefully as possible.
 133          */
 134         ret = wait_for_completion_interruptible_timeout(x, 10*hz);
 135         if (ret == 0) {
 136                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 137                 return -EIO;
 138         } else if (ret < 0) {
 139                 return ret;
 140         }
 141
 142         if (atomic_read(&dev_priv->mm.wedged)) {
 143                 /* GPU is hung, bump the completion count to account for
 144                  * the token we just consumed so that we never hit zero and
 145                  * end up waiting upon a subsequent completion event that
 146                  * will never happen.
 147                  */
 148                 spin_lock(&x->wait.lock);
 149                 x->done++;
 150                 spin_unlock(&x->wait.lock);
 151         }
 152         return 0;
 153 }
 154
 155 int i915_mutex_lock_interruptible(struct drm_device *dev)
 156 {
 157         int ret;
 158
 159         ret = i915_gem_wait_for_error(dev);
 160         if (ret != 0)
 161                 return (ret);
 162
 163         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 164         if (ret)
 165                 return -EINTR;
 166
 167 #if 0
 168         WARN_ON(i915_verify_lists(dev));
 169 #endif
 170         return 0;
 171 }
 172
 173 static inline bool
 174 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 175 {
 176         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
 177 }
 178
 179 int
 180 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 181     struct drm_file *file)
 182 {
 183         struct drm_i915_gem_init *args;
 184         drm_i915_private_t *dev_priv;
 185
 186         dev_priv = dev->dev_private;
 187         args = data;
 188
 189         if (args->gtt_start >= args->gtt_end ||
 190             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 191                 return (-EINVAL);
 192
 193         /*
 194          * XXXKIB. The second-time initialization should be guarded
 195          * against.
 196          */
 197         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 198         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 199         lockmgr(&dev->dev_lock, LK_RELEASE);
 200
 201         return 0;
 202 }
 203
 204 int
 205 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 206                             struct drm_file *file)
 207 {
 208         struct drm_i915_private *dev_priv;
 209         struct drm_i915_gem_get_aperture *args;
 210         struct drm_i915_gem_object *obj;
 211         size_t pinned;
 212
 213         dev_priv = dev->dev_private;
 214         args = data;
 215
 216         if (!(dev->driver->driver_features & DRIVER_GEM))
 217                 return (-ENODEV);
 218
 219         pinned = 0;
 220         DRM_LOCK(dev);
 221         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 222                 pinned += obj->gtt_space->size;
 223         DRM_UNLOCK(dev);
 224
 225         args->aper_size = dev_priv->mm.gtt_total;
 226         args->aper_available_size = args->aper_size - pinned;
 227
 228         return (0);
 229 }
 230
 231 static int
 232 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 233     uint32_t *handle_p)
 234 {
 235         struct drm_i915_gem_object *obj;
 236         uint32_t handle;
 237         int ret;
 238
 239         size = roundup(size, PAGE_SIZE);
 240         if (size == 0)
 241                 return (-EINVAL);
 242
 243         obj = i915_gem_alloc_object(dev, size);
 244         if (obj == NULL)
 245                 return (-ENOMEM);
 246
 247         handle = 0;
 248         ret = drm_gem_handle_create(file, &obj->base, &handle);
 249         if (ret != 0) {
 250                 drm_gem_object_release(&obj->base);
 251                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 252                 drm_free(obj, DRM_I915_GEM);
 253                 return (-ret);
 254         }
 255
 256         /* drop reference from allocate - handle holds it now */
 257         drm_gem_object_unreference(&obj->base);
 258         *handle_p = handle;
 259         return (0);
 260 }
 261
 262 int
 263 i915_gem_dumb_create(struct drm_file *file,
 264                      struct drm_device *dev,
 265                      struct drm_mode_create_dumb *args)
 266 {
 267
 268         /* have to work out size/pitch and return them */
 269         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 270         args->size = args->pitch * args->height;
 271         return (i915_gem_create(file, dev, args->size, &args->handle));
 272 }
 273
 274 int i915_gem_dumb_destroy(struct drm_file *file,
 275                           struct drm_device *dev,
 276                           uint32_t handle)
 277 {
 278
 279         return (drm_gem_handle_delete(file, handle));
 280 }
 281
 282 /**
 283  * Creates a new mm object and returns a handle to it.
 284  */
 285 int
 286 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 287                       struct drm_file *file)
 288 {
 289         struct drm_i915_gem_create *args = data;
 290
 291         return (i915_gem_create(file, dev, args->size, &args->handle));
 292 }
 293
 294 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 295 {
 296         drm_i915_private_t *dev_priv;
 297
 298         dev_priv = obj->base.dev->dev_private;
 299         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 300             obj->tiling_mode != I915_TILING_NONE);
 301 }
 302
 303 /**
 304  * Reads data from the object referenced by handle.
 305  *
 306  * On error, the contents of *data are undefined.
 307  */
 308 int
 309 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 310                      struct drm_file *file)
 311 {
 312         struct drm_i915_gem_pread *args;
 313
 314         args = data;
 315         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 316             args->offset, UIO_READ, file));
 317 }
 318
 319 /**
 320  * Writes data to the object referenced by handle.
 321  *
 322  * On error, the contents of the buffer that were to be modified are undefined.
 323  */
 324 int
 325 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 326                       struct drm_file *file)
 327 {
 328         struct drm_i915_gem_pwrite *args;
 329
 330         args = data;
 331         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 332             args->offset, UIO_WRITE, file));
 333 }
 334
 335 int
 336 i915_gem_check_wedge(struct drm_i915_private *dev_priv,
 337                      bool interruptible)
 338 {
 339         if (atomic_read(&dev_priv->mm.wedged)) {
 340                 struct completion *x = &dev_priv->error_completion;
 341                 bool recovery_complete;
 342
 343                 /* Give the error handler a chance to run. */
 344                 spin_lock(&x->wait.lock);
 345                 recovery_complete = x->done > 0;
 346                 spin_unlock(&x->wait.lock);
 347
 348                 /* Non-interruptible callers can't handle -EAGAIN, hence return
 349                  * -EIO unconditionally for these. */
 350                 if (!interruptible)
 351                         return -EIO;
 352
 353                 /* Recovery complete, but still wedged means reset failure. */
 354                 if (recovery_complete)
 355                         return -EIO;
 356
 357                 return -EAGAIN;
 358         }
 359
 360         return 0;
 361 }
 362
 363 /**
 364  * Waits for a sequence number to be signaled, and cleans up the
 365  * request and object lists appropriately for that event.
 366  */
 367 int
 368 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
 369 {
 370         struct drm_device *dev = ring->dev;
 371         struct drm_i915_private *dev_priv = dev->dev_private;
 372         bool interruptible = dev_priv->mm.interruptible;
 373         struct drm_i915_gem_request *request;
 374         uint32_t ier;
 375         int flags, ret;
 376         bool do_retire = true;
 377
 378         DRM_LOCK_ASSERT(dev);
 379         BUG_ON(seqno == 0);
 380
 381         ret = i915_gem_check_wedge(dev_priv, interruptible);
 382         if (ret)
 383                 return ret;
 384
 385         if (seqno == ring->outstanding_lazy_request) {
 386                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
 387                     M_WAITOK | M_ZERO);
 388                 if (request == NULL)
 389                         return (-ENOMEM);
 390
 391                 ret = i915_add_request(ring, NULL, request);
 392                 if (ret != 0) {
 393                         drm_free(request, DRM_I915_GEM);
 394                         return (ret);
 395                 }
 396
 397                 seqno = request->seqno;
 398         }
 399
 400         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
 401                 if (HAS_PCH_SPLIT(ring->dev))
 402                         ier = I915_READ(DEIER) | I915_READ(GTIER);
 403                 else
 404                         ier = I915_READ(IER);
 405                 if (!ier) {
 406                         DRM_ERROR("something (likely vbetool) disabled "
 407                                   "interrupts, re-enabling\n");
 408                         ring->dev->driver->irq_preinstall(ring->dev);
 409                         ring->dev->driver->irq_postinstall(ring->dev);
 410                 }
 411
 412                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
 413                 if (ring->irq_get(ring)) {
 414                         flags = dev_priv->mm.interruptible ? PCATCH : 0;
 415                         while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)
 416                             && !atomic_read(&dev_priv->mm.wedged) &&
 417                             ret == 0) {
 418                                 ret = -lksleep(ring, &ring->irq_lock, flags,
 419                                     "915gwr", 1*hz);
 420                         }
 421                         ring->irq_put(ring);
 422                         lockmgr(&ring->irq_lock, LK_RELEASE);
 423                 } else {
 424                         lockmgr(&ring->irq_lock, LK_RELEASE);
 425                         if (_intel_wait_for(ring->dev,
 426                             i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
 427                             atomic_read(&dev_priv->mm.wedged), 3000,
 428                             0, "i915wrq") != 0)
 429                                 ret = -EBUSY;
 430                 }
 431
 432         }
 433         if (atomic_read(&dev_priv->mm.wedged))
 434                 ret = -EAGAIN;
 435
 436         /* Directly dispatch request retiring.  While we have the work queue
 437          * to handle this, the waiter on a request often wants an associated
 438          * buffer to have made it to the inactive list, and we would need
 439          * a separate wait queue to handle that.
 440          */
 441         if (ret == 0 && do_retire)
 442                 i915_gem_retire_requests_ring(ring);
 443
 444         return (ret);
 445 }
 446
 447 /**
 448  * Ensures that all rendering to the object has completed and the object is
 449  * safe to unbind from the GTT or access from the CPU.
 450  */
 451 int
 452 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 453 {
 454         u32 seqno;
 455         int ret;
 456
 457
 458         seqno = obj->last_rendering_seqno;
 459         if (seqno == 0)
 460                 return 0;
 461
 462         if (obj->active) {
 463                 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
 464                 if (ret != 0)
 465                         return (ret);
 466         }
 467
 468         /* Manually manage the write flush as we may have not yet
 469          * retired the buffer.
 470          */
 471         if (obj->last_rendering_seqno &&
 472             i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
 473                 obj->last_rendering_seqno = 0;
 474                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
 475         }
 476
 477         return 0;
 478 }
 479
 480 /**
 481  * Called when user space prepares to use an object with the CPU, either
 482  * through the mmap ioctl's mapping or a GTT mapping.
 483  */
 484 int
 485 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 486                           struct drm_file *file)
 487 {
 488         struct drm_i915_gem_set_domain *args;
 489         struct drm_i915_gem_object *obj;
 490         uint32_t read_domains;
 491         uint32_t write_domain;
 492         int ret;
 493
 494         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 495                 return (-ENODEV);
 496
 497         args = data;
 498         read_domains = args->read_domains;
 499         write_domain = args->write_domain;
 500
 501         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
 502             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
 503             (write_domain != 0 && read_domains != write_domain))
 504                 return (-EINVAL);
 505
 506         ret = i915_mutex_lock_interruptible(dev);
 507         if (ret != 0)
 508                 return (ret);
 509
 510         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 511         if (&obj->base == NULL) {
 512                 ret = -ENOENT;
 513                 goto unlock;
 514         }
 515
 516         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
 517                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 518                 if (ret == -EINVAL)
 519                         ret = 0;
 520         } else
 521                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 522
 523         drm_gem_object_unreference(&obj->base);
 524 unlock:
 525         DRM_UNLOCK(dev);
 526         return (ret);
 527 }
 528
 529 /**
 530  * Called when user space has done writes to this buffer
 531  */
 532 int
 533 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 534                          struct drm_file *file)
 535 {
 536         struct drm_i915_gem_sw_finish *args;
 537         struct drm_i915_gem_object *obj;
 538         int ret;
 539
 540         args = data;
 541         ret = 0;
 542         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 543                 return (ENODEV);
 544         ret = i915_mutex_lock_interruptible(dev);
 545         if (ret != 0)
 546                 return (ret);
 547         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 548         if (&obj->base == NULL) {
 549                 ret = -ENOENT;
 550                 goto unlock;
 551         }
 552         if (obj->pin_count != 0)
 553                 i915_gem_object_flush_cpu_write_domain(obj);
 554         drm_gem_object_unreference(&obj->base);
 555 unlock:
 556         DRM_UNLOCK(dev);
 557         return (ret);
 558 }
 559
 560 /**
 561  * Maps the contents of an object, returning the address it is mapped
 562  * into.
 563  *
 564  * While the mapping holds a reference on the contents of the object, it doesn't
 565  * imply a ref on the object itself.
 566  */
 567 int
 568 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 569                     struct drm_file *file)
 570 {
 571         struct drm_i915_gem_mmap *args;
 572         struct drm_gem_object *obj;
 573         struct proc *p;
 574         vm_map_t map;
 575         vm_offset_t addr;
 576         vm_size_t size;
 577         int error, rv;
 578
 579         args = data;
 580
 581         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 582                 return (-ENODEV);
 583
 584         obj = drm_gem_object_lookup(dev, file, args->handle);
 585         if (obj == NULL)
 586                 return (-ENOENT);
 587         error = 0;
 588         if (args->size == 0)
 589                 goto out;
 590         p = curproc;
 591         map = &p->p_vmspace->vm_map;
 592         size = round_page(args->size);
 593         PROC_LOCK(p);
 594         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
 595                 PROC_UNLOCK(p);
 596                 error = ENOMEM;
 597                 goto out;
 598         }
 599         PROC_UNLOCK(p);
 600
 601         addr = 0;
 602         vm_object_hold(obj->vm_obj);
 603         vm_object_reference_locked(obj->vm_obj);
 604         vm_object_drop(obj->vm_obj);
 605         DRM_UNLOCK(dev);
 606         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
 607             PAGE_SIZE, /* align */
 608             TRUE, /* fitit */
 609             VM_MAPTYPE_NORMAL, /* maptype */
 610             VM_PROT_READ | VM_PROT_WRITE, /* prot */
 611             VM_PROT_READ | VM_PROT_WRITE, /* max */
 612             MAP_SHARED /* cow */);
 613         if (rv != KERN_SUCCESS) {
 614                 vm_object_deallocate(obj->vm_obj);
 615                 error = -vm_mmap_to_errno(rv);
 616         } else {
 617                 args->addr_ptr = (uint64_t)addr;
 618         }
 619         DRM_LOCK(dev);
 620 out:
 621         drm_gem_object_unreference(obj);
 622         return (error);
 623 }
 624
 625 /**
 626  * i915_gem_release_mmap - remove physical page mappings
 627  * @obj: obj in question
 628  *
 629  * Preserve the reservation of the mmapping with the DRM core code, but
 630  * relinquish ownership of the pages back to the system.
 631  *
 632  * It is vital that we remove the page mapping if we have mapped a tiled
 633  * object through the GTT and then lose the fence register due to
 634  * resource pressure. Similarly if the object has been moved out of the
 635  * aperture, than pages mapped into userspace must be revoked. Removing the
 636  * mapping will then trigger a page fault on the next user access, allowing
 637  * fixup by i915_gem_fault().
 638  */
 639 void
 640 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 641 {
 642         vm_object_t devobj;
 643         vm_page_t m;
 644         int i, page_count;
 645
 646         if (!obj->fault_mappable)
 647                 return;
 648
 649         devobj = cdev_pager_lookup(obj);
 650         if (devobj != NULL) {
 651                 page_count = OFF_TO_IDX(obj->base.size);
 652
 653                 VM_OBJECT_LOCK(devobj);
 654                 for (i = 0; i < page_count; i++) {
 655                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
 656                         if (m == NULL)
 657                                 continue;
 658                         cdev_pager_free_page(devobj, m);
 659                 }
 660                 VM_OBJECT_UNLOCK(devobj);
 661                 vm_object_deallocate(devobj);
 662         }
 663
 664         obj->fault_mappable = false;
 665 }
 666
 667 static uint32_t
 668 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 669 {
 670         uint32_t gtt_size;
 671
 672         if (INTEL_INFO(dev)->gen >= 4 ||
 673             tiling_mode == I915_TILING_NONE)
 674                 return (size);
 675
 676         /* Previous chips need a power-of-two fence region when tiling */
 677         if (INTEL_INFO(dev)->gen == 3)
 678                 gtt_size = 1024*1024;
 679         else
 680                 gtt_size = 512*1024;
 681
 682         while (gtt_size < size)
 683                 gtt_size <<= 1;
 684
 685         return (gtt_size);
 686 }
 687
 688 /**
 689  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 690  * @obj: object to check
 691  *
 692  * Return the required GTT alignment for an object, taking into account
 693  * potential fence register mapping.
 694  */
 695 static uint32_t
 696 i915_gem_get_gtt_alignment(struct drm_device *dev,
 697                            uint32_t size,
 698                            int tiling_mode)
 699 {
 700
 701         /*
 702          * Minimum alignment is 4k (GTT page size), but might be greater
 703          * if a fence register is needed for the object.
 704          */
 705         if (INTEL_INFO(dev)->gen >= 4 ||
 706             tiling_mode == I915_TILING_NONE)
 707                 return (4096);
 708
 709         /*
 710          * Previous chips need to be aligned to the size of the smallest
 711          * fence register that can contain the object.
 712          */
 713         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 714 }
 715
 716 /**
 717  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
 718  *                                       unfenced object
 719  * @dev: the device
 720  * @size: size of the object
 721  * @tiling_mode: tiling mode of the object
 722  *
 723  * Return the required GTT alignment for an object, only taking into account
 724  * unfenced tiled surface requirements.
 725  */
 726 uint32_t
 727 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 728                                     uint32_t size,
 729                                     int tiling_mode)
 730 {
 731
 732         if (tiling_mode == I915_TILING_NONE)
 733                 return (4096);
 734
 735         /*
 736          * Minimum alignment is 4k (GTT page size) for sane hw.
 737          */
 738         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
 739                 return (4096);
 740
 741         /*
 742          * Previous hardware however needs to be aligned to a power-of-two
 743          * tile height. The simplest method for determining this is to reuse
 744          * the power-of-tile object size.
 745          */
 746         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 747 }
 748
 749 int
 750 i915_gem_mmap_gtt(struct drm_file *file,
 751                   struct drm_device *dev,
 752                   uint32_t handle,
 753                   uint64_t *offset)
 754 {
 755         struct drm_i915_private *dev_priv;
 756         struct drm_i915_gem_object *obj;
 757         int ret;
 758
 759         if (!(dev->driver->driver_features & DRIVER_GEM))
 760                 return (-ENODEV);
 761
 762         dev_priv = dev->dev_private;
 763
 764         ret = i915_mutex_lock_interruptible(dev);
 765         if (ret != 0)
 766                 return (ret);
 767
 768         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 769         if (&obj->base == NULL) {
 770                 ret = -ENOENT;
 771                 goto unlock;
 772         }
 773
 774         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
 775                 ret = -E2BIG;
 776                 goto out;
 777         }
 778
 779         if (obj->madv != I915_MADV_WILLNEED) {
 780                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
 781                 ret = -EINVAL;
 782                 goto out;
 783         }
 784
 785         ret = drm_gem_create_mmap_offset(&obj->base);
 786         if (ret != 0)
 787                 goto out;
 788
 789         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
 790             DRM_GEM_MAPPING_KEY;
 791 out:
 792         drm_gem_object_unreference(&obj->base);
 793 unlock:
 794         DRM_UNLOCK(dev);
 795         return (ret);
 796 }
 797
 798 /**
 799  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 800  * @dev: DRM device
 801  * @data: GTT mapping ioctl data
 802  * @file: GEM object info
 803  *
 804  * Simply returns the fake offset to userspace so it can mmap it.
 805  * The mmap call will end up in drm_gem_mmap(), which will set things
 806  * up so we can get faults in the handler above.
 807  *
 808  * The fault handler will take care of binding the object into the GTT
 809  * (since it may have been evicted to make room for something), allocating
 810  * a fence register, and mapping the appropriate aperture address into
 811  * userspace.
 812  */
 813 int
 814 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 815                         struct drm_file *file)
 816 {
 817         struct drm_i915_private *dev_priv;
 818         struct drm_i915_gem_mmap_gtt *args;
 819
 820         dev_priv = dev->dev_private;
 821         args = data;
 822
 823         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
 824 }
 825
 826 /* Immediately discard the backing storage */
 827 static void
 828 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 829 {
 830         vm_object_t vm_obj;
 831
 832         vm_obj = obj->base.vm_obj;
 833         VM_OBJECT_LOCK(vm_obj);
 834         vm_object_page_remove(vm_obj, 0, 0, false);
 835         VM_OBJECT_UNLOCK(vm_obj);
 836         obj->madv = __I915_MADV_PURGED;
 837 }
 838
 839 static inline int
 840 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 841 {
 842         return obj->madv == I915_MADV_DONTNEED;
 843 }
 844
 845 static inline void vm_page_reference(vm_page_t m)
 846 {
 847         vm_page_flag_set(m, PG_REFERENCED);
 848 }
 849
 850 static void
 851 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 852 {
 853         vm_page_t m;
 854         int page_count, i;
 855
 856         BUG_ON(obj->madv == __I915_MADV_PURGED);
 857
 858         if (obj->tiling_mode != I915_TILING_NONE)
 859                 i915_gem_object_save_bit_17_swizzle(obj);
 860         if (obj->madv == I915_MADV_DONTNEED)
 861                 obj->dirty = 0;
 862         page_count = obj->base.size / PAGE_SIZE;
 863         VM_OBJECT_LOCK(obj->base.vm_obj);
 864 #if GEM_PARANOID_CHECK_GTT
 865         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
 866 #endif
 867         for (i = 0; i < page_count; i++) {
 868                 m = obj->pages[i];
 869                 if (obj->dirty)
 870                         vm_page_dirty(m);
 871                 if (obj->madv == I915_MADV_WILLNEED)
 872                         vm_page_reference(m);
 873                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
 874                 vm_page_unwire(obj->pages[i], 1);
 875                 vm_page_wakeup(obj->pages[i]);
 876                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 877         }
 878         VM_OBJECT_UNLOCK(obj->base.vm_obj);
 879         obj->dirty = 0;
 880         drm_free(obj->pages, DRM_I915_GEM);
 881         obj->pages = NULL;
 882 }
 883
 884 static int
 885 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 886     int flags)
 887 {
 888         struct drm_device *dev;
 889         vm_object_t vm_obj;
 890         vm_page_t m;
 891         int page_count, i, j;
 892
 893         dev = obj->base.dev;
 894         KASSERT(obj->pages == NULL, ("Obj already has pages"));
 895         page_count = obj->base.size / PAGE_SIZE;
 896         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 897             M_WAITOK);
 898         vm_obj = obj->base.vm_obj;
 899         VM_OBJECT_LOCK(vm_obj);
 900         for (i = 0; i < page_count; i++) {
 901                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
 902                         goto failed;
 903         }
 904         VM_OBJECT_UNLOCK(vm_obj);
 905         if (i915_gem_object_needs_bit17_swizzle(obj))
 906                 i915_gem_object_do_bit_17_swizzle(obj);
 907         return (0);
 908
 909 failed:
 910         for (j = 0; j < i; j++) {
 911                 m = obj->pages[j];
 912                 vm_page_busy_wait(m, FALSE, "i915gem");
 913                 vm_page_unwire(m, 0);
 914                 vm_page_wakeup(m);
 915                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 916         }
 917         VM_OBJECT_UNLOCK(vm_obj);
 918         drm_free(obj->pages, DRM_I915_GEM);
 919         obj->pages = NULL;
 920         return (-EIO);
 921 }
 922
 923 void
 924 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 925     struct intel_ring_buffer *ring, uint32_t seqno)
 926 {
 927         struct drm_device *dev = obj->base.dev;
 928         struct drm_i915_private *dev_priv = dev->dev_private;
 929         struct drm_i915_fence_reg *reg;
 930
 931         obj->ring = ring;
 932         KASSERT(ring != NULL, ("NULL ring"));
 933
 934         /* Add a reference if we're newly entering the active list. */
 935         if (!obj->active) {
 936                 drm_gem_object_reference(&obj->base);
 937                 obj->active = 1;
 938         }
 939
 940         /* Move from whatever list we were on to the tail of execution. */
 941         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 942         list_move_tail(&obj->ring_list, &ring->active_list);
 943
 944         obj->last_rendering_seqno = seqno;
 945         if (obj->fenced_gpu_access) {
 946                 obj->last_fenced_seqno = seqno;
 947                 obj->last_fenced_ring = ring;
 948
 949                 /* Bump MRU to take account of the delayed flush */
 950                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
 951                         reg = &dev_priv->fence_regs[obj->fence_reg];
 952                         list_move_tail(&reg->lru_list,
 953                                        &dev_priv->mm.fence_list);
 954                 }
 955         }
 956 }
 957
 958 static void
 959 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 960 {
 961         list_del_init(&obj->ring_list);
 962         obj->last_rendering_seqno = 0;
 963         obj->last_fenced_seqno = 0;
 964 }
 965
 966 static void
 967 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 968 {
 969         struct drm_device *dev = obj->base.dev;
 970         struct drm_i915_private *dev_priv = dev->dev_private;
 971
 972         if (obj->pin_count != 0)
 973                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
 974         else
 975                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 976
 977         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
 978         KASSERT(obj->active, ("Object not active"));
 979         obj->ring = NULL;
 980         obj->last_fenced_ring = NULL;
 981
 982         i915_gem_object_move_off_active(obj);
 983         obj->fenced_gpu_access = false;
 984
 985         obj->active = 0;
 986         obj->pending_gpu_write = false;
 987         drm_gem_object_unreference(&obj->base);
 988
 989 #if 1
 990         KIB_NOTYET();
 991 #else
 992         WARN_ON(i915_verify_lists(dev));
 993 #endif
 994 }
 995
 996 static u32
 997 i915_gem_get_seqno(struct drm_device *dev)
 998 {
 999         drm_i915_private_t *dev_priv = dev->dev_private;
1000         u32 seqno = dev_priv->next_seqno;
1001
1002         /* reserve 0 for non-seqno */
1003         if (++dev_priv->next_seqno == 0)
1004                 dev_priv->next_seqno = 1;
1005
1006         return seqno;
1007 }
1008
1009 int
1010 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
1011      struct drm_i915_gem_request *request)
1012 {
1013         drm_i915_private_t *dev_priv;
1014         struct drm_i915_file_private *file_priv;
1015         uint32_t seqno;
1016         u32 request_ring_position;
1017         int was_empty;
1018         int ret;
1019
1020         KASSERT(request != NULL, ("NULL request in add"));
1021         DRM_LOCK_ASSERT(ring->dev);
1022         dev_priv = ring->dev->dev_private;
1023
1024         seqno = i915_gem_next_request_seqno(ring);
1025         request_ring_position = intel_ring_get_tail(ring);
1026
1027         ret = ring->add_request(ring, &seqno);
1028         if (ret != 0)
1029             return ret;
1030
1031         request->seqno = seqno;
1032         request->ring = ring;
1033         request->tail = request_ring_position;
1034         request->emitted_jiffies = ticks;
1035         was_empty = list_empty(&ring->request_list);
1036         list_add_tail(&request->list, &ring->request_list);
1037
1038         if (file != NULL) {
1039                 file_priv = file->driver_priv;
1040
1041                 spin_lock(&file_priv->mm.lock);
1042                 request->file_priv = file_priv;
1043                 list_add_tail(&request->client_list,
1044                     &file_priv->mm.request_list);
1045                 spin_unlock(&file_priv->mm.lock);
1046         }
1047
1048         ring->outstanding_lazy_request = 0;
1049
1050         if (!dev_priv->mm.suspended) {
1051                 if (i915_enable_hangcheck) {
1052                         mod_timer(&dev_priv->hangcheck_timer,
1053                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
1054                 }
1055                 if (was_empty) {
1056                         queue_delayed_work(dev_priv->wq,
1057                                            &dev_priv->mm.retire_work,
1058                                            round_jiffies_up_relative(hz));
1059                         intel_mark_busy(dev_priv->dev);
1060                 }
1061         }
1062         return (0);
1063 }
1064
1065 static inline void
1066 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1067 {
1068         struct drm_i915_file_private *file_priv = request->file_priv;
1069
1070         if (!file_priv)
1071                 return;
1072
1073         DRM_LOCK_ASSERT(request->ring->dev);
1074
1075         spin_lock(&file_priv->mm.lock);
1076         if (request->file_priv != NULL) {
1077                 list_del(&request->client_list);
1078                 request->file_priv = NULL;
1079         }
1080         spin_unlock(&file_priv->mm.lock);
1081 }
1082
1083 static void
1084 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1085     struct intel_ring_buffer *ring)
1086 {
1087
1088         if (ring->dev != NULL)
1089                 DRM_LOCK_ASSERT(ring->dev);
1090
1091         while (!list_empty(&ring->request_list)) {
1092                 struct drm_i915_gem_request *request;
1093
1094                 request = list_first_entry(&ring->request_list,
1095                     struct drm_i915_gem_request, list);
1096
1097                 list_del(&request->list);
1098                 i915_gem_request_remove_from_client(request);
1099                 drm_free(request, DRM_I915_GEM);
1100         }
1101
1102         while (!list_empty(&ring->active_list)) {
1103                 struct drm_i915_gem_object *obj;
1104
1105                 obj = list_first_entry(&ring->active_list,
1106                     struct drm_i915_gem_object, ring_list);
1107
1108                 obj->base.write_domain = 0;
1109                 list_del_init(&obj->gpu_write_list);
1110                 i915_gem_object_move_to_inactive(obj);
1111         }
1112 }
1113
1114 static void
1115 i915_gem_reset_fences(struct drm_device *dev)
1116 {
1117         struct drm_i915_private *dev_priv = dev->dev_private;
1118         int i;
1119
1120         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1121                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1122                 struct drm_i915_gem_object *obj = reg->obj;
1123
1124                 if (!obj)
1125                         continue;
1126
1127                 if (obj->tiling_mode)
1128                         i915_gem_release_mmap(obj);
1129
1130                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1131                 reg->obj->fenced_gpu_access = false;
1132                 reg->obj->last_fenced_seqno = 0;
1133                 reg->obj->last_fenced_ring = NULL;
1134                 i915_gem_clear_fence_reg(dev, reg);
1135         }
1136 }
1137
1138 void i915_gem_reset(struct drm_device *dev)
1139 {
1140         struct drm_i915_private *dev_priv = dev->dev_private;
1141         struct drm_i915_gem_object *obj;
1142         int i;
1143
1144         for (i = 0; i < I915_NUM_RINGS; i++)
1145                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1146
1147         /* Remove anything from the flushing lists. The GPU cache is likely
1148          * to be lost on reset along with the data, so simply move the
1149          * lost bo to the inactive list.
1150          */
1151         while (!list_empty(&dev_priv->mm.flushing_list)) {
1152                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1153                                       struct drm_i915_gem_object,
1154                                       mm_list);
1155
1156                 obj->base.write_domain = 0;
1157                 list_del_init(&obj->gpu_write_list);
1158                 i915_gem_object_move_to_inactive(obj);
1159         }
1160
1161         /* Move everything out of the GPU domains to ensure we do any
1162          * necessary invalidation upon reuse.
1163          */
1164         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
1165                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1166         }
1167
1168         /* The fence registers are invalidated so clear them out */
1169         i915_gem_reset_fences(dev);
1170 }
1171
1172 static void
1173 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1174 {
1175         struct drm_device *dev = obj->base.dev;
1176         drm_i915_private_t *dev_priv = dev->dev_private;
1177
1178         KASSERT(obj->active, ("Object not active"));
1179         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1180
1181         i915_gem_object_move_off_active(obj);
1182 }
1183
1184 /**
1185  * This function clears the request list as sequence numbers are passed.
1186  */
1187 void
1188 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1189 {
1190         uint32_t seqno;
1191
1192         if (list_empty(&ring->request_list))
1193                 return;
1194
1195         seqno = ring->get_seqno(ring, true);
1196
1197         while (!list_empty(&ring->request_list)) {
1198                 struct drm_i915_gem_request *request;
1199
1200                 request = list_first_entry(&ring->request_list,
1201                                            struct drm_i915_gem_request,
1202                                            list);
1203
1204                 if (!i915_seqno_passed(seqno, request->seqno))
1205                         break;
1206
1207                 /* We know the GPU must have read the request to have
1208                  * sent us the seqno + interrupt, so use the position
1209                  * of tail of the request to update the last known position
1210                  * of the GPU head.
1211                  */
1212                 ring->last_retired_head = request->tail;
1213
1214                 list_del(&request->list);
1215                 i915_gem_request_remove_from_client(request);
1216                 drm_free(request, DRM_I915_GEM);
1217         }
1218
1219         /* Move any buffers on the active list that are no longer referenced
1220          * by the ringbuffer to the flushing/inactive lists as appropriate.
1221          */
1222         while (!list_empty(&ring->active_list)) {
1223                 struct drm_i915_gem_object *obj;
1224
1225                 obj = list_first_entry(&ring->active_list,
1226                                       struct drm_i915_gem_object,
1227                                       ring_list);
1228
1229                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1230                         break;
1231
1232                 if (obj->base.write_domain != 0)
1233                         i915_gem_object_move_to_flushing(obj);
1234                 else
1235                         i915_gem_object_move_to_inactive(obj);
1236         }
1237
1238         if (unlikely(ring->trace_irq_seqno &&
1239                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1240                 ring->irq_put(ring);
1241                 ring->trace_irq_seqno = 0;
1242         }
1243
1244 }
1245
1246 static void
1247 i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
1248
1249 void
1250 i915_gem_retire_requests(struct drm_device *dev)
1251 {
1252         drm_i915_private_t *dev_priv = dev->dev_private;
1253         struct drm_i915_gem_object *obj, *next;
1254         int i;
1255
1256         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1257                 list_for_each_entry_safe(obj, next,
1258                     &dev_priv->mm.deferred_free_list, mm_list)
1259                         i915_gem_free_object_tail(obj);
1260         }
1261
1262         for (i = 0; i < I915_NUM_RINGS; i++)
1263                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1264 }
1265
1266 static void
1267 i915_gem_retire_work_handler(struct work_struct *work)
1268 {
1269         drm_i915_private_t *dev_priv;
1270         struct drm_device *dev;
1271         struct intel_ring_buffer *ring;
1272         bool idle;
1273         int i;
1274
1275         dev_priv = container_of(work, drm_i915_private_t,
1276                                 mm.retire_work.work);
1277         dev = dev_priv->dev;
1278
1279         /* Come back later if the device is busy... */
1280         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
1281                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1282                                    round_jiffies_up_relative(hz));
1283                 return;
1284         }
1285
1286         i915_gem_retire_requests(dev);
1287
1288         /* Send a periodic flush down the ring so we don't hold onto GEM
1289          * objects indefinitely.
1290          */
1291         idle = true;
1292         for_each_ring(ring, dev_priv, i) {
1293                 if (ring->gpu_caches_dirty)
1294                         i915_add_request(ring, NULL, NULL);
1295
1296                 idle &= list_empty(&ring->request_list);
1297         }
1298
1299         if (!dev_priv->mm.suspended && !idle)
1300                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1301                                    round_jiffies_up_relative(hz));
1302         if (idle)
1303                 intel_mark_idle(dev);
1304
1305         DRM_UNLOCK(dev);
1306 }
1307
1308 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1309 {
1310         u32 old_write_domain, old_read_domains;
1311
1312         /* Act a barrier for all accesses through the GTT */
1313         cpu_mfence();
1314
1315         /* Force a pagefault for domain tracking on next user access */
1316         i915_gem_release_mmap(obj);
1317
1318         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1319                 return;
1320
1321         old_read_domains = obj->base.read_domains;
1322         old_write_domain = obj->base.write_domain;
1323
1324         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1325         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1326
1327 }
1328
1329 int
1330 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1331 {
1332         drm_i915_private_t *dev_priv;
1333         int ret;
1334
1335         dev_priv = obj->base.dev->dev_private;
1336         ret = 0;
1337         if (obj->gtt_space == NULL)
1338                 return (0);
1339         if (obj->pin_count != 0) {
1340                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1341                 return (-EINVAL);
1342         }
1343
1344         ret = i915_gem_object_finish_gpu(obj);
1345         if (ret == -ERESTART || ret == -EINTR)
1346                 return (ret);
1347
1348         i915_gem_object_finish_gtt(obj);
1349
1350         if (ret == 0)
1351                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1352         if (ret == -ERESTART || ret == -EINTR)
1353                 return (ret);
1354         if (ret != 0) {
1355                 i915_gem_clflush_object(obj);
1356                 obj->base.read_domains = obj->base.write_domain =
1357                     I915_GEM_DOMAIN_CPU;
1358         }
1359
1360         ret = i915_gem_object_put_fence(obj);
1361         if (ret == -ERESTART)
1362                 return (ret);
1363
1364         i915_gem_gtt_unbind_object(obj);
1365         if (obj->has_aliasing_ppgtt_mapping) {
1366                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
1367                 obj->has_aliasing_ppgtt_mapping = 0;
1368         }
1369         i915_gem_object_put_pages_gtt(obj);
1370
1371         list_del_init(&obj->gtt_list);
1372         list_del_init(&obj->mm_list);
1373         obj->map_and_fenceable = true;
1374
1375         drm_mm_put_block(obj->gtt_space);
1376         obj->gtt_space = NULL;
1377         obj->gtt_offset = 0;
1378
1379         if (i915_gem_object_is_purgeable(obj))
1380                 i915_gem_object_truncate(obj);
1381
1382         return (ret);
1383 }
1384
1385 int i915_gpu_idle(struct drm_device *dev)
1386 {
1387         drm_i915_private_t *dev_priv = dev->dev_private;
1388         struct intel_ring_buffer *ring;
1389         int ret, i;
1390
1391         /* Flush everything onto the inactive list. */
1392         for_each_ring(ring, dev_priv, i) {
1393                 ret = intel_ring_idle(ring);
1394                 if (ret)
1395                         return ret;
1396         }
1397
1398         return 0;
1399 }
1400
1401 static int
1402 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
1403     struct intel_ring_buffer *pipelined)
1404 {
1405         struct drm_device *dev = obj->base.dev;
1406         drm_i915_private_t *dev_priv = dev->dev_private;
1407         u32 size = obj->gtt_space->size;
1408         int regnum = obj->fence_reg;
1409         uint64_t val;
1410
1411         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1412                          0xfffff000) << 32;
1413         val |= obj->gtt_offset & 0xfffff000;
1414         val |= (uint64_t)((obj->stride / 128) - 1) <<
1415                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
1416
1417         if (obj->tiling_mode == I915_TILING_Y)
1418                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1419         val |= I965_FENCE_REG_VALID;
1420
1421         if (pipelined) {
1422                 int ret = intel_ring_begin(pipelined, 6);
1423                 if (ret)
1424                         return ret;
1425
1426                 intel_ring_emit(pipelined, MI_NOOP);
1427                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1428                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
1429                 intel_ring_emit(pipelined, (u32)val);
1430                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
1431                 intel_ring_emit(pipelined, (u32)(val >> 32));
1432                 intel_ring_advance(pipelined);
1433         } else
1434                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
1435
1436         return 0;
1437 }
1438
1439 static int
1440 i965_write_fence_reg(struct drm_i915_gem_object *obj,
1441     struct intel_ring_buffer *pipelined)
1442 {
1443         struct drm_device *dev = obj->base.dev;
1444         drm_i915_private_t *dev_priv = dev->dev_private;
1445         u32 size = obj->gtt_space->size;
1446         int regnum = obj->fence_reg;
1447         uint64_t val;
1448
1449         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1450                     0xfffff000) << 32;
1451         val |= obj->gtt_offset & 0xfffff000;
1452         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1453         if (obj->tiling_mode == I915_TILING_Y)
1454                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1455         val |= I965_FENCE_REG_VALID;
1456
1457         if (pipelined) {
1458                 int ret = intel_ring_begin(pipelined, 6);
1459                 if (ret)
1460                         return ret;
1461
1462                 intel_ring_emit(pipelined, MI_NOOP);
1463                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1464                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
1465                 intel_ring_emit(pipelined, (u32)val);
1466                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
1467                 intel_ring_emit(pipelined, (u32)(val >> 32));
1468                 intel_ring_advance(pipelined);
1469         } else
1470                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
1471
1472         return 0;
1473 }
1474
1475 static int
1476 i915_write_fence_reg(struct drm_i915_gem_object *obj,
1477     struct intel_ring_buffer *pipelined)
1478 {
1479         struct drm_device *dev = obj->base.dev;
1480         drm_i915_private_t *dev_priv = dev->dev_private;
1481         u32 size = obj->gtt_space->size;
1482         u32 fence_reg, val, pitch_val;
1483         int tile_width;
1484
1485         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
1486             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1487                 kprintf(
1488 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1489                  obj->gtt_offset, obj->map_and_fenceable, size);
1490                 return -EINVAL;
1491         }
1492
1493         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1494                 tile_width = 128;
1495         else
1496                 tile_width = 512;
1497
1498         /* Note: pitch better be a power of two tile widths */
1499         pitch_val = obj->stride / tile_width;
1500         pitch_val = ffs(pitch_val) - 1;
1501
1502         val = obj->gtt_offset;
1503         if (obj->tiling_mode == I915_TILING_Y)
1504                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1505         val |= I915_FENCE_SIZE_BITS(size);
1506         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1507         val |= I830_FENCE_REG_VALID;
1508
1509         fence_reg = obj->fence_reg;
1510         if (fence_reg < 8)
1511                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
1512         else
1513                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
1514
1515         if (pipelined) {
1516                 int ret = intel_ring_begin(pipelined, 4);
1517                 if (ret)
1518                         return ret;
1519
1520                 intel_ring_emit(pipelined, MI_NOOP);
1521                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1522                 intel_ring_emit(pipelined, fence_reg);
1523                 intel_ring_emit(pipelined, val);
1524                 intel_ring_advance(pipelined);
1525         } else
1526                 I915_WRITE(fence_reg, val);
1527
1528         return 0;
1529 }
1530
1531 static int
1532 i830_write_fence_reg(struct drm_i915_gem_object *obj,
1533     struct intel_ring_buffer *pipelined)
1534 {
1535         struct drm_device *dev = obj->base.dev;
1536         drm_i915_private_t *dev_priv = dev->dev_private;
1537         u32 size = obj->gtt_space->size;
1538         int regnum = obj->fence_reg;
1539         uint32_t val;
1540         uint32_t pitch_val;
1541
1542         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
1543             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1544                 kprintf(
1545 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1546                     obj->gtt_offset, size);
1547                 return -EINVAL;
1548         }
1549
1550         pitch_val = obj->stride / 128;
1551         pitch_val = ffs(pitch_val) - 1;
1552
1553         val = obj->gtt_offset;
1554         if (obj->tiling_mode == I915_TILING_Y)
1555                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1556         val |= I830_FENCE_SIZE_BITS(size);
1557         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1558         val |= I830_FENCE_REG_VALID;
1559
1560         if (pipelined) {
1561                 int ret = intel_ring_begin(pipelined, 4);
1562                 if (ret)
1563                         return ret;
1564
1565                 intel_ring_emit(pipelined, MI_NOOP);
1566                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1567                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
1568                 intel_ring_emit(pipelined, val);
1569                 intel_ring_advance(pipelined);
1570         } else
1571                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
1572
1573         return 0;
1574 }
1575
1576 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
1577 {
1578         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
1579 }
1580
1581 static int
1582 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
1583     struct intel_ring_buffer *pipelined)
1584 {
1585         int ret;
1586
1587         if (obj->fenced_gpu_access) {
1588                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1589                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
1590                             obj->base.write_domain);
1591                         if (ret)
1592                                 return ret;
1593                 }
1594
1595                 obj->fenced_gpu_access = false;
1596         }
1597
1598         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
1599                 if (!ring_passed_seqno(obj->last_fenced_ring,
1600                                        obj->last_fenced_seqno)) {
1601                         ret = i915_wait_seqno(obj->last_fenced_ring,
1602                                                 obj->last_fenced_seqno);
1603                         if (ret)
1604                                 return ret;
1605                 }
1606
1607                 obj->last_fenced_seqno = 0;
1608                 obj->last_fenced_ring = NULL;
1609         }
1610
1611         /* Ensure that all CPU reads are completed before installing a fence
1612          * and all writes before removing the fence.
1613          */
1614         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
1615                 cpu_mfence();
1616
1617         return 0;
1618 }
1619
1620 int
1621 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
1622 {
1623         int ret;
1624
1625         if (obj->tiling_mode)
1626                 i915_gem_release_mmap(obj);
1627
1628         ret = i915_gem_object_flush_fence(obj, NULL);
1629         if (ret)
1630                 return ret;
1631
1632         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1633                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1634
1635                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
1636                         kprintf("%s: pin_count %d\n", __func__,
1637                             dev_priv->fence_regs[obj->fence_reg].pin_count);
1638                 i915_gem_clear_fence_reg(obj->base.dev,
1639                                          &dev_priv->fence_regs[obj->fence_reg]);
1640
1641                 obj->fence_reg = I915_FENCE_REG_NONE;
1642         }
1643
1644         return 0;
1645 }
1646
1647 static struct drm_i915_fence_reg *
1648 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
1649 {
1650         struct drm_i915_private *dev_priv = dev->dev_private;
1651         struct drm_i915_fence_reg *reg, *first, *avail;
1652         int i;
1653
1654         /* First try to find a free reg */
1655         avail = NULL;
1656         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
1657                 reg = &dev_priv->fence_regs[i];
1658                 if (!reg->obj)
1659                         return reg;
1660
1661                 if (!reg->pin_count)
1662                         avail = reg;
1663         }
1664
1665         if (avail == NULL)
1666                 return NULL;
1667
1668         /* None available, try to steal one or wait for a user to finish */
1669         avail = first = NULL;
1670         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1671                 if (reg->pin_count)
1672                         continue;
1673
1674                 if (first == NULL)
1675                         first = reg;
1676
1677                 if (!pipelined ||
1678                     !reg->obj->last_fenced_ring ||
1679                     reg->obj->last_fenced_ring == pipelined) {
1680                         avail = reg;
1681                         break;
1682                 }
1683         }
1684
1685         if (avail == NULL)
1686                 avail = first;
1687
1688         return avail;
1689 }
1690
1691 int
1692 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1693     struct intel_ring_buffer *pipelined)
1694 {
1695         struct drm_device *dev = obj->base.dev;
1696         struct drm_i915_private *dev_priv = dev->dev_private;
1697         struct drm_i915_fence_reg *reg;
1698         int ret;
1699
1700         pipelined = NULL;
1701         ret = 0;
1702
1703         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1704                 reg = &dev_priv->fence_regs[obj->fence_reg];
1705                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1706
1707                 if (obj->tiling_changed) {
1708                         ret = i915_gem_object_flush_fence(obj, pipelined);
1709                         if (ret)
1710                                 return ret;
1711
1712                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1713                                 pipelined = NULL;
1714
1715                         if (pipelined) {
1716                                 reg->setup_seqno =
1717                                         i915_gem_next_request_seqno(pipelined);
1718                                 obj->last_fenced_seqno = reg->setup_seqno;
1719                                 obj->last_fenced_ring = pipelined;
1720                         }
1721
1722                         goto update;
1723                 }
1724
1725                 if (!pipelined) {
1726                         if (reg->setup_seqno) {
1727                                 if (!ring_passed_seqno(obj->last_fenced_ring,
1728                                     reg->setup_seqno)) {
1729                                         ret = i915_wait_seqno(
1730                                             obj->last_fenced_ring,
1731                                             reg->setup_seqno);
1732                                         if (ret)
1733                                                 return ret;
1734                                 }
1735
1736                                 reg->setup_seqno = 0;
1737                         }
1738                 } else if (obj->last_fenced_ring &&
1739                            obj->last_fenced_ring != pipelined) {
1740                         ret = i915_gem_object_flush_fence(obj, pipelined);
1741                         if (ret)
1742                                 return ret;
1743                 }
1744
1745                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1746                         pipelined = NULL;
1747                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
1748
1749                 if (obj->tiling_changed) {
1750                         if (pipelined) {
1751                                 reg->setup_seqno =
1752                                         i915_gem_next_request_seqno(pipelined);
1753                                 obj->last_fenced_seqno = reg->setup_seqno;
1754                                 obj->last_fenced_ring = pipelined;
1755                         }
1756                         goto update;
1757                 }
1758
1759                 return 0;
1760         }
1761
1762         reg = i915_find_fence_reg(dev, pipelined);
1763         if (reg == NULL)
1764                 return -EDEADLK;
1765
1766         ret = i915_gem_object_flush_fence(obj, pipelined);
1767         if (ret)
1768                 return ret;
1769
1770         if (reg->obj) {
1771                 struct drm_i915_gem_object *old = reg->obj;
1772
1773                 drm_gem_object_reference(&old->base);
1774
1775                 if (old->tiling_mode)
1776                         i915_gem_release_mmap(old);
1777
1778                 ret = i915_gem_object_flush_fence(old, pipelined);
1779                 if (ret) {
1780                         drm_gem_object_unreference(&old->base);
1781                         return ret;
1782                 }
1783
1784                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
1785                         pipelined = NULL;
1786
1787                 old->fence_reg = I915_FENCE_REG_NONE;
1788                 old->last_fenced_ring = pipelined;
1789                 old->last_fenced_seqno =
1790                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1791
1792                 drm_gem_object_unreference(&old->base);
1793         } else if (obj->last_fenced_seqno == 0)
1794                 pipelined = NULL;
1795
1796         reg->obj = obj;
1797         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1798         obj->fence_reg = reg - dev_priv->fence_regs;
1799         obj->last_fenced_ring = pipelined;
1800
1801         reg->setup_seqno =
1802                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1803         obj->last_fenced_seqno = reg->setup_seqno;
1804
1805 update:
1806         obj->tiling_changed = false;
1807         switch (INTEL_INFO(dev)->gen) {
1808         case 7:
1809         case 6:
1810                 ret = sandybridge_write_fence_reg(obj, pipelined);
1811                 break;
1812         case 5:
1813         case 4:
1814                 ret = i965_write_fence_reg(obj, pipelined);
1815                 break;
1816         case 3:
1817                 ret = i915_write_fence_reg(obj, pipelined);
1818                 break;
1819         case 2:
1820                 ret = i830_write_fence_reg(obj, pipelined);
1821                 break;
1822         }
1823
1824         return ret;
1825 }
1826
1827 static int
1828 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1829     unsigned alignment, bool map_and_fenceable)
1830 {
1831         struct drm_device *dev;
1832         struct drm_i915_private *dev_priv;
1833         struct drm_mm_node *free_space;
1834         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1835         bool mappable, fenceable;
1836         int ret;
1837
1838         dev = obj->base.dev;
1839         dev_priv = dev->dev_private;
1840
1841         if (obj->madv != I915_MADV_WILLNEED) {
1842                 DRM_ERROR("Attempting to bind a purgeable object\n");
1843                 return (-EINVAL);
1844         }
1845
1846         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1847             obj->tiling_mode);
1848         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1849             obj->tiling_mode);
1850         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1851             obj->base.size, obj->tiling_mode);
1852         if (alignment == 0)
1853                 alignment = map_and_fenceable ? fence_alignment :
1854                     unfenced_alignment;
1855         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1856                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1857                 return (-EINVAL);
1858         }
1859
1860         size = map_and_fenceable ? fence_size : obj->base.size;
1861
1862         /* If the object is bigger than the entire aperture, reject it early
1863          * before evicting everything in a vain attempt to find space.
1864          */
1865         if (obj->base.size > (map_and_fenceable ?
1866             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1867                 DRM_ERROR(
1868 "Attempting to bind an object larger than the aperture\n");
1869                 return (-E2BIG);
1870         }
1871
1872  search_free:
1873         if (map_and_fenceable)
1874                 free_space = drm_mm_search_free_in_range(
1875                     &dev_priv->mm.gtt_space, size, alignment, 0,
1876                     dev_priv->mm.gtt_mappable_end, 0);
1877         else
1878                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1879                     size, alignment, 0);
1880         if (free_space != NULL) {
1881                 int color = 0;
1882                 if (map_and_fenceable)
1883                         obj->gtt_space = drm_mm_get_block_range_generic(
1884                             free_space, size, alignment, color, 0,
1885                             dev_priv->mm.gtt_mappable_end, 1);
1886                 else
1887                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1888                             size, alignment, color, 1);
1889         }
1890         if (obj->gtt_space == NULL) {
1891                 ret = i915_gem_evict_something(dev, size, alignment,
1892                     map_and_fenceable);
1893                 if (ret != 0)
1894                         return (ret);
1895                 goto search_free;
1896         }
1897
1898         /*
1899          * NOTE: i915_gem_object_get_pages_gtt() cannot
1900          *       return ENOMEM, since we used VM_ALLOC_RETRY.
1901          */
1902         ret = i915_gem_object_get_pages_gtt(obj, 0);
1903         if (ret != 0) {
1904                 drm_mm_put_block(obj->gtt_space);
1905                 obj->gtt_space = NULL;
1906                 return (ret);
1907         }
1908
1909         i915_gem_gtt_bind_object(obj, obj->cache_level);
1910         if (ret != 0) {
1911                 i915_gem_object_put_pages_gtt(obj);
1912                 drm_mm_put_block(obj->gtt_space);
1913                 obj->gtt_space = NULL;
1914                 if (i915_gem_evict_everything(dev, false))
1915                         return (ret);
1916                 goto search_free;
1917         }
1918
1919         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1920         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1921
1922         obj->gtt_offset = obj->gtt_space->start;
1923
1924         fenceable =
1925                 obj->gtt_space->size == fence_size &&
1926                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1927
1928         mappable =
1929                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1930         obj->map_and_fenceable = mappable && fenceable;
1931
1932         return (0);
1933 }
1934
1935 void
1936 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1937 {
1938
1939         /* If we don't have a page list set up, then we're not pinned
1940          * to GPU, and we can ignore the cache flush because it'll happen
1941          * again at bind time.
1942          */
1943         if (obj->pages == NULL)
1944                 return;
1945
1946         /* If the GPU is snooping the contents of the CPU cache,
1947          * we do not need to manually clear the CPU cache lines.  However,
1948          * the caches are only snooped when the render cache is
1949          * flushed/invalidated.  As we always have to emit invalidations
1950          * and flushes when moving into and out of the RENDER domain, correct
1951          * snooping behaviour occurs naturally as the result of our domain
1952          * tracking.
1953          */
1954         if (obj->cache_level != I915_CACHE_NONE)
1955                 return;
1956
1957         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
1958 }
1959
1960 /** Flushes the GTT write domain for the object if it's dirty. */
1961 static void
1962 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
1963 {
1964         uint32_t old_write_domain;
1965
1966         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
1967                 return;
1968
1969         /* No actual flushing is required for the GTT write domain.  Writes
1970          * to it immediately go to main memory as far as we know, so there's
1971          * no chipset flush.  It also doesn't land in render cache.
1972          *
1973          * However, we do have to enforce the order so that all writes through
1974          * the GTT land before any writes to the device, such as updates to
1975          * the GATT itself.
1976          */
1977         cpu_sfence();
1978
1979         old_write_domain = obj->base.write_domain;
1980         obj->base.write_domain = 0;
1981 }
1982
1983 /** Flushes the CPU write domain for the object if it's dirty. */
1984 static void
1985 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
1986 {
1987         uint32_t old_write_domain;
1988
1989         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
1990                 return;
1991
1992         i915_gem_clflush_object(obj);
1993         intel_gtt_chipset_flush();
1994         old_write_domain = obj->base.write_domain;
1995         obj->base.write_domain = 0;
1996 }
1997
1998 static int
1999 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2000 {
2001
2002         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2003                 return (0);
2004         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
2005 }
2006
2007 /**
2008  * Moves a single object to the GTT read, and possibly write domain.
2009  *
2010  * This function returns when the move is complete, including waiting on
2011  * flushes to occur.
2012  */
2013 int
2014 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2015 {
2016         uint32_t old_write_domain, old_read_domains;
2017         int ret;
2018
2019         if (obj->gtt_space == NULL)
2020                 return (-EINVAL);
2021
2022         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2023                 return 0;
2024
2025         ret = i915_gem_object_flush_gpu_write_domain(obj);
2026         if (ret != 0)
2027                 return (ret);
2028
2029         if (obj->pending_gpu_write || write) {
2030                 ret = i915_gem_object_wait_rendering(obj);
2031                 if (ret != 0)
2032                         return (ret);
2033         }
2034
2035         i915_gem_object_flush_cpu_write_domain(obj);
2036
2037         old_write_domain = obj->base.write_domain;
2038         old_read_domains = obj->base.read_domains;
2039
2040         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2041             ("In GTT write domain"));
2042         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2043         if (write) {
2044                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2045                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2046                 obj->dirty = 1;
2047         }
2048
2049         return (0);
2050 }
2051
2052 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2053                                     enum i915_cache_level cache_level)
2054 {
2055         struct drm_device *dev = obj->base.dev;
2056         drm_i915_private_t *dev_priv = dev->dev_private;
2057         int ret;
2058
2059         if (obj->cache_level == cache_level)
2060                 return 0;
2061
2062         if (obj->pin_count) {
2063                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2064                 return -EBUSY;
2065         }
2066
2067         if (obj->gtt_space) {
2068                 ret = i915_gem_object_finish_gpu(obj);
2069                 if (ret != 0)
2070                         return (ret);
2071
2072                 i915_gem_object_finish_gtt(obj);
2073
2074                 /* Before SandyBridge, you could not use tiling or fence
2075                  * registers with snooped memory, so relinquish any fences
2076                  * currently pointing to our region in the aperture.
2077                  */
2078                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2079                         ret = i915_gem_object_put_fence(obj);
2080                         if (ret)
2081                                 return ret;
2082                 }
2083
2084                 if (obj->has_global_gtt_mapping)
2085                         i915_gem_gtt_bind_object(obj, cache_level);
2086                 if (obj->has_aliasing_ppgtt_mapping)
2087                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2088                                                obj, cache_level);
2089         }
2090
2091         if (cache_level == I915_CACHE_NONE) {
2092                 u32 old_read_domains, old_write_domain;
2093
2094                 /* If we're coming from LLC cached, then we haven't
2095                  * actually been tracking whether the data is in the
2096                  * CPU cache or not, since we only allow one bit set
2097                  * in obj->write_domain and have been skipping the clflushes.
2098                  * Just set it to the CPU cache for now.
2099                  */
2100                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
2101                     ("obj %p in CPU write domain", obj));
2102                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
2103                     ("obj %p in CPU read domain", obj));
2104
2105                 old_read_domains = obj->base.read_domains;
2106                 old_write_domain = obj->base.write_domain;
2107
2108                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2109                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2110
2111         }
2112
2113         obj->cache_level = cache_level;
2114         return 0;
2115 }
2116
2117 /*
2118  * Prepare buffer for display plane (scanout, cursors, etc).
2119  * Can be called from an uninterruptible phase (modesetting) and allows
2120  * any flushes to be pipelined (for pageflips).
2121  */
2122 int
2123 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2124                                      u32 alignment,
2125                                      struct intel_ring_buffer *pipelined)
2126 {
2127         u32 old_read_domains, old_write_domain;
2128         int ret;
2129
2130         ret = i915_gem_object_flush_gpu_write_domain(obj);
2131         if (ret != 0)
2132                 return (ret);
2133
2134         if (pipelined != obj->ring) {
2135                 ret = i915_gem_object_wait_rendering(obj);
2136                 if (ret == -ERESTART || ret == -EINTR)
2137                         return (ret);
2138         }
2139
2140         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2141         if (ret != 0)
2142                 return (ret);
2143
2144         ret = i915_gem_object_pin(obj, alignment, true);
2145         if (ret != 0)
2146                 return (ret);
2147
2148         i915_gem_object_flush_cpu_write_domain(obj);
2149
2150         old_write_domain = obj->base.write_domain;
2151         old_read_domains = obj->base.read_domains;
2152
2153         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2154             ("obj %p in GTT write domain", obj));
2155         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2156
2157         return (0);
2158 }
2159
2160 int
2161 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2162 {
2163         int ret;
2164
2165         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2166                 return (0);
2167
2168         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2169                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2170                 if (ret != 0)
2171                         return (ret);
2172         }
2173
2174         ret = i915_gem_object_wait_rendering(obj);
2175         if (ret != 0)
2176                 return (ret);
2177
2178         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2179
2180         return (0);
2181 }
2182
2183 /**
2184  * Moves a single object to the CPU read, and possibly write domain.
2185  *
2186  * This function returns when the move is complete, including waiting on
2187  * flushes to occur.
2188  */
2189 int
2190 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2191 {
2192         uint32_t old_write_domain, old_read_domains;
2193         int ret;
2194
2195         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2196                 return 0;
2197
2198         ret = i915_gem_object_flush_gpu_write_domain(obj);
2199         if (ret != 0)
2200                 return (ret);
2201
2202         ret = i915_gem_object_wait_rendering(obj);
2203         if (ret)
2204                 return ret;
2205
2206         i915_gem_object_flush_gtt_write_domain(obj);
2207
2208         old_write_domain = obj->base.write_domain;
2209         old_read_domains = obj->base.read_domains;
2210
2211         /* Flush the CPU cache if it's still invalid. */
2212         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2213                 i915_gem_clflush_object(obj);
2214
2215                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2216         }
2217
2218         /* It should now be out of any other write domains, and we can update
2219          * the domain values for our changes.
2220          */
2221         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2222
2223         /* If we're writing through the CPU, then the GPU read domains will
2224          * need to be invalidated at next use.
2225          */
2226         if (write) {
2227                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2228                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2229         }
2230
2231         return 0;
2232 }
2233
2234 /* Throttle our rendering by waiting until the ring has completed our requests
2235  * emitted over 20 msec ago.
2236  *
2237  * Note that if we were to use the current jiffies each time around the loop,
2238  * we wouldn't escape the function with any frames outstanding if the time to
2239  * render a frame was over 20ms.
2240  *
2241  * This should get us reasonable parallelism between CPU and GPU but also
2242  * relatively low latency when blocking on a particular request to finish.
2243  */
2244 static int
2245 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2246 {
2247         struct drm_i915_private *dev_priv = dev->dev_private;
2248         struct drm_i915_file_private *file_priv = file->driver_priv;
2249         unsigned long recent_enough = ticks - (20 * hz / 1000);
2250         struct drm_i915_gem_request *request;
2251         struct intel_ring_buffer *ring = NULL;
2252         u32 seqno = 0;
2253         int ret;
2254
2255         dev_priv = dev->dev_private;
2256         if (atomic_read(&dev_priv->mm.wedged))
2257                 return -EIO;
2258
2259         recent_enough = ticks - (20 * hz / 1000);
2260         ring = NULL;
2261         seqno = 0;
2262
2263         spin_lock(&file_priv->mm.lock);
2264         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2265                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2266                         break;
2267
2268                 ring = request->ring;
2269                 seqno = request->seqno;
2270         }
2271         spin_unlock(&file_priv->mm.lock);
2272
2273         if (seqno == 0)
2274                 return 0;
2275
2276         ret = 0;
2277         lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2278         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
2279                 if (ring->irq_get(ring)) {
2280                         while (ret == 0 &&
2281                             !(i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2282                             atomic_read(&dev_priv->mm.wedged)))
2283                                 ret = -lksleep(ring, &ring->irq_lock, PCATCH,
2284                                     "915thr", 1*hz);
2285                         ring->irq_put(ring);
2286                         if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
2287                                 ret = -EIO;
2288                 } else if (_intel_wait_for(dev,
2289                     i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2290                     atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
2291                         ret = -EBUSY;
2292                 }
2293         }
2294         lockmgr(&ring->irq_lock, LK_RELEASE);
2295
2296         if (ret == 0)
2297                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2298
2299         return ret;
2300 }
2301
2302 int
2303 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
2304      bool map_and_fenceable)
2305 {
2306         struct drm_device *dev;
2307         struct drm_i915_private *dev_priv;
2308         int ret;
2309
2310         dev = obj->base.dev;
2311         dev_priv = dev->dev_private;
2312
2313         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
2314             ("Max pin count"));
2315
2316         if (obj->gtt_space != NULL) {
2317                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
2318                     (map_and_fenceable && !obj->map_and_fenceable)) {
2319                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
2320                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2321                              " obj->map_and_fenceable=%d\n",
2322                              obj->gtt_offset, alignment,
2323                              map_and_fenceable,
2324                              obj->map_and_fenceable);
2325                         ret = i915_gem_object_unbind(obj);
2326                         if (ret != 0)
2327                                 return (ret);
2328                 }
2329         }
2330
2331         if (obj->gtt_space == NULL) {
2332                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
2333                     map_and_fenceable);
2334                 if (ret)
2335                         return (ret);
2336         }
2337
2338         if (obj->pin_count++ == 0 && !obj->active)
2339                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2340         obj->pin_mappable |= map_and_fenceable;
2341
2342 #if 1
2343         KIB_NOTYET();
2344 #else
2345         WARN_ON(i915_verify_lists(dev));
2346 #endif
2347         return (0);
2348 }
2349
2350 void
2351 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
2352 {
2353         struct drm_device *dev;
2354         drm_i915_private_t *dev_priv;
2355
2356         dev = obj->base.dev;
2357         dev_priv = dev->dev_private;
2358
2359 #if 1
2360         KIB_NOTYET();
2361 #else
2362         WARN_ON(i915_verify_lists(dev));
2363 #endif
2364
2365         KASSERT(obj->pin_count != 0, ("zero pin count"));
2366         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
2367
2368         if (--obj->pin_count == 0) {
2369                 if (!obj->active)
2370                         list_move_tail(&obj->mm_list,
2371                             &dev_priv->mm.inactive_list);
2372                 obj->pin_mappable = false;
2373         }
2374 #if 1
2375         KIB_NOTYET();
2376 #else
2377         WARN_ON(i915_verify_lists(dev));
2378 #endif
2379 }
2380
2381 int
2382 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2383     struct drm_file *file)
2384 {
2385         struct drm_i915_gem_pin *args;
2386         struct drm_i915_gem_object *obj;
2387         struct drm_gem_object *gobj;
2388         int ret;
2389
2390         args = data;
2391
2392         ret = i915_mutex_lock_interruptible(dev);
2393         if (ret != 0)
2394                 return ret;
2395
2396         gobj = drm_gem_object_lookup(dev, file, args->handle);
2397         if (gobj == NULL) {
2398                 ret = -ENOENT;
2399                 goto unlock;
2400         }
2401         obj = to_intel_bo(gobj);
2402
2403         if (obj->madv != I915_MADV_WILLNEED) {
2404                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2405                 ret = -EINVAL;
2406                 goto out;
2407         }
2408
2409         if (obj->pin_filp != NULL && obj->pin_filp != file) {
2410                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2411                     args->handle);
2412                 ret = -EINVAL;
2413                 goto out;
2414         }
2415
2416         obj->user_pin_count++;
2417         obj->pin_filp = file;
2418         if (obj->user_pin_count == 1) {
2419                 ret = i915_gem_object_pin(obj, args->alignment, true);
2420                 if (ret != 0)
2421                         goto out;
2422         }
2423
2424         /* XXX - flush the CPU caches for pinned objects
2425          * as the X server doesn't manage domains yet
2426          */
2427         i915_gem_object_flush_cpu_write_domain(obj);
2428         args->offset = obj->gtt_offset;
2429 out:
2430         drm_gem_object_unreference(&obj->base);
2431 unlock:
2432         DRM_UNLOCK(dev);
2433         return (ret);
2434 }
2435
2436 int
2437 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2438     struct drm_file *file)
2439 {
2440         struct drm_i915_gem_pin *args;
2441         struct drm_i915_gem_object *obj;
2442         int ret;
2443
2444         args = data;
2445         ret = i915_mutex_lock_interruptible(dev);
2446         if (ret != 0)
2447                 return (ret);
2448
2449         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2450         if (&obj->base == NULL) {
2451                 ret = -ENOENT;
2452                 goto unlock;
2453         }
2454
2455         if (obj->pin_filp != file) {
2456                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2457                     args->handle);
2458                 ret = -EINVAL;
2459                 goto out;
2460         }
2461         obj->user_pin_count--;
2462         if (obj->user_pin_count == 0) {
2463                 obj->pin_filp = NULL;
2464                 i915_gem_object_unpin(obj);
2465         }
2466
2467 out:
2468         drm_gem_object_unreference(&obj->base);
2469 unlock:
2470         DRM_UNLOCK(dev);
2471         return (ret);
2472 }
2473
2474 int
2475 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2476                     struct drm_file *file)
2477 {
2478         struct drm_i915_gem_busy *args;
2479         struct drm_i915_gem_object *obj;
2480         struct drm_i915_gem_request *request;
2481         int ret;
2482
2483         args = data;
2484
2485         ret = i915_mutex_lock_interruptible(dev);
2486         if (ret != 0)
2487                 return ret;
2488
2489         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2490         if (&obj->base == NULL) {
2491                 ret = -ENOENT;
2492                 goto unlock;
2493         }
2494
2495         args->busy = obj->active;
2496         if (args->busy) {
2497                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2498                         ret = i915_gem_flush_ring(obj->ring,
2499                             0, obj->base.write_domain);
2500                 } else if (obj->ring->outstanding_lazy_request ==
2501                     obj->last_rendering_seqno) {
2502                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
2503                             M_WAITOK | M_ZERO);
2504                         ret = i915_add_request(obj->ring, NULL, request);
2505                         if (ret != 0)
2506                                 drm_free(request, DRM_I915_GEM);
2507                 }
2508
2509                 i915_gem_retire_requests_ring(obj->ring);
2510                 args->busy = obj->active;
2511         }
2512
2513         drm_gem_object_unreference(&obj->base);
2514 unlock:
2515         DRM_UNLOCK(dev);
2516         return (ret);
2517 }
2518
2519 int
2520 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2521     struct drm_file *file_priv)
2522 {
2523
2524         return (i915_gem_ring_throttle(dev, file_priv));
2525 }
2526
2527 int
2528 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
2529                        struct drm_file *file_priv)
2530 {
2531         struct drm_i915_gem_madvise *args = data;
2532         struct drm_i915_gem_object *obj;
2533         int ret;
2534
2535         switch (args->madv) {
2536         case I915_MADV_DONTNEED:
2537         case I915_MADV_WILLNEED:
2538             break;
2539         default:
2540             return -EINVAL;
2541         }
2542
2543         ret = i915_mutex_lock_interruptible(dev);
2544         if (ret)
2545                 return ret;
2546
2547         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
2548         if (&obj->base == NULL) {
2549                 ret = -ENOENT;
2550                 goto unlock;
2551         }
2552
2553         if (obj->pin_count) {
2554                 ret = -EINVAL;
2555                 goto out;
2556         }
2557
2558         if (obj->madv != __I915_MADV_PURGED)
2559                 obj->madv = args->madv;
2560
2561         /* if the object is no longer attached, discard its backing storage */
2562         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
2563                 i915_gem_object_truncate(obj);
2564
2565         args->retained = obj->madv != __I915_MADV_PURGED;
2566
2567 out:
2568         drm_gem_object_unreference(&obj->base);
2569 unlock:
2570         DRM_UNLOCK(dev);
2571         return ret;
2572 }
2573
2574 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
2575                                                   size_t size)
2576 {
2577         struct drm_i915_private *dev_priv;
2578         struct drm_i915_gem_object *obj;
2579
2580         dev_priv = dev->dev_private;
2581
2582         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
2583
2584         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
2585                 drm_free(obj, DRM_I915_GEM);
2586                 return (NULL);
2587         }
2588
2589         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2590         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2591
2592         if (HAS_LLC(dev))
2593                 obj->cache_level = I915_CACHE_LLC;
2594         else
2595                 obj->cache_level = I915_CACHE_NONE;
2596         obj->base.driver_private = NULL;
2597         obj->fence_reg = I915_FENCE_REG_NONE;
2598         INIT_LIST_HEAD(&obj->mm_list);
2599         INIT_LIST_HEAD(&obj->gtt_list);
2600         INIT_LIST_HEAD(&obj->ring_list);
2601         INIT_LIST_HEAD(&obj->exec_list);
2602         INIT_LIST_HEAD(&obj->gpu_write_list);
2603         obj->madv = I915_MADV_WILLNEED;
2604         /* Avoid an unnecessary call to unbind on the first bind. */
2605         obj->map_and_fenceable = true;
2606
2607         i915_gem_info_add_obj(dev_priv, size);
2608
2609         return (obj);
2610 }
2611
2612 int i915_gem_init_object(struct drm_gem_object *obj)
2613 {
2614
2615         kprintf("i915_gem_init_object called\n");
2616         return (0);
2617 }
2618
2619 static void
2620 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
2621 {
2622         struct drm_device *dev;
2623         drm_i915_private_t *dev_priv;
2624         int ret;
2625
2626         dev = obj->base.dev;
2627         dev_priv = dev->dev_private;
2628
2629         ret = i915_gem_object_unbind(obj);
2630         if (ret == -ERESTART) {
2631                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
2632                 return;
2633         }
2634
2635         drm_gem_free_mmap_offset(&obj->base);
2636         drm_gem_object_release(&obj->base);
2637         i915_gem_info_remove_obj(dev_priv, obj->base.size);
2638
2639         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
2640         drm_free(obj->bit_17, DRM_I915_GEM);
2641         drm_free(obj, DRM_I915_GEM);
2642 }
2643
2644 void
2645 i915_gem_free_object(struct drm_gem_object *gem_obj)
2646 {
2647         struct drm_i915_gem_object *obj;
2648         struct drm_device *dev;
2649
2650         obj = to_intel_bo(gem_obj);
2651         dev = obj->base.dev;
2652
2653         while (obj->pin_count > 0)
2654                 i915_gem_object_unpin(obj);
2655
2656         if (obj->phys_obj != NULL)
2657                 i915_gem_detach_phys_object(dev, obj);
2658
2659         i915_gem_free_object_tail(obj);
2660 }
2661
2662 int
2663 i915_gem_do_init(struct drm_device *dev, unsigned long start,
2664     unsigned long mappable_end, unsigned long end)
2665 {
2666         drm_i915_private_t *dev_priv;
2667         unsigned long mappable;
2668         int error;
2669
2670         dev_priv = dev->dev_private;
2671         mappable = min(end, mappable_end) - start;
2672
2673         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
2674
2675         dev_priv->mm.gtt_start = start;
2676         dev_priv->mm.gtt_mappable_end = mappable_end;
2677         dev_priv->mm.gtt_end = end;
2678         dev_priv->mm.gtt_total = end - start;
2679         dev_priv->mm.mappable_gtt_total = mappable;
2680
2681         /* Take over this portion of the GTT */
2682         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
2683         device_printf(dev->dev,
2684             "taking over the fictitious range 0x%lx-0x%lx\n",
2685             dev->agp->base + start, dev->agp->base + start + mappable);
2686         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
2687             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2688         return (error);
2689 }
2690
2691 int
2692 i915_gem_idle(struct drm_device *dev)
2693 {
2694         drm_i915_private_t *dev_priv;
2695         int ret;
2696
2697         dev_priv = dev->dev_private;
2698         if (dev_priv->mm.suspended)
2699                 return (0);
2700
2701         ret = i915_gpu_idle(dev);
2702         if (ret != 0)
2703                 return (ret);
2704
2705         /* Under UMS, be paranoid and evict. */
2706         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
2707                 ret = i915_gem_evict_inactive(dev, false);
2708                 if (ret != 0)
2709                         return ret;
2710         }
2711
2712         i915_gem_reset_fences(dev);
2713
2714         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2715          * We need to replace this with a semaphore, or something.
2716          * And not confound mm.suspended!
2717          */
2718         dev_priv->mm.suspended = 1;
2719         del_timer_sync(&dev_priv->hangcheck_timer);
2720
2721         i915_kernel_lost_context(dev);
2722         i915_gem_cleanup_ringbuffer(dev);
2723
2724         /* Cancel the retire work handler, which should be idle now. */
2725         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2726
2727         return (ret);
2728 }
2729
2730 void i915_gem_l3_remap(struct drm_device *dev)
2731 {
2732         drm_i915_private_t *dev_priv = dev->dev_private;
2733         u32 misccpctl;
2734         int i;
2735
2736         if (!HAS_L3_GPU_CACHE(dev))
2737                 return;
2738
2739         if (!dev_priv->l3_parity.remap_info)
2740                 return;
2741
2742         misccpctl = I915_READ(GEN7_MISCCPCTL);
2743         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
2744         POSTING_READ(GEN7_MISCCPCTL);
2745
2746         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
2747                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
2748                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
2749                         DRM_DEBUG("0x%x was already programmed to %x\n",
2750                                   GEN7_L3LOG_BASE + i, remap);
2751                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
2752                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
2753                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
2754         }
2755
2756         /* Make sure all the writes land before disabling dop clock gating */
2757         POSTING_READ(GEN7_L3LOG_BASE);
2758
2759         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
2760 }
2761
2762 void
2763 i915_gem_init_swizzling(struct drm_device *dev)
2764 {
2765         drm_i915_private_t *dev_priv;
2766
2767         dev_priv = dev->dev_private;
2768
2769         if (INTEL_INFO(dev)->gen < 5 ||
2770             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
2771                 return;
2772
2773         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
2774                                  DISP_TILE_SURFACE_SWIZZLING);
2775
2776         if (IS_GEN5(dev))
2777                 return;
2778
2779         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
2780         if (IS_GEN6(dev))
2781                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
2782         else
2783                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
2784 }
2785
2786 static bool
2787 intel_enable_blt(struct drm_device *dev)
2788 {
2789         int revision;
2790
2791         if (!HAS_BLT(dev))
2792                 return false;
2793
2794         /* The blitter was dysfunctional on early prototypes */
2795         revision = pci_read_config(dev->dev, PCIR_REVID, 1);
2796         if (IS_GEN6(dev) && revision < 8) {
2797                 DRM_INFO("BLT not supported on this pre-production hardware;"
2798                          " graphics performance will be degraded.\n");
2799                 return false;
2800         }
2801
2802         return true;
2803 }
2804
2805 int
2806 i915_gem_init_hw(struct drm_device *dev)
2807 {
2808         drm_i915_private_t *dev_priv = dev->dev_private;
2809         int ret;
2810
2811         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
2812                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
2813
2814         i915_gem_l3_remap(dev);
2815
2816         i915_gem_init_swizzling(dev);
2817
2818         ret = intel_init_render_ring_buffer(dev);
2819         if (ret)
2820                 return ret;
2821
2822         if (HAS_BSD(dev)) {
2823                 ret = intel_init_bsd_ring_buffer(dev);
2824                 if (ret)
2825                         goto cleanup_render_ring;
2826         }
2827
2828         if (intel_enable_blt(dev)) {
2829                 ret = intel_init_blt_ring_buffer(dev);
2830                 if (ret)
2831                         goto cleanup_bsd_ring;
2832         }
2833
2834         dev_priv->next_seqno = 1;
2835
2836         /*
2837          * XXX: There was some w/a described somewhere suggesting loading
2838          * contexts before PPGTT.
2839          */
2840 #if 0   /* XXX: HW context support */
2841         i915_gem_context_init(dev);
2842 #endif
2843         i915_gem_init_ppgtt(dev);
2844
2845         return 0;
2846
2847 cleanup_bsd_ring:
2848         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
2849 cleanup_render_ring:
2850         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
2851         return ret;
2852 }
2853
2854 void
2855 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2856 {
2857         drm_i915_private_t *dev_priv;
2858         int i;
2859
2860         dev_priv = dev->dev_private;
2861         for (i = 0; i < I915_NUM_RINGS; i++)
2862                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
2863 }
2864
2865 int
2866 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2867                        struct drm_file *file_priv)
2868 {
2869         drm_i915_private_t *dev_priv = dev->dev_private;
2870         int ret;
2871
2872         if (drm_core_check_feature(dev, DRIVER_MODESET))
2873                 return 0;
2874
2875         if (atomic_read(&dev_priv->mm.wedged)) {
2876                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2877                 atomic_set(&dev_priv->mm.wedged, 0);
2878         }
2879
2880         DRM_LOCK(dev);
2881         dev_priv->mm.suspended = 0;
2882
2883         ret = i915_gem_init_hw(dev);
2884         if (ret != 0) {
2885                 DRM_UNLOCK(dev);
2886                 return ret;
2887         }
2888
2889         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
2890         DRM_UNLOCK(dev);
2891
2892         ret = drm_irq_install(dev);
2893         if (ret)
2894                 goto cleanup_ringbuffer;
2895
2896         return 0;
2897
2898 cleanup_ringbuffer:
2899         DRM_LOCK(dev);
2900         i915_gem_cleanup_ringbuffer(dev);
2901         dev_priv->mm.suspended = 1;
2902         DRM_UNLOCK(dev);
2903
2904         return ret;
2905 }
2906
2907 int
2908 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2909     struct drm_file *file_priv)
2910 {
2911
2912         if (drm_core_check_feature(dev, DRIVER_MODESET))
2913                 return 0;
2914
2915         drm_irq_uninstall(dev);
2916         return (i915_gem_idle(dev));
2917 }
2918
2919 void
2920 i915_gem_lastclose(struct drm_device *dev)
2921 {
2922         int ret;
2923
2924         if (drm_core_check_feature(dev, DRIVER_MODESET))
2925                 return;
2926
2927         ret = i915_gem_idle(dev);
2928         if (ret != 0)
2929                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2930 }
2931
2932 static void
2933 init_ring_lists(struct intel_ring_buffer *ring)
2934 {
2935
2936         INIT_LIST_HEAD(&ring->active_list);
2937         INIT_LIST_HEAD(&ring->request_list);
2938         INIT_LIST_HEAD(&ring->gpu_write_list);
2939 }
2940
2941 void
2942 i915_gem_load(struct drm_device *dev)
2943 {
2944         int i;
2945         drm_i915_private_t *dev_priv = dev->dev_private;
2946
2947         INIT_LIST_HEAD(&dev_priv->mm.active_list);
2948         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2949         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2950         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
2951         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2952         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
2953         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
2954         for (i = 0; i < I915_NUM_RINGS; i++)
2955                 init_ring_lists(&dev_priv->ring[i]);
2956         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2957                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2958         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2959                           i915_gem_retire_work_handler);
2960         init_completion(&dev_priv->error_completion);
2961
2962         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
2963         if (IS_GEN3(dev)) {
2964                 I915_WRITE(MI_ARB_STATE,
2965                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2966         }
2967
2968         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
2969
2970         /* Old X drivers will take 0-2 for front, back, depth buffers */
2971         if (!drm_core_check_feature(dev, DRIVER_MODESET))
2972                 dev_priv->fence_reg_start = 3;
2973
2974         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
2975                 dev_priv->num_fence_regs = 16;
2976         else
2977                 dev_priv->num_fence_regs = 8;
2978
2979         /* Initialize fence registers to zero */
2980         i915_gem_reset_fences(dev);
2981
2982         i915_gem_detect_bit_6_swizzle(dev);
2983
2984         dev_priv->mm.interruptible = true;
2985
2986         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
2987             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
2988 }
2989
2990 static int
2991 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
2992 {
2993         drm_i915_private_t *dev_priv;
2994         struct drm_i915_gem_phys_object *phys_obj;
2995         int ret;
2996
2997         dev_priv = dev->dev_private;
2998         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
2999                 return (0);
3000
3001         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
3002             M_WAITOK | M_ZERO);
3003
3004         phys_obj->id = id;
3005
3006         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
3007         if (phys_obj->handle == NULL) {
3008                 ret = -ENOMEM;
3009                 goto free_obj;
3010         }
3011         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
3012             size / PAGE_SIZE, PAT_WRITE_COMBINING);
3013
3014         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3015
3016         return (0);
3017
3018 free_obj:
3019         drm_free(phys_obj, DRM_I915_GEM);
3020         return (ret);
3021 }
3022
3023 static void
3024 i915_gem_free_phys_object(struct drm_device *dev, int id)
3025 {
3026         drm_i915_private_t *dev_priv;
3027         struct drm_i915_gem_phys_object *phys_obj;
3028
3029         dev_priv = dev->dev_private;
3030         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3031                 return;
3032
3033         phys_obj = dev_priv->mm.phys_objs[id - 1];
3034         if (phys_obj->cur_obj != NULL)
3035                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3036
3037         drm_pci_free(dev, phys_obj->handle);
3038         drm_free(phys_obj, DRM_I915_GEM);
3039         dev_priv->mm.phys_objs[id - 1] = NULL;
3040 }
3041
3042 void
3043 i915_gem_free_all_phys_object(struct drm_device *dev)
3044 {
3045         int i;
3046
3047         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3048                 i915_gem_free_phys_object(dev, i);
3049 }
3050
3051 void
3052 i915_gem_detach_phys_object(struct drm_device *dev,
3053     struct drm_i915_gem_object *obj)
3054 {
3055         vm_page_t m;
3056         struct sf_buf *sf;
3057         char *vaddr, *dst;
3058         int i, page_count;
3059
3060         if (obj->phys_obj == NULL)
3061                 return;
3062         vaddr = obj->phys_obj->handle->vaddr;
3063
3064         page_count = obj->base.size / PAGE_SIZE;
3065         VM_OBJECT_LOCK(obj->base.vm_obj);
3066         for (i = 0; i < page_count; i++) {
3067                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3068                 if (m == NULL)
3069                         continue; /* XXX */
3070
3071                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3072                 sf = sf_buf_alloc(m);
3073                 if (sf != NULL) {
3074                         dst = (char *)sf_buf_kva(sf);
3075                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3076                         sf_buf_free(sf);
3077                 }
3078                 drm_clflush_pages(&m, 1);
3079
3080                 VM_OBJECT_LOCK(obj->base.vm_obj);
3081                 vm_page_reference(m);
3082                 vm_page_dirty(m);
3083                 vm_page_busy_wait(m, FALSE, "i915gem");
3084                 vm_page_unwire(m, 0);
3085                 vm_page_wakeup(m);
3086                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3087         }
3088         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3089         intel_gtt_chipset_flush();
3090
3091         obj->phys_obj->cur_obj = NULL;
3092         obj->phys_obj = NULL;
3093 }
3094
3095 int
3096 i915_gem_attach_phys_object(struct drm_device *dev,
3097                             struct drm_i915_gem_object *obj,
3098                             int id,
3099                             int align)
3100 {
3101         drm_i915_private_t *dev_priv;
3102         vm_page_t m;
3103         struct sf_buf *sf;
3104         char *dst, *src;
3105         int i, page_count, ret;
3106
3107         if (id > I915_MAX_PHYS_OBJECT)
3108                 return (-EINVAL);
3109
3110         if (obj->phys_obj != NULL) {
3111                 if (obj->phys_obj->id == id)
3112                         return (0);
3113                 i915_gem_detach_phys_object(dev, obj);
3114         }
3115
3116         dev_priv = dev->dev_private;
3117         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3118                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3119                 if (ret != 0) {
3120                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3121                                   id, obj->base.size);
3122                         return (ret);
3123                 }
3124         }
3125
3126         /* bind to the object */
3127         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3128         obj->phys_obj->cur_obj = obj;
3129
3130         page_count = obj->base.size / PAGE_SIZE;
3131
3132         VM_OBJECT_LOCK(obj->base.vm_obj);
3133         ret = 0;
3134         for (i = 0; i < page_count; i++) {
3135                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3136                 if (m == NULL) {
3137                         ret = -EIO;
3138                         break;
3139                 }
3140                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3141                 sf = sf_buf_alloc(m);
3142                 src = (char *)sf_buf_kva(sf);
3143                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3144                 memcpy(dst, src, PAGE_SIZE);
3145                 sf_buf_free(sf);
3146
3147                 VM_OBJECT_LOCK(obj->base.vm_obj);
3148
3149                 vm_page_reference(m);
3150                 vm_page_busy_wait(m, FALSE, "i915gem");
3151                 vm_page_unwire(m, 0);
3152                 vm_page_wakeup(m);
3153                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3154         }
3155         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3156
3157         return (0);
3158 }
3159
3160 static int
3161 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3162     uint64_t data_ptr, uint64_t offset, uint64_t size,
3163     struct drm_file *file_priv)
3164 {
3165         char *user_data, *vaddr;
3166         int ret;
3167
3168         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3169         user_data = (char *)(uintptr_t)data_ptr;
3170
3171         if (copyin_nofault(user_data, vaddr, size) != 0) {
3172                 /* The physical object once assigned is fixed for the lifetime
3173                  * of the obj, so we can safely drop the lock and continue
3174                  * to access vaddr.
3175                  */
3176                 DRM_UNLOCK(dev);
3177                 ret = -copyin(user_data, vaddr, size);
3178                 DRM_LOCK(dev);
3179                 if (ret != 0)
3180                         return (ret);
3181         }
3182
3183         intel_gtt_chipset_flush();
3184         return (0);
3185 }
3186
3187 void
3188 i915_gem_release(struct drm_device *dev, struct drm_file *file)
3189 {
3190         struct drm_i915_file_private *file_priv;
3191         struct drm_i915_gem_request *request;
3192
3193         file_priv = file->driver_priv;
3194
3195         /* Clean up our request list when the client is going away, so that
3196          * later retire_requests won't dereference our soon-to-be-gone
3197          * file_priv.
3198          */
3199         spin_lock(&file_priv->mm.lock);
3200         while (!list_empty(&file_priv->mm.request_list)) {
3201                 request = list_first_entry(&file_priv->mm.request_list,
3202                                            struct drm_i915_gem_request,
3203                                            client_list);
3204                 list_del(&request->client_list);
3205                 request->file_priv = NULL;
3206         }
3207         spin_unlock(&file_priv->mm.lock);
3208 }
3209
3210 static int
3211 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
3212     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
3213     struct drm_file *file)
3214 {
3215         vm_object_t vm_obj;
3216         vm_page_t m;
3217         struct sf_buf *sf;
3218         vm_offset_t mkva;
3219         vm_pindex_t obj_pi;
3220         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
3221
3222         if (obj->gtt_offset != 0 && rw == UIO_READ)
3223                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
3224         else
3225                 do_bit17_swizzling = 0;
3226
3227         obj->dirty = 1;
3228         vm_obj = obj->base.vm_obj;
3229         ret = 0;
3230
3231         VM_OBJECT_LOCK(vm_obj);
3232         vm_object_pip_add(vm_obj, 1);
3233         while (size > 0) {
3234                 obj_pi = OFF_TO_IDX(offset);
3235                 obj_po = offset & PAGE_MASK;
3236
3237                 m = i915_gem_wire_page(vm_obj, obj_pi);
3238                 VM_OBJECT_UNLOCK(vm_obj);
3239
3240                 sf = sf_buf_alloc(m);
3241                 mkva = sf_buf_kva(sf);
3242                 length = min(size, PAGE_SIZE - obj_po);
3243                 while (length > 0) {
3244                         if (do_bit17_swizzling &&
3245                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
3246                                 cnt = roundup2(obj_po + 1, 64);
3247                                 cnt = min(cnt - obj_po, length);
3248                                 swizzled_po = obj_po ^ 64;
3249                         } else {
3250                                 cnt = length;
3251                                 swizzled_po = obj_po;
3252                         }
3253                         if (rw == UIO_READ)
3254                                 ret = -copyout_nofault(
3255                                     (char *)mkva + swizzled_po,
3256                                     (void *)(uintptr_t)data_ptr, cnt);
3257                         else
3258                                 ret = -copyin_nofault(
3259                                     (void *)(uintptr_t)data_ptr,
3260                                     (char *)mkva + swizzled_po, cnt);
3261                         if (ret != 0)
3262                                 break;
3263                         data_ptr += cnt;
3264                         size -= cnt;
3265                         length -= cnt;
3266                         offset += cnt;
3267                         obj_po += cnt;
3268                 }
3269                 sf_buf_free(sf);
3270                 VM_OBJECT_LOCK(vm_obj);
3271                 if (rw == UIO_WRITE)
3272                         vm_page_dirty(m);
3273                 vm_page_reference(m);
3274                 vm_page_busy_wait(m, FALSE, "i915gem");
3275                 vm_page_unwire(m, 1);
3276                 vm_page_wakeup(m);
3277                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3278
3279                 if (ret != 0)
3280                         break;
3281         }
3282         vm_object_pip_wakeup(vm_obj);
3283         VM_OBJECT_UNLOCK(vm_obj);
3284
3285         return (ret);
3286 }
3287
3288 static int
3289 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
3290     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
3291 {
3292         vm_offset_t mkva;
3293         int ret;
3294
3295         /*
3296          * Pass the unaligned physical address and size to pmap_mapdev_attr()
3297          * so it can properly calculate whether an extra page needs to be
3298          * mapped or not to cover the requested range.  The function will
3299          * add the page offset into the returned mkva for us.
3300          */
3301         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
3302             offset, size, PAT_WRITE_COMBINING);
3303         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
3304         pmap_unmapdev(mkva, size);
3305         return (ret);
3306 }
3307
3308 static int
3309 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
3310     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
3311 {
3312         struct drm_i915_gem_object *obj;
3313         vm_page_t *ma;
3314         vm_offset_t start, end;
3315         int npages, ret;
3316
3317         if (size == 0)
3318                 return (0);
3319         start = trunc_page(data_ptr);
3320         end = round_page(data_ptr + size);
3321         npages = howmany(end - start, PAGE_SIZE);
3322         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
3323             M_ZERO);
3324         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
3325             (vm_offset_t)data_ptr, size,
3326             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
3327         if (npages == -1) {
3328                 ret = -EFAULT;
3329                 goto free_ma;
3330         }
3331
3332         ret = i915_mutex_lock_interruptible(dev);
3333         if (ret != 0)
3334                 goto unlocked;
3335
3336         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
3337         if (&obj->base == NULL) {
3338                 ret = -ENOENT;
3339                 goto unlock;
3340         }
3341         if (offset > obj->base.size || size > obj->base.size - offset) {
3342                 ret = -EINVAL;
3343                 goto out;
3344         }
3345
3346         if (rw == UIO_READ) {
3347                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
3348                     offset, size);
3349                 if (ret != 0)
3350                         goto out;
3351                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3352                     UIO_READ, file);
3353         } else {
3354                 if (obj->phys_obj) {
3355                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
3356                             size, file);
3357                 } else if (obj->gtt_space &&
3358                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3359                         ret = i915_gem_object_pin(obj, 0, true);
3360                         if (ret != 0)
3361                                 goto out;
3362                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
3363                         if (ret != 0)
3364                                 goto out_unpin;
3365                         ret = i915_gem_object_put_fence(obj);
3366                         if (ret != 0)
3367                                 goto out_unpin;
3368                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
3369                             offset, file);
3370 out_unpin:
3371                         i915_gem_object_unpin(obj);
3372                 } else {
3373                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
3374                         if (ret != 0)
3375                                 goto out;
3376                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3377                             UIO_WRITE, file);
3378                 }
3379         }
3380 out:
3381         drm_gem_object_unreference(&obj->base);
3382 unlock:
3383         DRM_UNLOCK(dev);
3384 unlocked:
3385         vm_page_unhold_pages(ma, npages);
3386 free_ma:
3387         drm_free(ma, DRM_I915_GEM);
3388         return (ret);
3389 }
3390
3391 static int
3392 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
3393     vm_ooffset_t foff, struct ucred *cred, u_short *color)
3394 {
3395
3396         *color = 0; /* XXXKIB */
3397         return (0);
3398 }
3399
3400 int i915_intr_pf;
3401
3402 static int
3403 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
3404     vm_page_t *mres)
3405 {
3406         struct drm_gem_object *gem_obj;
3407         struct drm_i915_gem_object *obj;
3408         struct drm_device *dev;
3409         drm_i915_private_t *dev_priv;
3410         vm_page_t m, oldm;
3411         int cause, ret;
3412         bool write;
3413
3414         gem_obj = vm_obj->handle;
3415         obj = to_intel_bo(gem_obj);
3416         dev = obj->base.dev;
3417         dev_priv = dev->dev_private;
3418 #if 0
3419         write = (prot & VM_PROT_WRITE) != 0;
3420 #else
3421         write = true;
3422 #endif
3423         vm_object_pip_add(vm_obj, 1);
3424
3425         /*
3426          * Remove the placeholder page inserted by vm_fault() from the
3427          * object before dropping the object lock. If
3428          * i915_gem_release_mmap() is active in parallel on this gem
3429          * object, then it owns the drm device sx and might find the
3430          * placeholder already. Then, since the page is busy,
3431          * i915_gem_release_mmap() sleeps waiting for the busy state
3432          * of the page cleared. We will be not able to acquire drm
3433          * device lock until i915_gem_release_mmap() is able to make a
3434          * progress.
3435          */
3436         if (*mres != NULL) {
3437                 oldm = *mres;
3438                 vm_page_remove(oldm);
3439                 *mres = NULL;
3440         } else
3441                 oldm = NULL;
3442 retry:
3443         VM_OBJECT_UNLOCK(vm_obj);
3444 unlocked_vmobj:
3445         cause = ret = 0;
3446         m = NULL;
3447
3448         if (i915_intr_pf) {
3449                 ret = i915_mutex_lock_interruptible(dev);
3450                 if (ret != 0) {
3451                         cause = 10;
3452                         goto out;
3453                 }
3454         } else
3455                 DRM_LOCK(dev);
3456
3457         /*
3458          * Since the object lock was dropped, other thread might have
3459          * faulted on the same GTT address and instantiated the
3460          * mapping for the page.  Recheck.
3461          */
3462         VM_OBJECT_LOCK(vm_obj);
3463         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
3464         if (m != NULL) {
3465                 if ((m->flags & PG_BUSY) != 0) {
3466                         DRM_UNLOCK(dev);
3467 #if 0 /* XXX */
3468                         vm_page_sleep(m, "915pee");
3469 #endif
3470                         goto retry;
3471                 }
3472                 goto have_page;
3473         } else
3474                 VM_OBJECT_UNLOCK(vm_obj);
3475
3476         /* Now bind it into the GTT if needed */
3477         if (!obj->map_and_fenceable) {
3478                 ret = i915_gem_object_unbind(obj);
3479                 if (ret != 0) {
3480                         cause = 20;
3481                         goto unlock;
3482                 }
3483         }
3484         if (!obj->gtt_space) {
3485                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
3486                 if (ret != 0) {
3487                         cause = 30;
3488                         goto unlock;
3489                 }
3490
3491                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
3492                 if (ret != 0) {
3493                         cause = 40;
3494                         goto unlock;
3495                 }
3496         }
3497
3498         if (obj->tiling_mode == I915_TILING_NONE)
3499                 ret = i915_gem_object_put_fence(obj);
3500         else
3501                 ret = i915_gem_object_get_fence(obj, NULL);
3502         if (ret != 0) {
3503                 cause = 50;
3504                 goto unlock;
3505         }
3506
3507         if (i915_gem_object_is_inactive(obj))
3508                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3509
3510         obj->fault_mappable = true;
3511         VM_OBJECT_LOCK(vm_obj);
3512         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
3513             offset);
3514         if (m == NULL) {
3515                 cause = 60;
3516                 ret = -EFAULT;
3517                 goto unlock;
3518         }
3519         KASSERT((m->flags & PG_FICTITIOUS) != 0,
3520             ("not fictitious %p", m));
3521         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
3522
3523         if ((m->flags & PG_BUSY) != 0) {
3524                 DRM_UNLOCK(dev);
3525 #if 0 /* XXX */
3526                 vm_page_sleep(m, "915pbs");
3527 #endif
3528                 goto retry;
3529         }
3530         m->valid = VM_PAGE_BITS_ALL;
3531         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
3532 have_page:
3533         *mres = m;
3534         vm_page_busy_try(m, false);
3535
3536         DRM_UNLOCK(dev);
3537         if (oldm != NULL) {
3538                 vm_page_free(oldm);
3539         }
3540         vm_object_pip_wakeup(vm_obj);
3541         return (VM_PAGER_OK);
3542
3543 unlock:
3544         DRM_UNLOCK(dev);
3545 out:
3546         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
3547         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
3548                 goto unlocked_vmobj;
3549         }
3550         VM_OBJECT_LOCK(vm_obj);
3551         vm_object_pip_wakeup(vm_obj);
3552         return (VM_PAGER_ERROR);
3553 }
3554
3555 static void
3556 i915_gem_pager_dtor(void *handle)
3557 {
3558         struct drm_gem_object *obj;
3559         struct drm_device *dev;
3560
3561         obj = handle;
3562         dev = obj->dev;
3563
3564         DRM_LOCK(dev);
3565         drm_gem_free_mmap_offset(obj);
3566         i915_gem_release_mmap(to_intel_bo(obj));
3567         drm_gem_object_unreference(obj);
3568         DRM_UNLOCK(dev);
3569 }
3570
3571 struct cdev_pager_ops i915_gem_pager_ops = {
3572         .cdev_pg_fault  = i915_gem_pager_fault,
3573         .cdev_pg_ctor   = i915_gem_pager_ctor,
3574         .cdev_pg_dtor   = i915_gem_pager_dtor
3575 };
3576
3577 static int
3578 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3579     uint64_t offset, uint64_t size)
3580 {
3581         uint32_t old_read_domains;
3582         int i, ret;
3583
3584         if (offset == 0 && size == obj->base.size)
3585                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
3586
3587         ret = i915_gem_object_flush_gpu_write_domain(obj);
3588         if (ret != 0)
3589                 return (ret);
3590         ret = i915_gem_object_wait_rendering(obj);
3591         if (ret != 0)
3592                 return (ret);
3593
3594         i915_gem_object_flush_gtt_write_domain(obj);
3595
3596         if (obj->page_cpu_valid == NULL &&
3597             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3598                 return (0);
3599
3600         if (obj->page_cpu_valid == NULL) {
3601                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
3602                     DRM_I915_GEM, M_WAITOK | M_ZERO);
3603         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3604                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3605
3606         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3607              i++) {
3608                 if (obj->page_cpu_valid[i])
3609                         continue;
3610                 drm_clflush_pages(obj->pages + i, 1);
3611                 obj->page_cpu_valid[i] = 1;
3612         }
3613
3614         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
3615             ("In gpu write domain"));
3616
3617         old_read_domains = obj->base.read_domains;
3618         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3619
3620         return (0);
3621 }
3622
3623 #define GEM_PARANOID_CHECK_GTT 0
3624 #if GEM_PARANOID_CHECK_GTT
3625 static void
3626 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
3627     int page_count)
3628 {
3629         struct drm_i915_private *dev_priv;
3630         vm_paddr_t pa;
3631         unsigned long start, end;
3632         u_int i;
3633         int j;
3634
3635         dev_priv = dev->dev_private;
3636         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
3637         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
3638         for (i = start; i < end; i++) {
3639                 pa = intel_gtt_read_pte_paddr(i);
3640                 for (j = 0; j < page_count; j++) {
3641                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
3642                                 panic("Page %p in GTT pte index %d pte %x",
3643                                     ma[i], i, intel_gtt_read_pte(i));
3644                         }
3645                 }
3646         }
3647 }
3648 #endif
3649
3650 static void
3651 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
3652     uint32_t flush_domains)
3653 {
3654         struct drm_i915_gem_object *obj, *next;
3655         uint32_t old_write_domain;
3656
3657         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
3658             gpu_write_list) {
3659                 if (obj->base.write_domain & flush_domains) {
3660                         old_write_domain = obj->base.write_domain;
3661                         obj->base.write_domain = 0;
3662                         list_del_init(&obj->gpu_write_list);
3663                         i915_gem_object_move_to_active(obj, ring,
3664                             i915_gem_next_request_seqno(ring));
3665                 }
3666         }
3667 }
3668
3669 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3670
3671 static vm_page_t
3672 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3673 {
3674         vm_page_t m;
3675         int rv;
3676
3677         VM_OBJECT_LOCK_ASSERT_OWNED(object);
3678         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
3679         if (m->valid != VM_PAGE_BITS_ALL) {
3680                 if (vm_pager_has_page(object, pindex)) {
3681                         rv = vm_pager_get_page(object, &m, 1);
3682                         m = vm_page_lookup(object, pindex);
3683                         if (m == NULL)
3684                                 return (NULL);
3685                         if (rv != VM_PAGER_OK) {
3686                                 vm_page_free(m);
3687                                 return (NULL);
3688                         }
3689                 } else {
3690                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
3691                         m->valid = VM_PAGE_BITS_ALL;
3692                         m->dirty = 0;
3693                 }
3694         }
3695         vm_page_wire(m);
3696         vm_page_wakeup(m);
3697         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3698         return (m);
3699 }
3700
3701 int
3702 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
3703     uint32_t flush_domains)
3704 {
3705         int ret;
3706
3707         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
3708                 return 0;
3709
3710         ret = ring->flush(ring, invalidate_domains, flush_domains);
3711         if (ret)
3712                 return ret;
3713
3714         if (flush_domains & I915_GEM_GPU_DOMAINS)
3715                 i915_gem_process_flushing_list(ring, flush_domains);
3716         return 0;
3717 }
3718
3719 u32
3720 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
3721 {
3722         if (ring->outstanding_lazy_request == 0)
3723                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
3724
3725         return ring->outstanding_lazy_request;
3726 }
3727
3728 static void
3729 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3730 {
3731         drm_i915_private_t *dev_priv = dev->dev_private;
3732         uint32_t fence_reg = reg - dev_priv->fence_regs;
3733
3734         switch (INTEL_INFO(dev)->gen) {
3735         case 7:
3736         case 6:
3737                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3738                 break;
3739         case 5:
3740         case 4:
3741                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3742                 break;
3743         case 3:
3744                 if (fence_reg >= 8)
3745                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3746                 else
3747         case 2:
3748                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3749
3750                 I915_WRITE(fence_reg, 0);
3751                 break;
3752         }
3753
3754         list_del_init(&reg->lru_list);
3755         reg->obj = NULL;
3756         reg->setup_seqno = 0;
3757         reg->pin_count = 0;
3758 }
3759
3760 static int
3761 i915_gpu_is_active(struct drm_device *dev)
3762 {
3763         drm_i915_private_t *dev_priv;
3764
3765         dev_priv = dev->dev_private;
3766         return (!list_empty(&dev_priv->mm.flushing_list) ||
3767             !list_empty(&dev_priv->mm.active_list));
3768 }
3769
3770 static void
3771 i915_gem_lowmem(void *arg)
3772 {
3773         struct drm_device *dev;
3774         struct drm_i915_private *dev_priv;
3775         struct drm_i915_gem_object *obj, *next;
3776         int cnt, cnt_fail, cnt_total;
3777
3778         dev = arg;
3779         dev_priv = dev->dev_private;
3780
3781         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3782                 return;
3783
3784 rescan:
3785         /* first scan for clean buffers */
3786         i915_gem_retire_requests(dev);
3787
3788         cnt_total = cnt_fail = cnt = 0;
3789
3790         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3791             mm_list) {
3792                 if (i915_gem_object_is_purgeable(obj)) {
3793                         if (i915_gem_object_unbind(obj) != 0)
3794                                 cnt_total++;
3795                 } else
3796                         cnt_total++;
3797         }
3798
3799         /* second pass, evict/count anything still on the inactive list */
3800         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3801             mm_list) {
3802                 if (i915_gem_object_unbind(obj) == 0)
3803                         cnt++;
3804                 else
3805                         cnt_fail++;
3806         }
3807
3808         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3809                 /*
3810                  * We are desperate for pages, so as a last resort, wait
3811                  * for the GPU to finish and discard whatever we can.
3812                  * This has a dramatic impact to reduce the number of
3813                  * OOM-killer events whilst running the GPU aggressively.
3814                  */
3815                 if (i915_gpu_idle(dev) == 0)
3816                         goto rescan;
3817         }
3818         DRM_UNLOCK(dev);
3819 }
3820
3821 void
3822 i915_gem_unload(struct drm_device *dev)
3823 {
3824         struct drm_i915_private *dev_priv;
3825
3826         dev_priv = dev->dev_private;
3827         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3828 }