sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64
  65 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  66 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  67 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  68     unsigned alignment, bool map_and_fenceable);
  69
  70 static int i915_gem_phys_pwrite(struct drm_device *dev,
  71     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  72     uint64_t size, struct drm_file *file_priv);
  73
  74 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  75     int tiling_mode);
  76 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  77     uint32_t size, int tiling_mode);
  78 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  79     int flags);
  80 static int i915_gem_object_set_cpu_read_domain_range(
  81     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  82 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  83 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  84 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  85 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  86 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  87 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  88 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  89     uint32_t flush_domains);
  90 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  91     struct drm_i915_fence_reg *reg);
  92 static void i915_gem_reset_fences(struct drm_device *dev);
  93 static void i915_gem_lowmem(void *arg);
  94
  95 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
  96     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file);
  97
  98 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
  99 long i915_gem_wired_pages_cnt;
 100
 101 /* some bookkeeping */
 102 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 103                                   size_t size)
 104 {
 105
 106         dev_priv->mm.object_count++;
 107         dev_priv->mm.object_memory += size;
 108 }
 109
 110 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 111                                      size_t size)
 112 {
 113
 114         dev_priv->mm.object_count--;
 115         dev_priv->mm.object_memory -= size;
 116 }
 117
 118 static int
 119 i915_gem_wait_for_error(struct drm_device *dev)
 120 {
 121         struct drm_i915_private *dev_priv = dev->dev_private;
 122         int ret;
 123
 124         if (!atomic_read(&dev_priv->mm.wedged))
 125                 return 0;
 126
 127         lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 128         while (dev_priv->error_completion == 0) {
 129                 ret = -lksleep(&dev_priv->error_completion,
 130                     &dev_priv->error_completion_lock, PCATCH, "915wco", 0);
 131                 if (ret != 0) {
 132                         lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 133                         return (ret);
 134                 }
 135         }
 136         lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 137
 138         if (atomic_read(&dev_priv->mm.wedged)) {
 139                 /* GPU is hung, bump the completion count to account for
 140                  * the token we just consumed so that we never hit zero and
 141                  * end up waiting upon a subsequent completion event that
 142                  * will never happen.
 143                  */
 144                 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 145                 dev_priv->error_completion++;
 146                 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 147         }
 148         return 0;
 149 }
 150
 151 int i915_mutex_lock_interruptible(struct drm_device *dev)
 152 {
 153         int ret;
 154
 155         ret = i915_gem_wait_for_error(dev);
 156         if (ret != 0)
 157                 return (ret);
 158
 159         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 160         if (ret)
 161                 return -EINTR;
 162
 163 #if 0
 164         WARN_ON(i915_verify_lists(dev));
 165 #endif
 166         return 0;
 167 }
 168
 169 static inline bool
 170 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 171 {
 172         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
 173 }
 174
 175 int
 176 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 177     struct drm_file *file)
 178 {
 179         struct drm_i915_gem_init *args;
 180         drm_i915_private_t *dev_priv;
 181
 182         dev_priv = dev->dev_private;
 183         args = data;
 184
 185         if (args->gtt_start >= args->gtt_end ||
 186             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 187                 return (-EINVAL);
 188
 189         /*
 190          * XXXKIB. The second-time initialization should be guarded
 191          * against.
 192          */
 193         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 194         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 195         lockmgr(&dev->dev_lock, LK_RELEASE);
 196
 197         return 0;
 198 }
 199
 200 int
 201 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 202                             struct drm_file *file)
 203 {
 204         struct drm_i915_private *dev_priv;
 205         struct drm_i915_gem_get_aperture *args;
 206         struct drm_i915_gem_object *obj;
 207         size_t pinned;
 208
 209         dev_priv = dev->dev_private;
 210         args = data;
 211
 212         if (!(dev->driver->driver_features & DRIVER_GEM))
 213                 return (-ENODEV);
 214
 215         pinned = 0;
 216         DRM_LOCK(dev);
 217         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 218                 pinned += obj->gtt_space->size;
 219         DRM_UNLOCK(dev);
 220
 221         args->aper_size = dev_priv->mm.gtt_total;
 222         args->aper_available_size = args->aper_size - pinned;
 223
 224         return (0);
 225 }
 226
 227 int
 228 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 229     uint32_t *handle_p)
 230 {
 231         struct drm_i915_gem_object *obj;
 232         uint32_t handle;
 233         int ret;
 234
 235         size = roundup(size, PAGE_SIZE);
 236         if (size == 0)
 237                 return (-EINVAL);
 238
 239         obj = i915_gem_alloc_object(dev, size);
 240         if (obj == NULL)
 241                 return (-ENOMEM);
 242
 243         handle = 0;
 244         ret = drm_gem_handle_create(file, &obj->base, &handle);
 245         if (ret != 0) {
 246                 drm_gem_object_release(&obj->base);
 247                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 248                 drm_free(obj, DRM_I915_GEM);
 249                 return (-ret);
 250         }
 251
 252         /* drop reference from allocate - handle holds it now */
 253         drm_gem_object_unreference(&obj->base);
 254         *handle_p = handle;
 255         return (0);
 256 }
 257
 258 int
 259 i915_gem_dumb_create(struct drm_file *file,
 260                      struct drm_device *dev,
 261                      struct drm_mode_create_dumb *args)
 262 {
 263
 264         /* have to work out size/pitch and return them */
 265         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 266         args->size = args->pitch * args->height;
 267         return (i915_gem_create(file, dev, args->size, &args->handle));
 268 }
 269
 270 int i915_gem_dumb_destroy(struct drm_file *file,
 271                           struct drm_device *dev,
 272                           uint32_t handle)
 273 {
 274
 275         return (drm_gem_handle_delete(file, handle));
 276 }
 277
 278 /**
 279  * Creates a new mm object and returns a handle to it.
 280  */
 281 int
 282 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 283                       struct drm_file *file)
 284 {
 285         struct drm_i915_gem_create *args = data;
 286
 287         return (i915_gem_create(file, dev, args->size, &args->handle));
 288 }
 289
 290 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 291 {
 292         drm_i915_private_t *dev_priv;
 293
 294         dev_priv = obj->base.dev->dev_private;
 295         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 296             obj->tiling_mode != I915_TILING_NONE);
 297 }
 298
 299 /**
 300  * Reads data from the object referenced by handle.
 301  *
 302  * On error, the contents of *data are undefined.
 303  */
 304 int
 305 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 306                      struct drm_file *file)
 307 {
 308         struct drm_i915_gem_pread *args;
 309
 310         args = data;
 311         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 312             args->offset, UIO_READ, file));
 313 }
 314
 315 /**
 316  * Writes data to the object referenced by handle.
 317  *
 318  * On error, the contents of the buffer that were to be modified are undefined.
 319  */
 320 int
 321 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 322                       struct drm_file *file)
 323 {
 324         struct drm_i915_gem_pwrite *args;
 325
 326         args = data;
 327         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 328             args->offset, UIO_WRITE, file));
 329 }
 330
 331 /**
 332  * Waits for a sequence number to be signaled, and cleans up the
 333  * request and object lists appropriately for that event.
 334  */
 335 int
 336 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
 337 {
 338         drm_i915_private_t *dev_priv;
 339         struct drm_i915_gem_request *request;
 340         uint32_t ier;
 341         int flags, ret;
 342         bool recovery_complete;
 343         bool do_retire = true;
 344
 345         KASSERT(seqno != 0, ("Zero seqno"));
 346
 347         dev_priv = ring->dev->dev_private;
 348         ret = 0;
 349
 350         if (atomic_read(&dev_priv->mm.wedged) != 0) {
 351                 /* Give the error handler a chance to run. */
 352                 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE);
 353                 recovery_complete = (&dev_priv->error_completion) > 0;
 354                 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE);
 355                 return (recovery_complete ? -EIO : -EAGAIN);
 356         }
 357
 358         if (seqno == ring->outstanding_lazy_request) {
 359                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
 360                     M_WAITOK | M_ZERO);
 361                 if (request == NULL)
 362                         return (-ENOMEM);
 363
 364                 ret = i915_add_request(ring, NULL, request);
 365                 if (ret != 0) {
 366                         drm_free(request, DRM_I915_GEM);
 367                         return (ret);
 368                 }
 369
 370                 seqno = request->seqno;
 371         }
 372
 373         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
 374                 if (HAS_PCH_SPLIT(ring->dev))
 375                         ier = I915_READ(DEIER) | I915_READ(GTIER);
 376                 else
 377                         ier = I915_READ(IER);
 378                 if (!ier) {
 379                         DRM_ERROR("something (likely vbetool) disabled "
 380                                   "interrupts, re-enabling\n");
 381                         ring->dev->driver->irq_preinstall(ring->dev);
 382                         ring->dev->driver->irq_postinstall(ring->dev);
 383                 }
 384
 385                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
 386                 if (ring->irq_get(ring)) {
 387                         flags = dev_priv->mm.interruptible ? PCATCH : 0;
 388                         while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)
 389                             && !atomic_read(&dev_priv->mm.wedged) &&
 390                             ret == 0) {
 391                                 ret = -lksleep(ring, &ring->irq_lock, flags,
 392                                     "915gwr", 1*hz);
 393                         }
 394                         ring->irq_put(ring);
 395                         lockmgr(&ring->irq_lock, LK_RELEASE);
 396                 } else {
 397                         lockmgr(&ring->irq_lock, LK_RELEASE);
 398                         if (_intel_wait_for(ring->dev,
 399                             i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
 400                             atomic_read(&dev_priv->mm.wedged), 3000,
 401                             0, "i915wrq") != 0)
 402                                 ret = -EBUSY;
 403                 }
 404
 405         }
 406         if (atomic_read(&dev_priv->mm.wedged))
 407                 ret = -EAGAIN;
 408
 409         /* Directly dispatch request retiring.  While we have the work queue
 410          * to handle this, the waiter on a request often wants an associated
 411          * buffer to have made it to the inactive list, and we would need
 412          * a separate wait queue to handle that.
 413          */
 414         if (ret == 0 && do_retire)
 415                 i915_gem_retire_requests_ring(ring);
 416
 417         return (ret);
 418 }
 419
 420 /**
 421  * Ensures that all rendering to the object has completed and the object is
 422  * safe to unbind from the GTT or access from the CPU.
 423  */
 424 int
 425 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 426 {
 427         u32 seqno;
 428         int ret;
 429
 430
 431         seqno = obj->last_rendering_seqno;
 432         if (seqno == 0)
 433                 return 0;
 434
 435         if (obj->active) {
 436                 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
 437                 if (ret != 0)
 438                         return (ret);
 439         }
 440
 441         /* Manually manage the write flush as we may have not yet
 442          * retired the buffer.
 443          */
 444         if (obj->last_rendering_seqno &&
 445             i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
 446                 obj->last_rendering_seqno = 0;
 447                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
 448         }
 449
 450         return 0;
 451 }
 452
 453 /**
 454  * Called when user space prepares to use an object with the CPU, either
 455  * through the mmap ioctl's mapping or a GTT mapping.
 456  */
 457 int
 458 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 459                           struct drm_file *file)
 460 {
 461         struct drm_i915_gem_set_domain *args;
 462         struct drm_i915_gem_object *obj;
 463         uint32_t read_domains;
 464         uint32_t write_domain;
 465         int ret;
 466
 467         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 468                 return (-ENODEV);
 469
 470         args = data;
 471         read_domains = args->read_domains;
 472         write_domain = args->write_domain;
 473
 474         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
 475             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
 476             (write_domain != 0 && read_domains != write_domain))
 477                 return (-EINVAL);
 478
 479         ret = i915_mutex_lock_interruptible(dev);
 480         if (ret != 0)
 481                 return (ret);
 482
 483         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 484         if (&obj->base == NULL) {
 485                 ret = -ENOENT;
 486                 goto unlock;
 487         }
 488
 489         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
 490                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 491                 if (ret == -EINVAL)
 492                         ret = 0;
 493         } else
 494                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 495
 496         drm_gem_object_unreference(&obj->base);
 497 unlock:
 498         DRM_UNLOCK(dev);
 499         return (ret);
 500 }
 501
 502 /**
 503  * Called when user space has done writes to this buffer
 504  */
 505 int
 506 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 507                          struct drm_file *file)
 508 {
 509         struct drm_i915_gem_sw_finish *args;
 510         struct drm_i915_gem_object *obj;
 511         int ret;
 512
 513         args = data;
 514         ret = 0;
 515         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 516                 return (ENODEV);
 517         ret = i915_mutex_lock_interruptible(dev);
 518         if (ret != 0)
 519                 return (ret);
 520         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 521         if (&obj->base == NULL) {
 522                 ret = -ENOENT;
 523                 goto unlock;
 524         }
 525         if (obj->pin_count != 0)
 526                 i915_gem_object_flush_cpu_write_domain(obj);
 527         drm_gem_object_unreference(&obj->base);
 528 unlock:
 529         DRM_UNLOCK(dev);
 530         return (ret);
 531 }
 532
 533 /**
 534  * Maps the contents of an object, returning the address it is mapped
 535  * into.
 536  *
 537  * While the mapping holds a reference on the contents of the object, it doesn't
 538  * imply a ref on the object itself.
 539  */
 540 int
 541 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 542                     struct drm_file *file)
 543 {
 544         struct drm_i915_gem_mmap *args;
 545         struct drm_gem_object *obj;
 546         struct proc *p;
 547         vm_map_t map;
 548         vm_offset_t addr;
 549         vm_size_t size;
 550         int error, rv;
 551
 552         args = data;
 553
 554         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 555                 return (-ENODEV);
 556
 557         obj = drm_gem_object_lookup(dev, file, args->handle);
 558         if (obj == NULL)
 559                 return (-ENOENT);
 560         error = 0;
 561         if (args->size == 0)
 562                 goto out;
 563         p = curproc;
 564         map = &p->p_vmspace->vm_map;
 565         size = round_page(args->size);
 566         PROC_LOCK(p);
 567         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
 568                 PROC_UNLOCK(p);
 569                 error = ENOMEM;
 570                 goto out;
 571         }
 572         PROC_UNLOCK(p);
 573
 574         addr = 0;
 575         vm_object_hold(obj->vm_obj);
 576         vm_object_reference_locked(obj->vm_obj);
 577         vm_object_drop(obj->vm_obj);
 578         DRM_UNLOCK(dev);
 579         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
 580             PAGE_SIZE, /* align */
 581             TRUE, /* fitit */
 582             VM_MAPTYPE_NORMAL, /* maptype */
 583             VM_PROT_READ | VM_PROT_WRITE, /* prot */
 584             VM_PROT_READ | VM_PROT_WRITE, /* max */
 585             MAP_SHARED /* cow */);
 586         if (rv != KERN_SUCCESS) {
 587                 vm_object_deallocate(obj->vm_obj);
 588                 error = -vm_mmap_to_errno(rv);
 589         } else {
 590                 args->addr_ptr = (uint64_t)addr;
 591         }
 592         DRM_LOCK(dev);
 593 out:
 594         drm_gem_object_unreference(obj);
 595         return (error);
 596 }
 597
 598 /**
 599  * i915_gem_release_mmap - remove physical page mappings
 600  * @obj: obj in question
 601  *
 602  * Preserve the reservation of the mmapping with the DRM core code, but
 603  * relinquish ownership of the pages back to the system.
 604  *
 605  * It is vital that we remove the page mapping if we have mapped a tiled
 606  * object through the GTT and then lose the fence register due to
 607  * resource pressure. Similarly if the object has been moved out of the
 608  * aperture, than pages mapped into userspace must be revoked. Removing the
 609  * mapping will then trigger a page fault on the next user access, allowing
 610  * fixup by i915_gem_fault().
 611  */
 612 void
 613 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 614 {
 615         vm_object_t devobj;
 616         vm_page_t m;
 617         int i, page_count;
 618
 619         if (!obj->fault_mappable)
 620                 return;
 621
 622         devobj = cdev_pager_lookup(obj);
 623         if (devobj != NULL) {
 624                 page_count = OFF_TO_IDX(obj->base.size);
 625
 626                 VM_OBJECT_LOCK(devobj);
 627                 for (i = 0; i < page_count; i++) {
 628                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
 629                         if (m == NULL)
 630                                 continue;
 631                         cdev_pager_free_page(devobj, m);
 632                 }
 633                 VM_OBJECT_UNLOCK(devobj);
 634                 vm_object_deallocate(devobj);
 635         }
 636
 637         obj->fault_mappable = false;
 638 }
 639
 640 static uint32_t
 641 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 642 {
 643         uint32_t gtt_size;
 644
 645         if (INTEL_INFO(dev)->gen >= 4 ||
 646             tiling_mode == I915_TILING_NONE)
 647                 return (size);
 648
 649         /* Previous chips need a power-of-two fence region when tiling */
 650         if (INTEL_INFO(dev)->gen == 3)
 651                 gtt_size = 1024*1024;
 652         else
 653                 gtt_size = 512*1024;
 654
 655         while (gtt_size < size)
 656                 gtt_size <<= 1;
 657
 658         return (gtt_size);
 659 }
 660
 661 /**
 662  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 663  * @obj: object to check
 664  *
 665  * Return the required GTT alignment for an object, taking into account
 666  * potential fence register mapping.
 667  */
 668 static uint32_t
 669 i915_gem_get_gtt_alignment(struct drm_device *dev,
 670                            uint32_t size,
 671                            int tiling_mode)
 672 {
 673
 674         /*
 675          * Minimum alignment is 4k (GTT page size), but might be greater
 676          * if a fence register is needed for the object.
 677          */
 678         if (INTEL_INFO(dev)->gen >= 4 ||
 679             tiling_mode == I915_TILING_NONE)
 680                 return (4096);
 681
 682         /*
 683          * Previous chips need to be aligned to the size of the smallest
 684          * fence register that can contain the object.
 685          */
 686         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 687 }
 688
 689 /**
 690  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
 691  *                                       unfenced object
 692  * @dev: the device
 693  * @size: size of the object
 694  * @tiling_mode: tiling mode of the object
 695  *
 696  * Return the required GTT alignment for an object, only taking into account
 697  * unfenced tiled surface requirements.
 698  */
 699 uint32_t
 700 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 701                                     uint32_t size,
 702                                     int tiling_mode)
 703 {
 704
 705         if (tiling_mode == I915_TILING_NONE)
 706                 return (4096);
 707
 708         /*
 709          * Minimum alignment is 4k (GTT page size) for sane hw.
 710          */
 711         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
 712                 return (4096);
 713
 714         /*
 715          * Previous hardware however needs to be aligned to a power-of-two
 716          * tile height. The simplest method for determining this is to reuse
 717          * the power-of-tile object size.
 718          */
 719         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 720 }
 721
 722 int
 723 i915_gem_mmap_gtt(struct drm_file *file,
 724                   struct drm_device *dev,
 725                   uint32_t handle,
 726                   uint64_t *offset)
 727 {
 728         struct drm_i915_private *dev_priv;
 729         struct drm_i915_gem_object *obj;
 730         int ret;
 731
 732         if (!(dev->driver->driver_features & DRIVER_GEM))
 733                 return (-ENODEV);
 734
 735         dev_priv = dev->dev_private;
 736
 737         ret = i915_mutex_lock_interruptible(dev);
 738         if (ret != 0)
 739                 return (ret);
 740
 741         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 742         if (&obj->base == NULL) {
 743                 ret = -ENOENT;
 744                 goto unlock;
 745         }
 746
 747         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
 748                 ret = -E2BIG;
 749                 goto out;
 750         }
 751
 752         if (obj->madv != I915_MADV_WILLNEED) {
 753                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
 754                 ret = -EINVAL;
 755                 goto out;
 756         }
 757
 758         ret = drm_gem_create_mmap_offset(&obj->base);
 759         if (ret != 0)
 760                 goto out;
 761
 762         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
 763             DRM_GEM_MAPPING_KEY;
 764 out:
 765         drm_gem_object_unreference(&obj->base);
 766 unlock:
 767         DRM_UNLOCK(dev);
 768         return (ret);
 769 }
 770
 771 /**
 772  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 773  * @dev: DRM device
 774  * @data: GTT mapping ioctl data
 775  * @file: GEM object info
 776  *
 777  * Simply returns the fake offset to userspace so it can mmap it.
 778  * The mmap call will end up in drm_gem_mmap(), which will set things
 779  * up so we can get faults in the handler above.
 780  *
 781  * The fault handler will take care of binding the object into the GTT
 782  * (since it may have been evicted to make room for something), allocating
 783  * a fence register, and mapping the appropriate aperture address into
 784  * userspace.
 785  */
 786 int
 787 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 788                         struct drm_file *file)
 789 {
 790         struct drm_i915_private *dev_priv;
 791         struct drm_i915_gem_mmap_gtt *args;
 792
 793         dev_priv = dev->dev_private;
 794         args = data;
 795
 796         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
 797 }
 798
 799 /* Immediately discard the backing storage */
 800 static void
 801 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 802 {
 803         vm_object_t vm_obj;
 804
 805         vm_obj = obj->base.vm_obj;
 806         VM_OBJECT_LOCK(vm_obj);
 807         vm_object_page_remove(vm_obj, 0, 0, false);
 808         VM_OBJECT_UNLOCK(vm_obj);
 809         obj->madv = __I915_MADV_PURGED;
 810 }
 811
 812 static inline int
 813 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 814 {
 815         return obj->madv == I915_MADV_DONTNEED;
 816 }
 817
 818 static inline void vm_page_reference(vm_page_t m)
 819 {
 820         vm_page_flag_set(m, PG_REFERENCED);
 821 }
 822
 823 static void
 824 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 825 {
 826         vm_page_t m;
 827         int page_count, i;
 828
 829         BUG_ON(obj->madv == __I915_MADV_PURGED);
 830
 831         if (obj->tiling_mode != I915_TILING_NONE)
 832                 i915_gem_object_save_bit_17_swizzle(obj);
 833         if (obj->madv == I915_MADV_DONTNEED)
 834                 obj->dirty = 0;
 835         page_count = obj->base.size / PAGE_SIZE;
 836         VM_OBJECT_LOCK(obj->base.vm_obj);
 837 #if GEM_PARANOID_CHECK_GTT
 838         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
 839 #endif
 840         for (i = 0; i < page_count; i++) {
 841                 m = obj->pages[i];
 842                 if (obj->dirty)
 843                         vm_page_dirty(m);
 844                 if (obj->madv == I915_MADV_WILLNEED)
 845                         vm_page_reference(m);
 846                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
 847                 vm_page_unwire(obj->pages[i], 1);
 848                 vm_page_wakeup(obj->pages[i]);
 849                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 850         }
 851         VM_OBJECT_UNLOCK(obj->base.vm_obj);
 852         obj->dirty = 0;
 853         drm_free(obj->pages, DRM_I915_GEM);
 854         obj->pages = NULL;
 855 }
 856
 857 static int
 858 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 859     int flags)
 860 {
 861         struct drm_device *dev;
 862         vm_object_t vm_obj;
 863         vm_page_t m;
 864         int page_count, i, j;
 865
 866         dev = obj->base.dev;
 867         KASSERT(obj->pages == NULL, ("Obj already has pages"));
 868         page_count = obj->base.size / PAGE_SIZE;
 869         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 870             M_WAITOK);
 871         vm_obj = obj->base.vm_obj;
 872         VM_OBJECT_LOCK(vm_obj);
 873         for (i = 0; i < page_count; i++) {
 874                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
 875                         goto failed;
 876         }
 877         VM_OBJECT_UNLOCK(vm_obj);
 878         if (i915_gem_object_needs_bit17_swizzle(obj))
 879                 i915_gem_object_do_bit_17_swizzle(obj);
 880         return (0);
 881
 882 failed:
 883         for (j = 0; j < i; j++) {
 884                 m = obj->pages[j];
 885                 vm_page_busy_wait(m, FALSE, "i915gem");
 886                 vm_page_unwire(m, 0);
 887                 vm_page_wakeup(m);
 888                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 889         }
 890         VM_OBJECT_UNLOCK(vm_obj);
 891         drm_free(obj->pages, DRM_I915_GEM);
 892         obj->pages = NULL;
 893         return (-EIO);
 894 }
 895
 896 void
 897 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 898     struct intel_ring_buffer *ring, uint32_t seqno)
 899 {
 900         struct drm_device *dev = obj->base.dev;
 901         struct drm_i915_private *dev_priv = dev->dev_private;
 902         struct drm_i915_fence_reg *reg;
 903
 904         obj->ring = ring;
 905         KASSERT(ring != NULL, ("NULL ring"));
 906
 907         /* Add a reference if we're newly entering the active list. */
 908         if (!obj->active) {
 909                 drm_gem_object_reference(&obj->base);
 910                 obj->active = 1;
 911         }
 912
 913         /* Move from whatever list we were on to the tail of execution. */
 914         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 915         list_move_tail(&obj->ring_list, &ring->active_list);
 916
 917         obj->last_rendering_seqno = seqno;
 918         if (obj->fenced_gpu_access) {
 919                 obj->last_fenced_seqno = seqno;
 920                 obj->last_fenced_ring = ring;
 921
 922                 /* Bump MRU to take account of the delayed flush */
 923                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
 924                         reg = &dev_priv->fence_regs[obj->fence_reg];
 925                         list_move_tail(&reg->lru_list,
 926                                        &dev_priv->mm.fence_list);
 927                 }
 928         }
 929 }
 930
 931 static void
 932 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 933 {
 934         list_del_init(&obj->ring_list);
 935         obj->last_rendering_seqno = 0;
 936         obj->last_fenced_seqno = 0;
 937 }
 938
 939 static void
 940 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 941 {
 942         struct drm_device *dev = obj->base.dev;
 943         struct drm_i915_private *dev_priv = dev->dev_private;
 944
 945         if (obj->pin_count != 0)
 946                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
 947         else
 948                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 949
 950         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
 951         KASSERT(obj->active, ("Object not active"));
 952         obj->ring = NULL;
 953         obj->last_fenced_ring = NULL;
 954
 955         i915_gem_object_move_off_active(obj);
 956         obj->fenced_gpu_access = false;
 957
 958         obj->active = 0;
 959         obj->pending_gpu_write = false;
 960         drm_gem_object_unreference(&obj->base);
 961
 962 #if 1
 963         KIB_NOTYET();
 964 #else
 965         WARN_ON(i915_verify_lists(dev));
 966 #endif
 967 }
 968
 969 static u32
 970 i915_gem_get_seqno(struct drm_device *dev)
 971 {
 972         drm_i915_private_t *dev_priv = dev->dev_private;
 973         u32 seqno = dev_priv->next_seqno;
 974
 975         /* reserve 0 for non-seqno */
 976         if (++dev_priv->next_seqno == 0)
 977                 dev_priv->next_seqno = 1;
 978
 979         return seqno;
 980 }
 981
 982 int
 983 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
 984      struct drm_i915_gem_request *request)
 985 {
 986         drm_i915_private_t *dev_priv;
 987         struct drm_i915_file_private *file_priv;
 988         uint32_t seqno;
 989         u32 request_ring_position;
 990         int was_empty;
 991         int ret;
 992
 993         KASSERT(request != NULL, ("NULL request in add"));
 994         DRM_LOCK_ASSERT(ring->dev);
 995         dev_priv = ring->dev->dev_private;
 996
 997         seqno = i915_gem_next_request_seqno(ring);
 998         request_ring_position = intel_ring_get_tail(ring);
 999
1000         ret = ring->add_request(ring, &seqno);
1001         if (ret != 0)
1002             return ret;
1003
1004         request->seqno = seqno;
1005         request->ring = ring;
1006         request->tail = request_ring_position;
1007         request->emitted_jiffies = ticks;
1008         was_empty = list_empty(&ring->request_list);
1009         list_add_tail(&request->list, &ring->request_list);
1010
1011         if (file != NULL) {
1012                 file_priv = file->driver_priv;
1013
1014                 spin_lock(&file_priv->mm.lock);
1015                 request->file_priv = file_priv;
1016                 list_add_tail(&request->client_list,
1017                     &file_priv->mm.request_list);
1018                 spin_unlock(&file_priv->mm.lock);
1019         }
1020
1021         ring->outstanding_lazy_request = 0;
1022
1023         if (!dev_priv->mm.suspended) {
1024                 if (i915_enable_hangcheck) {
1025                         mod_timer(&dev_priv->hangcheck_timer,
1026                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
1027                 }
1028                 if (was_empty) {
1029                         queue_delayed_work(dev_priv->wq,
1030                                            &dev_priv->mm.retire_work,
1031                                            round_jiffies_up_relative(hz));
1032                         intel_mark_busy(dev_priv->dev);
1033                 }
1034         }
1035         return (0);
1036 }
1037
1038 static inline void
1039 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1040 {
1041         struct drm_i915_file_private *file_priv = request->file_priv;
1042
1043         if (!file_priv)
1044                 return;
1045
1046         DRM_LOCK_ASSERT(request->ring->dev);
1047
1048         spin_lock(&file_priv->mm.lock);
1049         if (request->file_priv != NULL) {
1050                 list_del(&request->client_list);
1051                 request->file_priv = NULL;
1052         }
1053         spin_unlock(&file_priv->mm.lock);
1054 }
1055
1056 static void
1057 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1058     struct intel_ring_buffer *ring)
1059 {
1060
1061         if (ring->dev != NULL)
1062                 DRM_LOCK_ASSERT(ring->dev);
1063
1064         while (!list_empty(&ring->request_list)) {
1065                 struct drm_i915_gem_request *request;
1066
1067                 request = list_first_entry(&ring->request_list,
1068                     struct drm_i915_gem_request, list);
1069
1070                 list_del(&request->list);
1071                 i915_gem_request_remove_from_client(request);
1072                 drm_free(request, DRM_I915_GEM);
1073         }
1074
1075         while (!list_empty(&ring->active_list)) {
1076                 struct drm_i915_gem_object *obj;
1077
1078                 obj = list_first_entry(&ring->active_list,
1079                     struct drm_i915_gem_object, ring_list);
1080
1081                 obj->base.write_domain = 0;
1082                 list_del_init(&obj->gpu_write_list);
1083                 i915_gem_object_move_to_inactive(obj);
1084         }
1085 }
1086
1087 static void
1088 i915_gem_reset_fences(struct drm_device *dev)
1089 {
1090         struct drm_i915_private *dev_priv = dev->dev_private;
1091         int i;
1092
1093         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1094                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1095                 struct drm_i915_gem_object *obj = reg->obj;
1096
1097                 if (!obj)
1098                         continue;
1099
1100                 if (obj->tiling_mode)
1101                         i915_gem_release_mmap(obj);
1102
1103                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1104                 reg->obj->fenced_gpu_access = false;
1105                 reg->obj->last_fenced_seqno = 0;
1106                 reg->obj->last_fenced_ring = NULL;
1107                 i915_gem_clear_fence_reg(dev, reg);
1108         }
1109 }
1110
1111 void i915_gem_reset(struct drm_device *dev)
1112 {
1113         struct drm_i915_private *dev_priv = dev->dev_private;
1114         struct drm_i915_gem_object *obj;
1115         int i;
1116
1117         for (i = 0; i < I915_NUM_RINGS; i++)
1118                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1119
1120         /* Remove anything from the flushing lists. The GPU cache is likely
1121          * to be lost on reset along with the data, so simply move the
1122          * lost bo to the inactive list.
1123          */
1124         while (!list_empty(&dev_priv->mm.flushing_list)) {
1125                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1126                                       struct drm_i915_gem_object,
1127                                       mm_list);
1128
1129                 obj->base.write_domain = 0;
1130                 list_del_init(&obj->gpu_write_list);
1131                 i915_gem_object_move_to_inactive(obj);
1132         }
1133
1134         /* Move everything out of the GPU domains to ensure we do any
1135          * necessary invalidation upon reuse.
1136          */
1137         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
1138                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1139         }
1140
1141         /* The fence registers are invalidated so clear them out */
1142         i915_gem_reset_fences(dev);
1143 }
1144
1145 static void
1146 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1147 {
1148         struct drm_device *dev = obj->base.dev;
1149         drm_i915_private_t *dev_priv = dev->dev_private;
1150
1151         KASSERT(obj->active, ("Object not active"));
1152         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1153
1154         i915_gem_object_move_off_active(obj);
1155 }
1156
1157 /**
1158  * This function clears the request list as sequence numbers are passed.
1159  */
1160 void
1161 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1162 {
1163         uint32_t seqno;
1164
1165         if (list_empty(&ring->request_list))
1166                 return;
1167
1168         seqno = ring->get_seqno(ring, true);
1169
1170         while (!list_empty(&ring->request_list)) {
1171                 struct drm_i915_gem_request *request;
1172
1173                 request = list_first_entry(&ring->request_list,
1174                                            struct drm_i915_gem_request,
1175                                            list);
1176
1177                 if (!i915_seqno_passed(seqno, request->seqno))
1178                         break;
1179
1180                 /* We know the GPU must have read the request to have
1181                  * sent us the seqno + interrupt, so use the position
1182                  * of tail of the request to update the last known position
1183                  * of the GPU head.
1184                  */
1185                 ring->last_retired_head = request->tail;
1186
1187                 list_del(&request->list);
1188                 i915_gem_request_remove_from_client(request);
1189                 drm_free(request, DRM_I915_GEM);
1190         }
1191
1192         /* Move any buffers on the active list that are no longer referenced
1193          * by the ringbuffer to the flushing/inactive lists as appropriate.
1194          */
1195         while (!list_empty(&ring->active_list)) {
1196                 struct drm_i915_gem_object *obj;
1197
1198                 obj = list_first_entry(&ring->active_list,
1199                                       struct drm_i915_gem_object,
1200                                       ring_list);
1201
1202                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1203                         break;
1204
1205                 if (obj->base.write_domain != 0)
1206                         i915_gem_object_move_to_flushing(obj);
1207                 else
1208                         i915_gem_object_move_to_inactive(obj);
1209         }
1210
1211         if (unlikely(ring->trace_irq_seqno &&
1212                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1213                 ring->irq_put(ring);
1214                 ring->trace_irq_seqno = 0;
1215         }
1216
1217 }
1218
1219 static void
1220 i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
1221
1222 void
1223 i915_gem_retire_requests(struct drm_device *dev)
1224 {
1225         drm_i915_private_t *dev_priv = dev->dev_private;
1226         struct drm_i915_gem_object *obj, *next;
1227         int i;
1228
1229         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1230                 list_for_each_entry_safe(obj, next,
1231                     &dev_priv->mm.deferred_free_list, mm_list)
1232                         i915_gem_free_object_tail(obj);
1233         }
1234
1235         for (i = 0; i < I915_NUM_RINGS; i++)
1236                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1237 }
1238
1239 static void
1240 i915_gem_retire_work_handler(struct work_struct *work)
1241 {
1242         drm_i915_private_t *dev_priv;
1243         struct drm_device *dev;
1244         struct intel_ring_buffer *ring;
1245         bool idle;
1246         int i;
1247
1248         dev_priv = container_of(work, drm_i915_private_t,
1249                                 mm.retire_work.work);
1250         dev = dev_priv->dev;
1251
1252         /* Come back later if the device is busy... */
1253         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
1254                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1255                                    round_jiffies_up_relative(hz));
1256                 return;
1257         }
1258
1259         i915_gem_retire_requests(dev);
1260
1261         /* Send a periodic flush down the ring so we don't hold onto GEM
1262          * objects indefinitely.
1263          */
1264         idle = true;
1265         for_each_ring(ring, dev_priv, i) {
1266                 if (ring->gpu_caches_dirty)
1267                         i915_add_request(ring, NULL, NULL);
1268
1269                 idle &= list_empty(&ring->request_list);
1270         }
1271
1272         if (!dev_priv->mm.suspended && !idle)
1273                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1274                                    round_jiffies_up_relative(hz));
1275         if (idle)
1276                 intel_mark_idle(dev);
1277
1278         DRM_UNLOCK(dev);
1279 }
1280
1281 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1282 {
1283         u32 old_write_domain, old_read_domains;
1284
1285         /* Act a barrier for all accesses through the GTT */
1286         cpu_mfence();
1287
1288         /* Force a pagefault for domain tracking on next user access */
1289         i915_gem_release_mmap(obj);
1290
1291         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1292                 return;
1293
1294         old_read_domains = obj->base.read_domains;
1295         old_write_domain = obj->base.write_domain;
1296
1297         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1298         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1299
1300 }
1301
1302 int
1303 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1304 {
1305         drm_i915_private_t *dev_priv;
1306         int ret;
1307
1308         dev_priv = obj->base.dev->dev_private;
1309         ret = 0;
1310         if (obj->gtt_space == NULL)
1311                 return (0);
1312         if (obj->pin_count != 0) {
1313                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1314                 return (-EINVAL);
1315         }
1316
1317         ret = i915_gem_object_finish_gpu(obj);
1318         if (ret == -ERESTART || ret == -EINTR)
1319                 return (ret);
1320
1321         i915_gem_object_finish_gtt(obj);
1322
1323         if (ret == 0)
1324                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1325         if (ret == -ERESTART || ret == -EINTR)
1326                 return (ret);
1327         if (ret != 0) {
1328                 i915_gem_clflush_object(obj);
1329                 obj->base.read_domains = obj->base.write_domain =
1330                     I915_GEM_DOMAIN_CPU;
1331         }
1332
1333         ret = i915_gem_object_put_fence(obj);
1334         if (ret == -ERESTART)
1335                 return (ret);
1336
1337         i915_gem_gtt_unbind_object(obj);
1338         if (obj->has_aliasing_ppgtt_mapping) {
1339                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
1340                 obj->has_aliasing_ppgtt_mapping = 0;
1341         }
1342         i915_gem_object_put_pages_gtt(obj);
1343
1344         list_del_init(&obj->gtt_list);
1345         list_del_init(&obj->mm_list);
1346         obj->map_and_fenceable = true;
1347
1348         drm_mm_put_block(obj->gtt_space);
1349         obj->gtt_space = NULL;
1350         obj->gtt_offset = 0;
1351
1352         if (i915_gem_object_is_purgeable(obj))
1353                 i915_gem_object_truncate(obj);
1354
1355         return (ret);
1356 }
1357
1358 int i915_gpu_idle(struct drm_device *dev)
1359 {
1360         drm_i915_private_t *dev_priv = dev->dev_private;
1361         struct intel_ring_buffer *ring;
1362         int ret, i;
1363
1364         /* Flush everything onto the inactive list. */
1365         for_each_ring(ring, dev_priv, i) {
1366                 ret = intel_ring_idle(ring);
1367                 if (ret)
1368                         return ret;
1369         }
1370
1371         return 0;
1372 }
1373
1374 static int
1375 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
1376     struct intel_ring_buffer *pipelined)
1377 {
1378         struct drm_device *dev = obj->base.dev;
1379         drm_i915_private_t *dev_priv = dev->dev_private;
1380         u32 size = obj->gtt_space->size;
1381         int regnum = obj->fence_reg;
1382         uint64_t val;
1383
1384         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1385                          0xfffff000) << 32;
1386         val |= obj->gtt_offset & 0xfffff000;
1387         val |= (uint64_t)((obj->stride / 128) - 1) <<
1388                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
1389
1390         if (obj->tiling_mode == I915_TILING_Y)
1391                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1392         val |= I965_FENCE_REG_VALID;
1393
1394         if (pipelined) {
1395                 int ret = intel_ring_begin(pipelined, 6);
1396                 if (ret)
1397                         return ret;
1398
1399                 intel_ring_emit(pipelined, MI_NOOP);
1400                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1401                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
1402                 intel_ring_emit(pipelined, (u32)val);
1403                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
1404                 intel_ring_emit(pipelined, (u32)(val >> 32));
1405                 intel_ring_advance(pipelined);
1406         } else
1407                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
1408
1409         return 0;
1410 }
1411
1412 static int
1413 i965_write_fence_reg(struct drm_i915_gem_object *obj,
1414     struct intel_ring_buffer *pipelined)
1415 {
1416         struct drm_device *dev = obj->base.dev;
1417         drm_i915_private_t *dev_priv = dev->dev_private;
1418         u32 size = obj->gtt_space->size;
1419         int regnum = obj->fence_reg;
1420         uint64_t val;
1421
1422         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1423                     0xfffff000) << 32;
1424         val |= obj->gtt_offset & 0xfffff000;
1425         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1426         if (obj->tiling_mode == I915_TILING_Y)
1427                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1428         val |= I965_FENCE_REG_VALID;
1429
1430         if (pipelined) {
1431                 int ret = intel_ring_begin(pipelined, 6);
1432                 if (ret)
1433                         return ret;
1434
1435                 intel_ring_emit(pipelined, MI_NOOP);
1436                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1437                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
1438                 intel_ring_emit(pipelined, (u32)val);
1439                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
1440                 intel_ring_emit(pipelined, (u32)(val >> 32));
1441                 intel_ring_advance(pipelined);
1442         } else
1443                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
1444
1445         return 0;
1446 }
1447
1448 static int
1449 i915_write_fence_reg(struct drm_i915_gem_object *obj,
1450     struct intel_ring_buffer *pipelined)
1451 {
1452         struct drm_device *dev = obj->base.dev;
1453         drm_i915_private_t *dev_priv = dev->dev_private;
1454         u32 size = obj->gtt_space->size;
1455         u32 fence_reg, val, pitch_val;
1456         int tile_width;
1457
1458         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
1459             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1460                 kprintf(
1461 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1462                  obj->gtt_offset, obj->map_and_fenceable, size);
1463                 return -EINVAL;
1464         }
1465
1466         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1467                 tile_width = 128;
1468         else
1469                 tile_width = 512;
1470
1471         /* Note: pitch better be a power of two tile widths */
1472         pitch_val = obj->stride / tile_width;
1473         pitch_val = ffs(pitch_val) - 1;
1474
1475         val = obj->gtt_offset;
1476         if (obj->tiling_mode == I915_TILING_Y)
1477                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1478         val |= I915_FENCE_SIZE_BITS(size);
1479         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1480         val |= I830_FENCE_REG_VALID;
1481
1482         fence_reg = obj->fence_reg;
1483         if (fence_reg < 8)
1484                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
1485         else
1486                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
1487
1488         if (pipelined) {
1489                 int ret = intel_ring_begin(pipelined, 4);
1490                 if (ret)
1491                         return ret;
1492
1493                 intel_ring_emit(pipelined, MI_NOOP);
1494                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1495                 intel_ring_emit(pipelined, fence_reg);
1496                 intel_ring_emit(pipelined, val);
1497                 intel_ring_advance(pipelined);
1498         } else
1499                 I915_WRITE(fence_reg, val);
1500
1501         return 0;
1502 }
1503
1504 static int
1505 i830_write_fence_reg(struct drm_i915_gem_object *obj,
1506     struct intel_ring_buffer *pipelined)
1507 {
1508         struct drm_device *dev = obj->base.dev;
1509         drm_i915_private_t *dev_priv = dev->dev_private;
1510         u32 size = obj->gtt_space->size;
1511         int regnum = obj->fence_reg;
1512         uint32_t val;
1513         uint32_t pitch_val;
1514
1515         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
1516             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1517                 kprintf(
1518 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1519                     obj->gtt_offset, size);
1520                 return -EINVAL;
1521         }
1522
1523         pitch_val = obj->stride / 128;
1524         pitch_val = ffs(pitch_val) - 1;
1525
1526         val = obj->gtt_offset;
1527         if (obj->tiling_mode == I915_TILING_Y)
1528                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1529         val |= I830_FENCE_SIZE_BITS(size);
1530         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1531         val |= I830_FENCE_REG_VALID;
1532
1533         if (pipelined) {
1534                 int ret = intel_ring_begin(pipelined, 4);
1535                 if (ret)
1536                         return ret;
1537
1538                 intel_ring_emit(pipelined, MI_NOOP);
1539                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1540                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
1541                 intel_ring_emit(pipelined, val);
1542                 intel_ring_advance(pipelined);
1543         } else
1544                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
1545
1546         return 0;
1547 }
1548
1549 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
1550 {
1551         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
1552 }
1553
1554 static int
1555 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
1556     struct intel_ring_buffer *pipelined)
1557 {
1558         int ret;
1559
1560         if (obj->fenced_gpu_access) {
1561                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1562                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
1563                             obj->base.write_domain);
1564                         if (ret)
1565                                 return ret;
1566                 }
1567
1568                 obj->fenced_gpu_access = false;
1569         }
1570
1571         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
1572                 if (!ring_passed_seqno(obj->last_fenced_ring,
1573                                        obj->last_fenced_seqno)) {
1574                         ret = i915_wait_seqno(obj->last_fenced_ring,
1575                                                 obj->last_fenced_seqno);
1576                         if (ret)
1577                                 return ret;
1578                 }
1579
1580                 obj->last_fenced_seqno = 0;
1581                 obj->last_fenced_ring = NULL;
1582         }
1583
1584         /* Ensure that all CPU reads are completed before installing a fence
1585          * and all writes before removing the fence.
1586          */
1587         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
1588                 cpu_mfence();
1589
1590         return 0;
1591 }
1592
1593 int
1594 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
1595 {
1596         int ret;
1597
1598         if (obj->tiling_mode)
1599                 i915_gem_release_mmap(obj);
1600
1601         ret = i915_gem_object_flush_fence(obj, NULL);
1602         if (ret)
1603                 return ret;
1604
1605         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1606                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1607
1608                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
1609                         kprintf("%s: pin_count %d\n", __func__,
1610                             dev_priv->fence_regs[obj->fence_reg].pin_count);
1611                 i915_gem_clear_fence_reg(obj->base.dev,
1612                                          &dev_priv->fence_regs[obj->fence_reg]);
1613
1614                 obj->fence_reg = I915_FENCE_REG_NONE;
1615         }
1616
1617         return 0;
1618 }
1619
1620 static struct drm_i915_fence_reg *
1621 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
1622 {
1623         struct drm_i915_private *dev_priv = dev->dev_private;
1624         struct drm_i915_fence_reg *reg, *first, *avail;
1625         int i;
1626
1627         /* First try to find a free reg */
1628         avail = NULL;
1629         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
1630                 reg = &dev_priv->fence_regs[i];
1631                 if (!reg->obj)
1632                         return reg;
1633
1634                 if (!reg->pin_count)
1635                         avail = reg;
1636         }
1637
1638         if (avail == NULL)
1639                 return NULL;
1640
1641         /* None available, try to steal one or wait for a user to finish */
1642         avail = first = NULL;
1643         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1644                 if (reg->pin_count)
1645                         continue;
1646
1647                 if (first == NULL)
1648                         first = reg;
1649
1650                 if (!pipelined ||
1651                     !reg->obj->last_fenced_ring ||
1652                     reg->obj->last_fenced_ring == pipelined) {
1653                         avail = reg;
1654                         break;
1655                 }
1656         }
1657
1658         if (avail == NULL)
1659                 avail = first;
1660
1661         return avail;
1662 }
1663
1664 int
1665 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1666     struct intel_ring_buffer *pipelined)
1667 {
1668         struct drm_device *dev = obj->base.dev;
1669         struct drm_i915_private *dev_priv = dev->dev_private;
1670         struct drm_i915_fence_reg *reg;
1671         int ret;
1672
1673         pipelined = NULL;
1674         ret = 0;
1675
1676         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1677                 reg = &dev_priv->fence_regs[obj->fence_reg];
1678                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1679
1680                 if (obj->tiling_changed) {
1681                         ret = i915_gem_object_flush_fence(obj, pipelined);
1682                         if (ret)
1683                                 return ret;
1684
1685                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1686                                 pipelined = NULL;
1687
1688                         if (pipelined) {
1689                                 reg->setup_seqno =
1690                                         i915_gem_next_request_seqno(pipelined);
1691                                 obj->last_fenced_seqno = reg->setup_seqno;
1692                                 obj->last_fenced_ring = pipelined;
1693                         }
1694
1695                         goto update;
1696                 }
1697
1698                 if (!pipelined) {
1699                         if (reg->setup_seqno) {
1700                                 if (!ring_passed_seqno(obj->last_fenced_ring,
1701                                     reg->setup_seqno)) {
1702                                         ret = i915_wait_seqno(
1703                                             obj->last_fenced_ring,
1704                                             reg->setup_seqno);
1705                                         if (ret)
1706                                                 return ret;
1707                                 }
1708
1709                                 reg->setup_seqno = 0;
1710                         }
1711                 } else if (obj->last_fenced_ring &&
1712                            obj->last_fenced_ring != pipelined) {
1713                         ret = i915_gem_object_flush_fence(obj, pipelined);
1714                         if (ret)
1715                                 return ret;
1716                 }
1717
1718                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1719                         pipelined = NULL;
1720                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
1721
1722                 if (obj->tiling_changed) {
1723                         if (pipelined) {
1724                                 reg->setup_seqno =
1725                                         i915_gem_next_request_seqno(pipelined);
1726                                 obj->last_fenced_seqno = reg->setup_seqno;
1727                                 obj->last_fenced_ring = pipelined;
1728                         }
1729                         goto update;
1730                 }
1731
1732                 return 0;
1733         }
1734
1735         reg = i915_find_fence_reg(dev, pipelined);
1736         if (reg == NULL)
1737                 return -EDEADLK;
1738
1739         ret = i915_gem_object_flush_fence(obj, pipelined);
1740         if (ret)
1741                 return ret;
1742
1743         if (reg->obj) {
1744                 struct drm_i915_gem_object *old = reg->obj;
1745
1746                 drm_gem_object_reference(&old->base);
1747
1748                 if (old->tiling_mode)
1749                         i915_gem_release_mmap(old);
1750
1751                 ret = i915_gem_object_flush_fence(old, pipelined);
1752                 if (ret) {
1753                         drm_gem_object_unreference(&old->base);
1754                         return ret;
1755                 }
1756
1757                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
1758                         pipelined = NULL;
1759
1760                 old->fence_reg = I915_FENCE_REG_NONE;
1761                 old->last_fenced_ring = pipelined;
1762                 old->last_fenced_seqno =
1763                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1764
1765                 drm_gem_object_unreference(&old->base);
1766         } else if (obj->last_fenced_seqno == 0)
1767                 pipelined = NULL;
1768
1769         reg->obj = obj;
1770         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1771         obj->fence_reg = reg - dev_priv->fence_regs;
1772         obj->last_fenced_ring = pipelined;
1773
1774         reg->setup_seqno =
1775                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1776         obj->last_fenced_seqno = reg->setup_seqno;
1777
1778 update:
1779         obj->tiling_changed = false;
1780         switch (INTEL_INFO(dev)->gen) {
1781         case 7:
1782         case 6:
1783                 ret = sandybridge_write_fence_reg(obj, pipelined);
1784                 break;
1785         case 5:
1786         case 4:
1787                 ret = i965_write_fence_reg(obj, pipelined);
1788                 break;
1789         case 3:
1790                 ret = i915_write_fence_reg(obj, pipelined);
1791                 break;
1792         case 2:
1793                 ret = i830_write_fence_reg(obj, pipelined);
1794                 break;
1795         }
1796
1797         return ret;
1798 }
1799
1800 static int
1801 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1802     unsigned alignment, bool map_and_fenceable)
1803 {
1804         struct drm_device *dev;
1805         struct drm_i915_private *dev_priv;
1806         struct drm_mm_node *free_space;
1807         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1808         bool mappable, fenceable;
1809         int ret;
1810
1811         dev = obj->base.dev;
1812         dev_priv = dev->dev_private;
1813
1814         if (obj->madv != I915_MADV_WILLNEED) {
1815                 DRM_ERROR("Attempting to bind a purgeable object\n");
1816                 return (-EINVAL);
1817         }
1818
1819         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1820             obj->tiling_mode);
1821         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1822             obj->tiling_mode);
1823         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1824             obj->base.size, obj->tiling_mode);
1825         if (alignment == 0)
1826                 alignment = map_and_fenceable ? fence_alignment :
1827                     unfenced_alignment;
1828         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1829                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1830                 return (-EINVAL);
1831         }
1832
1833         size = map_and_fenceable ? fence_size : obj->base.size;
1834
1835         /* If the object is bigger than the entire aperture, reject it early
1836          * before evicting everything in a vain attempt to find space.
1837          */
1838         if (obj->base.size > (map_and_fenceable ?
1839             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1840                 DRM_ERROR(
1841 "Attempting to bind an object larger than the aperture\n");
1842                 return (-E2BIG);
1843         }
1844
1845  search_free:
1846         if (map_and_fenceable)
1847                 free_space = drm_mm_search_free_in_range(
1848                     &dev_priv->mm.gtt_space, size, alignment, 0,
1849                     dev_priv->mm.gtt_mappable_end, 0);
1850         else
1851                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1852                     size, alignment, 0);
1853         if (free_space != NULL) {
1854                 int color = 0;
1855                 if (map_and_fenceable)
1856                         obj->gtt_space = drm_mm_get_block_range_generic(
1857                             free_space, size, alignment, color, 0,
1858                             dev_priv->mm.gtt_mappable_end, 1);
1859                 else
1860                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1861                             size, alignment, color, 1);
1862         }
1863         if (obj->gtt_space == NULL) {
1864                 ret = i915_gem_evict_something(dev, size, alignment,
1865                     map_and_fenceable);
1866                 if (ret != 0)
1867                         return (ret);
1868                 goto search_free;
1869         }
1870
1871         /*
1872          * NOTE: i915_gem_object_get_pages_gtt() cannot
1873          *       return ENOMEM, since we used VM_ALLOC_RETRY.
1874          */
1875         ret = i915_gem_object_get_pages_gtt(obj, 0);
1876         if (ret != 0) {
1877                 drm_mm_put_block(obj->gtt_space);
1878                 obj->gtt_space = NULL;
1879                 return (ret);
1880         }
1881
1882         i915_gem_gtt_bind_object(obj, obj->cache_level);
1883         if (ret != 0) {
1884                 i915_gem_object_put_pages_gtt(obj);
1885                 drm_mm_put_block(obj->gtt_space);
1886                 obj->gtt_space = NULL;
1887                 if (i915_gem_evict_everything(dev, false))
1888                         return (ret);
1889                 goto search_free;
1890         }
1891
1892         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1893         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1894
1895         obj->gtt_offset = obj->gtt_space->start;
1896
1897         fenceable =
1898                 obj->gtt_space->size == fence_size &&
1899                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1900
1901         mappable =
1902                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1903         obj->map_and_fenceable = mappable && fenceable;
1904
1905         return (0);
1906 }
1907
1908 void
1909 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1910 {
1911
1912         /* If we don't have a page list set up, then we're not pinned
1913          * to GPU, and we can ignore the cache flush because it'll happen
1914          * again at bind time.
1915          */
1916         if (obj->pages == NULL)
1917                 return;
1918
1919         /* If the GPU is snooping the contents of the CPU cache,
1920          * we do not need to manually clear the CPU cache lines.  However,
1921          * the caches are only snooped when the render cache is
1922          * flushed/invalidated.  As we always have to emit invalidations
1923          * and flushes when moving into and out of the RENDER domain, correct
1924          * snooping behaviour occurs naturally as the result of our domain
1925          * tracking.
1926          */
1927         if (obj->cache_level != I915_CACHE_NONE)
1928                 return;
1929
1930         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
1931 }
1932
1933 /** Flushes the GTT write domain for the object if it's dirty. */
1934 static void
1935 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
1936 {
1937         uint32_t old_write_domain;
1938
1939         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
1940                 return;
1941
1942         /* No actual flushing is required for the GTT write domain.  Writes
1943          * to it immediately go to main memory as far as we know, so there's
1944          * no chipset flush.  It also doesn't land in render cache.
1945          *
1946          * However, we do have to enforce the order so that all writes through
1947          * the GTT land before any writes to the device, such as updates to
1948          * the GATT itself.
1949          */
1950         cpu_sfence();
1951
1952         old_write_domain = obj->base.write_domain;
1953         obj->base.write_domain = 0;
1954 }
1955
1956 /** Flushes the CPU write domain for the object if it's dirty. */
1957 static void
1958 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
1959 {
1960         uint32_t old_write_domain;
1961
1962         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
1963                 return;
1964
1965         i915_gem_clflush_object(obj);
1966         intel_gtt_chipset_flush();
1967         old_write_domain = obj->base.write_domain;
1968         obj->base.write_domain = 0;
1969 }
1970
1971 static int
1972 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
1973 {
1974
1975         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
1976                 return (0);
1977         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
1978 }
1979
1980 /**
1981  * Moves a single object to the GTT read, and possibly write domain.
1982  *
1983  * This function returns when the move is complete, including waiting on
1984  * flushes to occur.
1985  */
1986 int
1987 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
1988 {
1989         uint32_t old_write_domain, old_read_domains;
1990         int ret;
1991
1992         if (obj->gtt_space == NULL)
1993                 return (-EINVAL);
1994
1995         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
1996                 return 0;
1997
1998         ret = i915_gem_object_flush_gpu_write_domain(obj);
1999         if (ret != 0)
2000                 return (ret);
2001
2002         if (obj->pending_gpu_write || write) {
2003                 ret = i915_gem_object_wait_rendering(obj);
2004                 if (ret != 0)
2005                         return (ret);
2006         }
2007
2008         i915_gem_object_flush_cpu_write_domain(obj);
2009
2010         old_write_domain = obj->base.write_domain;
2011         old_read_domains = obj->base.read_domains;
2012
2013         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2014             ("In GTT write domain"));
2015         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2016         if (write) {
2017                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2018                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2019                 obj->dirty = 1;
2020         }
2021
2022         return (0);
2023 }
2024
2025 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2026                                     enum i915_cache_level cache_level)
2027 {
2028         struct drm_device *dev = obj->base.dev;
2029         drm_i915_private_t *dev_priv = dev->dev_private;
2030         int ret;
2031
2032         if (obj->cache_level == cache_level)
2033                 return 0;
2034
2035         if (obj->pin_count) {
2036                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2037                 return -EBUSY;
2038         }
2039
2040         if (obj->gtt_space) {
2041                 ret = i915_gem_object_finish_gpu(obj);
2042                 if (ret != 0)
2043                         return (ret);
2044
2045                 i915_gem_object_finish_gtt(obj);
2046
2047                 /* Before SandyBridge, you could not use tiling or fence
2048                  * registers with snooped memory, so relinquish any fences
2049                  * currently pointing to our region in the aperture.
2050                  */
2051                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2052                         ret = i915_gem_object_put_fence(obj);
2053                         if (ret)
2054                                 return ret;
2055                 }
2056
2057                 if (obj->has_global_gtt_mapping)
2058                         i915_gem_gtt_bind_object(obj, cache_level);
2059                 if (obj->has_aliasing_ppgtt_mapping)
2060                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2061                                                obj, cache_level);
2062         }
2063
2064         if (cache_level == I915_CACHE_NONE) {
2065                 u32 old_read_domains, old_write_domain;
2066
2067                 /* If we're coming from LLC cached, then we haven't
2068                  * actually been tracking whether the data is in the
2069                  * CPU cache or not, since we only allow one bit set
2070                  * in obj->write_domain and have been skipping the clflushes.
2071                  * Just set it to the CPU cache for now.
2072                  */
2073                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
2074                     ("obj %p in CPU write domain", obj));
2075                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
2076                     ("obj %p in CPU read domain", obj));
2077
2078                 old_read_domains = obj->base.read_domains;
2079                 old_write_domain = obj->base.write_domain;
2080
2081                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2082                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2083
2084         }
2085
2086         obj->cache_level = cache_level;
2087         return 0;
2088 }
2089
2090 /*
2091  * Prepare buffer for display plane (scanout, cursors, etc).
2092  * Can be called from an uninterruptible phase (modesetting) and allows
2093  * any flushes to be pipelined (for pageflips).
2094  */
2095 int
2096 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2097                                      u32 alignment,
2098                                      struct intel_ring_buffer *pipelined)
2099 {
2100         u32 old_read_domains, old_write_domain;
2101         int ret;
2102
2103         ret = i915_gem_object_flush_gpu_write_domain(obj);
2104         if (ret != 0)
2105                 return (ret);
2106
2107         if (pipelined != obj->ring) {
2108                 ret = i915_gem_object_wait_rendering(obj);
2109                 if (ret == -ERESTART || ret == -EINTR)
2110                         return (ret);
2111         }
2112
2113         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2114         if (ret != 0)
2115                 return (ret);
2116
2117         ret = i915_gem_object_pin(obj, alignment, true);
2118         if (ret != 0)
2119                 return (ret);
2120
2121         i915_gem_object_flush_cpu_write_domain(obj);
2122
2123         old_write_domain = obj->base.write_domain;
2124         old_read_domains = obj->base.read_domains;
2125
2126         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2127             ("obj %p in GTT write domain", obj));
2128         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2129
2130         return (0);
2131 }
2132
2133 int
2134 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2135 {
2136         int ret;
2137
2138         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2139                 return (0);
2140
2141         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2142                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2143                 if (ret != 0)
2144                         return (ret);
2145         }
2146
2147         ret = i915_gem_object_wait_rendering(obj);
2148         if (ret != 0)
2149                 return (ret);
2150
2151         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2152
2153         return (0);
2154 }
2155
2156 /**
2157  * Moves a single object to the CPU read, and possibly write domain.
2158  *
2159  * This function returns when the move is complete, including waiting on
2160  * flushes to occur.
2161  */
2162 int
2163 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2164 {
2165         uint32_t old_write_domain, old_read_domains;
2166         int ret;
2167
2168         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2169                 return 0;
2170
2171         ret = i915_gem_object_flush_gpu_write_domain(obj);
2172         if (ret != 0)
2173                 return (ret);
2174
2175         ret = i915_gem_object_wait_rendering(obj);
2176         if (ret)
2177                 return ret;
2178
2179         i915_gem_object_flush_gtt_write_domain(obj);
2180
2181         old_write_domain = obj->base.write_domain;
2182         old_read_domains = obj->base.read_domains;
2183
2184         /* Flush the CPU cache if it's still invalid. */
2185         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2186                 i915_gem_clflush_object(obj);
2187
2188                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2189         }
2190
2191         /* It should now be out of any other write domains, and we can update
2192          * the domain values for our changes.
2193          */
2194         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2195
2196         /* If we're writing through the CPU, then the GPU read domains will
2197          * need to be invalidated at next use.
2198          */
2199         if (write) {
2200                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2201                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2202         }
2203
2204         return 0;
2205 }
2206
2207 /* Throttle our rendering by waiting until the ring has completed our requests
2208  * emitted over 20 msec ago.
2209  *
2210  * Note that if we were to use the current jiffies each time around the loop,
2211  * we wouldn't escape the function with any frames outstanding if the time to
2212  * render a frame was over 20ms.
2213  *
2214  * This should get us reasonable parallelism between CPU and GPU but also
2215  * relatively low latency when blocking on a particular request to finish.
2216  */
2217 static int
2218 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2219 {
2220         struct drm_i915_private *dev_priv = dev->dev_private;
2221         struct drm_i915_file_private *file_priv = file->driver_priv;
2222         unsigned long recent_enough = ticks - (20 * hz / 1000);
2223         struct drm_i915_gem_request *request;
2224         struct intel_ring_buffer *ring = NULL;
2225         u32 seqno = 0;
2226         int ret;
2227
2228         dev_priv = dev->dev_private;
2229         if (atomic_read(&dev_priv->mm.wedged))
2230                 return -EIO;
2231
2232         recent_enough = ticks - (20 * hz / 1000);
2233         ring = NULL;
2234         seqno = 0;
2235
2236         spin_lock(&file_priv->mm.lock);
2237         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2238                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2239                         break;
2240
2241                 ring = request->ring;
2242                 seqno = request->seqno;
2243         }
2244         spin_unlock(&file_priv->mm.lock);
2245
2246         if (seqno == 0)
2247                 return 0;
2248
2249         ret = 0;
2250         lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2251         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
2252                 if (ring->irq_get(ring)) {
2253                         while (ret == 0 &&
2254                             !(i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2255                             atomic_read(&dev_priv->mm.wedged)))
2256                                 ret = -lksleep(ring, &ring->irq_lock, PCATCH,
2257                                     "915thr", 1*hz);
2258                         ring->irq_put(ring);
2259                         if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
2260                                 ret = -EIO;
2261                 } else if (_intel_wait_for(dev,
2262                     i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2263                     atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
2264                         ret = -EBUSY;
2265                 }
2266         }
2267         lockmgr(&ring->irq_lock, LK_RELEASE);
2268
2269         if (ret == 0)
2270                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2271
2272         return ret;
2273 }
2274
2275 int
2276 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
2277      bool map_and_fenceable)
2278 {
2279         struct drm_device *dev;
2280         struct drm_i915_private *dev_priv;
2281         int ret;
2282
2283         dev = obj->base.dev;
2284         dev_priv = dev->dev_private;
2285
2286         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
2287             ("Max pin count"));
2288
2289         if (obj->gtt_space != NULL) {
2290                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
2291                     (map_and_fenceable && !obj->map_and_fenceable)) {
2292                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
2293                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2294                              " obj->map_and_fenceable=%d\n",
2295                              obj->gtt_offset, alignment,
2296                              map_and_fenceable,
2297                              obj->map_and_fenceable);
2298                         ret = i915_gem_object_unbind(obj);
2299                         if (ret != 0)
2300                                 return (ret);
2301                 }
2302         }
2303
2304         if (obj->gtt_space == NULL) {
2305                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
2306                     map_and_fenceable);
2307                 if (ret)
2308                         return (ret);
2309         }
2310
2311         if (obj->pin_count++ == 0 && !obj->active)
2312                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2313         obj->pin_mappable |= map_and_fenceable;
2314
2315 #if 1
2316         KIB_NOTYET();
2317 #else
2318         WARN_ON(i915_verify_lists(dev));
2319 #endif
2320         return (0);
2321 }
2322
2323 void
2324 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
2325 {
2326         struct drm_device *dev;
2327         drm_i915_private_t *dev_priv;
2328
2329         dev = obj->base.dev;
2330         dev_priv = dev->dev_private;
2331
2332 #if 1
2333         KIB_NOTYET();
2334 #else
2335         WARN_ON(i915_verify_lists(dev));
2336 #endif
2337
2338         KASSERT(obj->pin_count != 0, ("zero pin count"));
2339         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
2340
2341         if (--obj->pin_count == 0) {
2342                 if (!obj->active)
2343                         list_move_tail(&obj->mm_list,
2344                             &dev_priv->mm.inactive_list);
2345                 obj->pin_mappable = false;
2346         }
2347 #if 1
2348         KIB_NOTYET();
2349 #else
2350         WARN_ON(i915_verify_lists(dev));
2351 #endif
2352 }
2353
2354 int
2355 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2356     struct drm_file *file)
2357 {
2358         struct drm_i915_gem_pin *args;
2359         struct drm_i915_gem_object *obj;
2360         struct drm_gem_object *gobj;
2361         int ret;
2362
2363         args = data;
2364
2365         ret = i915_mutex_lock_interruptible(dev);
2366         if (ret != 0)
2367                 return ret;
2368
2369         gobj = drm_gem_object_lookup(dev, file, args->handle);
2370         if (gobj == NULL) {
2371                 ret = -ENOENT;
2372                 goto unlock;
2373         }
2374         obj = to_intel_bo(gobj);
2375
2376         if (obj->madv != I915_MADV_WILLNEED) {
2377                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2378                 ret = -EINVAL;
2379                 goto out;
2380         }
2381
2382         if (obj->pin_filp != NULL && obj->pin_filp != file) {
2383                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2384                     args->handle);
2385                 ret = -EINVAL;
2386                 goto out;
2387         }
2388
2389         obj->user_pin_count++;
2390         obj->pin_filp = file;
2391         if (obj->user_pin_count == 1) {
2392                 ret = i915_gem_object_pin(obj, args->alignment, true);
2393                 if (ret != 0)
2394                         goto out;
2395         }
2396
2397         /* XXX - flush the CPU caches for pinned objects
2398          * as the X server doesn't manage domains yet
2399          */
2400         i915_gem_object_flush_cpu_write_domain(obj);
2401         args->offset = obj->gtt_offset;
2402 out:
2403         drm_gem_object_unreference(&obj->base);
2404 unlock:
2405         DRM_UNLOCK(dev);
2406         return (ret);
2407 }
2408
2409 int
2410 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2411     struct drm_file *file)
2412 {
2413         struct drm_i915_gem_pin *args;
2414         struct drm_i915_gem_object *obj;
2415         int ret;
2416
2417         args = data;
2418         ret = i915_mutex_lock_interruptible(dev);
2419         if (ret != 0)
2420                 return (ret);
2421
2422         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2423         if (&obj->base == NULL) {
2424                 ret = -ENOENT;
2425                 goto unlock;
2426         }
2427
2428         if (obj->pin_filp != file) {
2429                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2430                     args->handle);
2431                 ret = -EINVAL;
2432                 goto out;
2433         }
2434         obj->user_pin_count--;
2435         if (obj->user_pin_count == 0) {
2436                 obj->pin_filp = NULL;
2437                 i915_gem_object_unpin(obj);
2438         }
2439
2440 out:
2441         drm_gem_object_unreference(&obj->base);
2442 unlock:
2443         DRM_UNLOCK(dev);
2444         return (ret);
2445 }
2446
2447 int
2448 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2449                     struct drm_file *file)
2450 {
2451         struct drm_i915_gem_busy *args;
2452         struct drm_i915_gem_object *obj;
2453         struct drm_i915_gem_request *request;
2454         int ret;
2455
2456         args = data;
2457
2458         ret = i915_mutex_lock_interruptible(dev);
2459         if (ret != 0)
2460                 return ret;
2461
2462         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2463         if (&obj->base == NULL) {
2464                 ret = -ENOENT;
2465                 goto unlock;
2466         }
2467
2468         args->busy = obj->active;
2469         if (args->busy) {
2470                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2471                         ret = i915_gem_flush_ring(obj->ring,
2472                             0, obj->base.write_domain);
2473                 } else if (obj->ring->outstanding_lazy_request ==
2474                     obj->last_rendering_seqno) {
2475                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
2476                             M_WAITOK | M_ZERO);
2477                         ret = i915_add_request(obj->ring, NULL, request);
2478                         if (ret != 0)
2479                                 drm_free(request, DRM_I915_GEM);
2480                 }
2481
2482                 i915_gem_retire_requests_ring(obj->ring);
2483                 args->busy = obj->active;
2484         }
2485
2486         drm_gem_object_unreference(&obj->base);
2487 unlock:
2488         DRM_UNLOCK(dev);
2489         return (ret);
2490 }
2491
2492 int
2493 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2494     struct drm_file *file_priv)
2495 {
2496
2497         return (i915_gem_ring_throttle(dev, file_priv));
2498 }
2499
2500 int
2501 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
2502                        struct drm_file *file_priv)
2503 {
2504         struct drm_i915_gem_madvise *args = data;
2505         struct drm_i915_gem_object *obj;
2506         int ret;
2507
2508         switch (args->madv) {
2509         case I915_MADV_DONTNEED:
2510         case I915_MADV_WILLNEED:
2511             break;
2512         default:
2513             return -EINVAL;
2514         }
2515
2516         ret = i915_mutex_lock_interruptible(dev);
2517         if (ret)
2518                 return ret;
2519
2520         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
2521         if (&obj->base == NULL) {
2522                 ret = -ENOENT;
2523                 goto unlock;
2524         }
2525
2526         if (obj->pin_count) {
2527                 ret = -EINVAL;
2528                 goto out;
2529         }
2530
2531         if (obj->madv != __I915_MADV_PURGED)
2532                 obj->madv = args->madv;
2533
2534         /* if the object is no longer attached, discard its backing storage */
2535         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
2536                 i915_gem_object_truncate(obj);
2537
2538         args->retained = obj->madv != __I915_MADV_PURGED;
2539
2540 out:
2541         drm_gem_object_unreference(&obj->base);
2542 unlock:
2543         DRM_UNLOCK(dev);
2544         return ret;
2545 }
2546
2547 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
2548                                                   size_t size)
2549 {
2550         struct drm_i915_private *dev_priv;
2551         struct drm_i915_gem_object *obj;
2552
2553         dev_priv = dev->dev_private;
2554
2555         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
2556
2557         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
2558                 drm_free(obj, DRM_I915_GEM);
2559                 return (NULL);
2560         }
2561
2562         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2563         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2564
2565         if (HAS_LLC(dev))
2566                 obj->cache_level = I915_CACHE_LLC;
2567         else
2568                 obj->cache_level = I915_CACHE_NONE;
2569         obj->base.driver_private = NULL;
2570         obj->fence_reg = I915_FENCE_REG_NONE;
2571         INIT_LIST_HEAD(&obj->mm_list);
2572         INIT_LIST_HEAD(&obj->gtt_list);
2573         INIT_LIST_HEAD(&obj->ring_list);
2574         INIT_LIST_HEAD(&obj->exec_list);
2575         INIT_LIST_HEAD(&obj->gpu_write_list);
2576         obj->madv = I915_MADV_WILLNEED;
2577         /* Avoid an unnecessary call to unbind on the first bind. */
2578         obj->map_and_fenceable = true;
2579
2580         i915_gem_info_add_obj(dev_priv, size);
2581
2582         return (obj);
2583 }
2584
2585 int i915_gem_init_object(struct drm_gem_object *obj)
2586 {
2587
2588         kprintf("i915_gem_init_object called\n");
2589         return (0);
2590 }
2591
2592 static void
2593 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
2594 {
2595         struct drm_device *dev;
2596         drm_i915_private_t *dev_priv;
2597         int ret;
2598
2599         dev = obj->base.dev;
2600         dev_priv = dev->dev_private;
2601
2602         ret = i915_gem_object_unbind(obj);
2603         if (ret == -ERESTART) {
2604                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
2605                 return;
2606         }
2607
2608         drm_gem_free_mmap_offset(&obj->base);
2609         drm_gem_object_release(&obj->base);
2610         i915_gem_info_remove_obj(dev_priv, obj->base.size);
2611
2612         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
2613         drm_free(obj->bit_17, DRM_I915_GEM);
2614         drm_free(obj, DRM_I915_GEM);
2615 }
2616
2617 void
2618 i915_gem_free_object(struct drm_gem_object *gem_obj)
2619 {
2620         struct drm_i915_gem_object *obj;
2621         struct drm_device *dev;
2622
2623         obj = to_intel_bo(gem_obj);
2624         dev = obj->base.dev;
2625
2626         while (obj->pin_count > 0)
2627                 i915_gem_object_unpin(obj);
2628
2629         if (obj->phys_obj != NULL)
2630                 i915_gem_detach_phys_object(dev, obj);
2631
2632         i915_gem_free_object_tail(obj);
2633 }
2634
2635 int
2636 i915_gem_do_init(struct drm_device *dev, unsigned long start,
2637     unsigned long mappable_end, unsigned long end)
2638 {
2639         drm_i915_private_t *dev_priv;
2640         unsigned long mappable;
2641         int error;
2642
2643         dev_priv = dev->dev_private;
2644         mappable = min(end, mappable_end) - start;
2645
2646         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
2647
2648         dev_priv->mm.gtt_start = start;
2649         dev_priv->mm.gtt_mappable_end = mappable_end;
2650         dev_priv->mm.gtt_end = end;
2651         dev_priv->mm.gtt_total = end - start;
2652         dev_priv->mm.mappable_gtt_total = mappable;
2653
2654         /* Take over this portion of the GTT */
2655         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
2656         device_printf(dev->dev,
2657             "taking over the fictitious range 0x%lx-0x%lx\n",
2658             dev->agp->base + start, dev->agp->base + start + mappable);
2659         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
2660             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2661         return (error);
2662 }
2663
2664 int
2665 i915_gem_idle(struct drm_device *dev)
2666 {
2667         drm_i915_private_t *dev_priv;
2668         int ret;
2669
2670         dev_priv = dev->dev_private;
2671         if (dev_priv->mm.suspended)
2672                 return (0);
2673
2674         ret = i915_gpu_idle(dev);
2675         if (ret != 0)
2676                 return (ret);
2677
2678         /* Under UMS, be paranoid and evict. */
2679         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
2680                 ret = i915_gem_evict_inactive(dev, false);
2681                 if (ret != 0)
2682                         return ret;
2683         }
2684
2685         i915_gem_reset_fences(dev);
2686
2687         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2688          * We need to replace this with a semaphore, or something.
2689          * And not confound mm.suspended!
2690          */
2691         dev_priv->mm.suspended = 1;
2692         del_timer_sync(&dev_priv->hangcheck_timer);
2693
2694         i915_kernel_lost_context(dev);
2695         i915_gem_cleanup_ringbuffer(dev);
2696
2697         /* Cancel the retire work handler, which should be idle now. */
2698         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2699
2700         return (ret);
2701 }
2702
2703 void
2704 i915_gem_init_swizzling(struct drm_device *dev)
2705 {
2706         drm_i915_private_t *dev_priv;
2707
2708         dev_priv = dev->dev_private;
2709
2710         if (INTEL_INFO(dev)->gen < 5 ||
2711             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
2712                 return;
2713
2714         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
2715                                  DISP_TILE_SURFACE_SWIZZLING);
2716
2717         if (IS_GEN5(dev))
2718                 return;
2719
2720         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
2721         if (IS_GEN6(dev))
2722                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
2723         else
2724                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
2725 }
2726
2727 int
2728 i915_gem_init_hw(struct drm_device *dev)
2729 {
2730         drm_i915_private_t *dev_priv;
2731         int ret;
2732
2733         dev_priv = dev->dev_private;
2734
2735         i915_gem_init_swizzling(dev);
2736
2737         ret = intel_init_render_ring_buffer(dev);
2738         if (ret != 0)
2739                 return (ret);
2740
2741         if (HAS_BSD(dev)) {
2742                 ret = intel_init_bsd_ring_buffer(dev);
2743                 if (ret != 0)
2744                         goto cleanup_render_ring;
2745         }
2746
2747         if (HAS_BLT(dev)) {
2748                 ret = intel_init_blt_ring_buffer(dev);
2749                 if (ret != 0)
2750                         goto cleanup_bsd_ring;
2751         }
2752
2753         dev_priv->next_seqno = 1;
2754         i915_gem_init_ppgtt(dev);
2755         return (0);
2756
2757 cleanup_bsd_ring:
2758         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
2759 cleanup_render_ring:
2760         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
2761         return (ret);
2762 }
2763
2764 void
2765 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2766 {
2767         drm_i915_private_t *dev_priv;
2768         int i;
2769
2770         dev_priv = dev->dev_private;
2771         for (i = 0; i < I915_NUM_RINGS; i++)
2772                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
2773 }
2774
2775 int
2776 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2777                        struct drm_file *file_priv)
2778 {
2779         drm_i915_private_t *dev_priv = dev->dev_private;
2780         int ret;
2781
2782         if (drm_core_check_feature(dev, DRIVER_MODESET))
2783                 return 0;
2784
2785         if (atomic_read(&dev_priv->mm.wedged)) {
2786                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2787                 atomic_set(&dev_priv->mm.wedged, 0);
2788         }
2789
2790         DRM_LOCK(dev);
2791         dev_priv->mm.suspended = 0;
2792
2793         ret = i915_gem_init_hw(dev);
2794         if (ret != 0) {
2795                 DRM_UNLOCK(dev);
2796                 return ret;
2797         }
2798
2799         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
2800         DRM_UNLOCK(dev);
2801
2802         ret = drm_irq_install(dev);
2803         if (ret)
2804                 goto cleanup_ringbuffer;
2805
2806         return 0;
2807
2808 cleanup_ringbuffer:
2809         DRM_LOCK(dev);
2810         i915_gem_cleanup_ringbuffer(dev);
2811         dev_priv->mm.suspended = 1;
2812         DRM_UNLOCK(dev);
2813
2814         return ret;
2815 }
2816
2817 int
2818 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2819     struct drm_file *file_priv)
2820 {
2821
2822         if (drm_core_check_feature(dev, DRIVER_MODESET))
2823                 return 0;
2824
2825         drm_irq_uninstall(dev);
2826         return (i915_gem_idle(dev));
2827 }
2828
2829 void
2830 i915_gem_lastclose(struct drm_device *dev)
2831 {
2832         int ret;
2833
2834         if (drm_core_check_feature(dev, DRIVER_MODESET))
2835                 return;
2836
2837         ret = i915_gem_idle(dev);
2838         if (ret != 0)
2839                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2840 }
2841
2842 static void
2843 init_ring_lists(struct intel_ring_buffer *ring)
2844 {
2845
2846         INIT_LIST_HEAD(&ring->active_list);
2847         INIT_LIST_HEAD(&ring->request_list);
2848         INIT_LIST_HEAD(&ring->gpu_write_list);
2849 }
2850
2851 void
2852 i915_gem_load(struct drm_device *dev)
2853 {
2854         int i;
2855         drm_i915_private_t *dev_priv = dev->dev_private;
2856
2857         INIT_LIST_HEAD(&dev_priv->mm.active_list);
2858         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2859         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2860         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
2861         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2862         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
2863         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
2864         for (i = 0; i < I915_NUM_RINGS; i++)
2865                 init_ring_lists(&dev_priv->ring[i]);
2866         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2867                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2868         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2869                           i915_gem_retire_work_handler);
2870         dev_priv->error_completion = 0;
2871
2872         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
2873         if (IS_GEN3(dev)) {
2874                 I915_WRITE(MI_ARB_STATE,
2875                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2876         }
2877
2878         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
2879
2880         /* Old X drivers will take 0-2 for front, back, depth buffers */
2881         if (!drm_core_check_feature(dev, DRIVER_MODESET))
2882                 dev_priv->fence_reg_start = 3;
2883
2884         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
2885                 dev_priv->num_fence_regs = 16;
2886         else
2887                 dev_priv->num_fence_regs = 8;
2888
2889         /* Initialize fence registers to zero */
2890         i915_gem_reset_fences(dev);
2891
2892         i915_gem_detect_bit_6_swizzle(dev);
2893
2894         dev_priv->mm.interruptible = true;
2895
2896         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
2897             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
2898 }
2899
2900 static int
2901 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
2902 {
2903         drm_i915_private_t *dev_priv;
2904         struct drm_i915_gem_phys_object *phys_obj;
2905         int ret;
2906
2907         dev_priv = dev->dev_private;
2908         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
2909                 return (0);
2910
2911         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
2912             M_WAITOK | M_ZERO);
2913
2914         phys_obj->id = id;
2915
2916         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
2917         if (phys_obj->handle == NULL) {
2918                 ret = -ENOMEM;
2919                 goto free_obj;
2920         }
2921         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
2922             size / PAGE_SIZE, PAT_WRITE_COMBINING);
2923
2924         dev_priv->mm.phys_objs[id - 1] = phys_obj;
2925
2926         return (0);
2927
2928 free_obj:
2929         drm_free(phys_obj, DRM_I915_GEM);
2930         return (ret);
2931 }
2932
2933 static void
2934 i915_gem_free_phys_object(struct drm_device *dev, int id)
2935 {
2936         drm_i915_private_t *dev_priv;
2937         struct drm_i915_gem_phys_object *phys_obj;
2938
2939         dev_priv = dev->dev_private;
2940         if (dev_priv->mm.phys_objs[id - 1] == NULL)
2941                 return;
2942
2943         phys_obj = dev_priv->mm.phys_objs[id - 1];
2944         if (phys_obj->cur_obj != NULL)
2945                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
2946
2947         drm_pci_free(dev, phys_obj->handle);
2948         drm_free(phys_obj, DRM_I915_GEM);
2949         dev_priv->mm.phys_objs[id - 1] = NULL;
2950 }
2951
2952 void
2953 i915_gem_free_all_phys_object(struct drm_device *dev)
2954 {
2955         int i;
2956
2957         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
2958                 i915_gem_free_phys_object(dev, i);
2959 }
2960
2961 void
2962 i915_gem_detach_phys_object(struct drm_device *dev,
2963     struct drm_i915_gem_object *obj)
2964 {
2965         vm_page_t m;
2966         struct sf_buf *sf;
2967         char *vaddr, *dst;
2968         int i, page_count;
2969
2970         if (obj->phys_obj == NULL)
2971                 return;
2972         vaddr = obj->phys_obj->handle->vaddr;
2973
2974         page_count = obj->base.size / PAGE_SIZE;
2975         VM_OBJECT_LOCK(obj->base.vm_obj);
2976         for (i = 0; i < page_count; i++) {
2977                 m = i915_gem_wire_page(obj->base.vm_obj, i);
2978                 if (m == NULL)
2979                         continue; /* XXX */
2980
2981                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
2982                 sf = sf_buf_alloc(m);
2983                 if (sf != NULL) {
2984                         dst = (char *)sf_buf_kva(sf);
2985                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
2986                         sf_buf_free(sf);
2987                 }
2988                 drm_clflush_pages(&m, 1);
2989
2990                 VM_OBJECT_LOCK(obj->base.vm_obj);
2991                 vm_page_reference(m);
2992                 vm_page_dirty(m);
2993                 vm_page_busy_wait(m, FALSE, "i915gem");
2994                 vm_page_unwire(m, 0);
2995                 vm_page_wakeup(m);
2996                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2997         }
2998         VM_OBJECT_UNLOCK(obj->base.vm_obj);
2999         intel_gtt_chipset_flush();
3000
3001         obj->phys_obj->cur_obj = NULL;
3002         obj->phys_obj = NULL;
3003 }
3004
3005 int
3006 i915_gem_attach_phys_object(struct drm_device *dev,
3007                             struct drm_i915_gem_object *obj,
3008                             int id,
3009                             int align)
3010 {
3011         drm_i915_private_t *dev_priv;
3012         vm_page_t m;
3013         struct sf_buf *sf;
3014         char *dst, *src;
3015         int i, page_count, ret;
3016
3017         if (id > I915_MAX_PHYS_OBJECT)
3018                 return (-EINVAL);
3019
3020         if (obj->phys_obj != NULL) {
3021                 if (obj->phys_obj->id == id)
3022                         return (0);
3023                 i915_gem_detach_phys_object(dev, obj);
3024         }
3025
3026         dev_priv = dev->dev_private;
3027         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3028                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3029                 if (ret != 0) {
3030                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3031                                   id, obj->base.size);
3032                         return (ret);
3033                 }
3034         }
3035
3036         /* bind to the object */
3037         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3038         obj->phys_obj->cur_obj = obj;
3039
3040         page_count = obj->base.size / PAGE_SIZE;
3041
3042         VM_OBJECT_LOCK(obj->base.vm_obj);
3043         ret = 0;
3044         for (i = 0; i < page_count; i++) {
3045                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3046                 if (m == NULL) {
3047                         ret = -EIO;
3048                         break;
3049                 }
3050                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3051                 sf = sf_buf_alloc(m);
3052                 src = (char *)sf_buf_kva(sf);
3053                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3054                 memcpy(dst, src, PAGE_SIZE);
3055                 sf_buf_free(sf);
3056
3057                 VM_OBJECT_LOCK(obj->base.vm_obj);
3058
3059                 vm_page_reference(m);
3060                 vm_page_busy_wait(m, FALSE, "i915gem");
3061                 vm_page_unwire(m, 0);
3062                 vm_page_wakeup(m);
3063                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3064         }
3065         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3066
3067         return (0);
3068 }
3069
3070 static int
3071 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3072     uint64_t data_ptr, uint64_t offset, uint64_t size,
3073     struct drm_file *file_priv)
3074 {
3075         char *user_data, *vaddr;
3076         int ret;
3077
3078         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3079         user_data = (char *)(uintptr_t)data_ptr;
3080
3081         if (copyin_nofault(user_data, vaddr, size) != 0) {
3082                 /* The physical object once assigned is fixed for the lifetime
3083                  * of the obj, so we can safely drop the lock and continue
3084                  * to access vaddr.
3085                  */
3086                 DRM_UNLOCK(dev);
3087                 ret = -copyin(user_data, vaddr, size);
3088                 DRM_LOCK(dev);
3089                 if (ret != 0)
3090                         return (ret);
3091         }
3092
3093         intel_gtt_chipset_flush();
3094         return (0);
3095 }
3096
3097 void
3098 i915_gem_release(struct drm_device *dev, struct drm_file *file)
3099 {
3100         struct drm_i915_file_private *file_priv;
3101         struct drm_i915_gem_request *request;
3102
3103         file_priv = file->driver_priv;
3104
3105         /* Clean up our request list when the client is going away, so that
3106          * later retire_requests won't dereference our soon-to-be-gone
3107          * file_priv.
3108          */
3109         spin_lock(&file_priv->mm.lock);
3110         while (!list_empty(&file_priv->mm.request_list)) {
3111                 request = list_first_entry(&file_priv->mm.request_list,
3112                                            struct drm_i915_gem_request,
3113                                            client_list);
3114                 list_del(&request->client_list);
3115                 request->file_priv = NULL;
3116         }
3117         spin_unlock(&file_priv->mm.lock);
3118 }
3119
3120 static int
3121 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
3122     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
3123     struct drm_file *file)
3124 {
3125         vm_object_t vm_obj;
3126         vm_page_t m;
3127         struct sf_buf *sf;
3128         vm_offset_t mkva;
3129         vm_pindex_t obj_pi;
3130         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
3131
3132         if (obj->gtt_offset != 0 && rw == UIO_READ)
3133                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
3134         else
3135                 do_bit17_swizzling = 0;
3136
3137         obj->dirty = 1;
3138         vm_obj = obj->base.vm_obj;
3139         ret = 0;
3140
3141         VM_OBJECT_LOCK(vm_obj);
3142         vm_object_pip_add(vm_obj, 1);
3143         while (size > 0) {
3144                 obj_pi = OFF_TO_IDX(offset);
3145                 obj_po = offset & PAGE_MASK;
3146
3147                 m = i915_gem_wire_page(vm_obj, obj_pi);
3148                 VM_OBJECT_UNLOCK(vm_obj);
3149
3150                 sf = sf_buf_alloc(m);
3151                 mkva = sf_buf_kva(sf);
3152                 length = min(size, PAGE_SIZE - obj_po);
3153                 while (length > 0) {
3154                         if (do_bit17_swizzling &&
3155                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
3156                                 cnt = roundup2(obj_po + 1, 64);
3157                                 cnt = min(cnt - obj_po, length);
3158                                 swizzled_po = obj_po ^ 64;
3159                         } else {
3160                                 cnt = length;
3161                                 swizzled_po = obj_po;
3162                         }
3163                         if (rw == UIO_READ)
3164                                 ret = -copyout_nofault(
3165                                     (char *)mkva + swizzled_po,
3166                                     (void *)(uintptr_t)data_ptr, cnt);
3167                         else
3168                                 ret = -copyin_nofault(
3169                                     (void *)(uintptr_t)data_ptr,
3170                                     (char *)mkva + swizzled_po, cnt);
3171                         if (ret != 0)
3172                                 break;
3173                         data_ptr += cnt;
3174                         size -= cnt;
3175                         length -= cnt;
3176                         offset += cnt;
3177                         obj_po += cnt;
3178                 }
3179                 sf_buf_free(sf);
3180                 VM_OBJECT_LOCK(vm_obj);
3181                 if (rw == UIO_WRITE)
3182                         vm_page_dirty(m);
3183                 vm_page_reference(m);
3184                 vm_page_busy_wait(m, FALSE, "i915gem");
3185                 vm_page_unwire(m, 1);
3186                 vm_page_wakeup(m);
3187                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3188
3189                 if (ret != 0)
3190                         break;
3191         }
3192         vm_object_pip_wakeup(vm_obj);
3193         VM_OBJECT_UNLOCK(vm_obj);
3194
3195         return (ret);
3196 }
3197
3198 static int
3199 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
3200     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
3201 {
3202         vm_offset_t mkva;
3203         int ret;
3204
3205         /*
3206          * Pass the unaligned physical address and size to pmap_mapdev_attr()
3207          * so it can properly calculate whether an extra page needs to be
3208          * mapped or not to cover the requested range.  The function will
3209          * add the page offset into the returned mkva for us.
3210          */
3211         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
3212             offset, size, PAT_WRITE_COMBINING);
3213         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
3214         pmap_unmapdev(mkva, size);
3215         return (ret);
3216 }
3217
3218 static int
3219 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
3220     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
3221 {
3222         struct drm_i915_gem_object *obj;
3223         vm_page_t *ma;
3224         vm_offset_t start, end;
3225         int npages, ret;
3226
3227         if (size == 0)
3228                 return (0);
3229         start = trunc_page(data_ptr);
3230         end = round_page(data_ptr + size);
3231         npages = howmany(end - start, PAGE_SIZE);
3232         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
3233             M_ZERO);
3234         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
3235             (vm_offset_t)data_ptr, size,
3236             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
3237         if (npages == -1) {
3238                 ret = -EFAULT;
3239                 goto free_ma;
3240         }
3241
3242         ret = i915_mutex_lock_interruptible(dev);
3243         if (ret != 0)
3244                 goto unlocked;
3245
3246         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
3247         if (&obj->base == NULL) {
3248                 ret = -ENOENT;
3249                 goto unlock;
3250         }
3251         if (offset > obj->base.size || size > obj->base.size - offset) {
3252                 ret = -EINVAL;
3253                 goto out;
3254         }
3255
3256         if (rw == UIO_READ) {
3257                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
3258                     offset, size);
3259                 if (ret != 0)
3260                         goto out;
3261                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3262                     UIO_READ, file);
3263         } else {
3264                 if (obj->phys_obj) {
3265                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
3266                             size, file);
3267                 } else if (obj->gtt_space &&
3268                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3269                         ret = i915_gem_object_pin(obj, 0, true);
3270                         if (ret != 0)
3271                                 goto out;
3272                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
3273                         if (ret != 0)
3274                                 goto out_unpin;
3275                         ret = i915_gem_object_put_fence(obj);
3276                         if (ret != 0)
3277                                 goto out_unpin;
3278                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
3279                             offset, file);
3280 out_unpin:
3281                         i915_gem_object_unpin(obj);
3282                 } else {
3283                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
3284                         if (ret != 0)
3285                                 goto out;
3286                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3287                             UIO_WRITE, file);
3288                 }
3289         }
3290 out:
3291         drm_gem_object_unreference(&obj->base);
3292 unlock:
3293         DRM_UNLOCK(dev);
3294 unlocked:
3295         vm_page_unhold_pages(ma, npages);
3296 free_ma:
3297         drm_free(ma, DRM_I915_GEM);
3298         return (ret);
3299 }
3300
3301 static int
3302 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
3303     vm_ooffset_t foff, struct ucred *cred, u_short *color)
3304 {
3305
3306         *color = 0; /* XXXKIB */
3307         return (0);
3308 }
3309
3310 int i915_intr_pf;
3311
3312 static int
3313 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
3314     vm_page_t *mres)
3315 {
3316         struct drm_gem_object *gem_obj;
3317         struct drm_i915_gem_object *obj;
3318         struct drm_device *dev;
3319         drm_i915_private_t *dev_priv;
3320         vm_page_t m, oldm;
3321         int cause, ret;
3322         bool write;
3323
3324         gem_obj = vm_obj->handle;
3325         obj = to_intel_bo(gem_obj);
3326         dev = obj->base.dev;
3327         dev_priv = dev->dev_private;
3328 #if 0
3329         write = (prot & VM_PROT_WRITE) != 0;
3330 #else
3331         write = true;
3332 #endif
3333         vm_object_pip_add(vm_obj, 1);
3334
3335         /*
3336          * Remove the placeholder page inserted by vm_fault() from the
3337          * object before dropping the object lock. If
3338          * i915_gem_release_mmap() is active in parallel on this gem
3339          * object, then it owns the drm device sx and might find the
3340          * placeholder already. Then, since the page is busy,
3341          * i915_gem_release_mmap() sleeps waiting for the busy state
3342          * of the page cleared. We will be not able to acquire drm
3343          * device lock until i915_gem_release_mmap() is able to make a
3344          * progress.
3345          */
3346         if (*mres != NULL) {
3347                 oldm = *mres;
3348                 vm_page_remove(oldm);
3349                 *mres = NULL;
3350         } else
3351                 oldm = NULL;
3352 retry:
3353         VM_OBJECT_UNLOCK(vm_obj);
3354 unlocked_vmobj:
3355         cause = ret = 0;
3356         m = NULL;
3357
3358         if (i915_intr_pf) {
3359                 ret = i915_mutex_lock_interruptible(dev);
3360                 if (ret != 0) {
3361                         cause = 10;
3362                         goto out;
3363                 }
3364         } else
3365                 DRM_LOCK(dev);
3366
3367         /*
3368          * Since the object lock was dropped, other thread might have
3369          * faulted on the same GTT address and instantiated the
3370          * mapping for the page.  Recheck.
3371          */
3372         VM_OBJECT_LOCK(vm_obj);
3373         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
3374         if (m != NULL) {
3375                 if ((m->flags & PG_BUSY) != 0) {
3376                         DRM_UNLOCK(dev);
3377 #if 0 /* XXX */
3378                         vm_page_sleep(m, "915pee");
3379 #endif
3380                         goto retry;
3381                 }
3382                 goto have_page;
3383         } else
3384                 VM_OBJECT_UNLOCK(vm_obj);
3385
3386         /* Now bind it into the GTT if needed */
3387         if (!obj->map_and_fenceable) {
3388                 ret = i915_gem_object_unbind(obj);
3389                 if (ret != 0) {
3390                         cause = 20;
3391                         goto unlock;
3392                 }
3393         }
3394         if (!obj->gtt_space) {
3395                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
3396                 if (ret != 0) {
3397                         cause = 30;
3398                         goto unlock;
3399                 }
3400
3401                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
3402                 if (ret != 0) {
3403                         cause = 40;
3404                         goto unlock;
3405                 }
3406         }
3407
3408         if (obj->tiling_mode == I915_TILING_NONE)
3409                 ret = i915_gem_object_put_fence(obj);
3410         else
3411                 ret = i915_gem_object_get_fence(obj, NULL);
3412         if (ret != 0) {
3413                 cause = 50;
3414                 goto unlock;
3415         }
3416
3417         if (i915_gem_object_is_inactive(obj))
3418                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3419
3420         obj->fault_mappable = true;
3421         VM_OBJECT_LOCK(vm_obj);
3422         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
3423             offset);
3424         if (m == NULL) {
3425                 cause = 60;
3426                 ret = -EFAULT;
3427                 goto unlock;
3428         }
3429         KASSERT((m->flags & PG_FICTITIOUS) != 0,
3430             ("not fictitious %p", m));
3431         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
3432
3433         if ((m->flags & PG_BUSY) != 0) {
3434                 DRM_UNLOCK(dev);
3435 #if 0 /* XXX */
3436                 vm_page_sleep(m, "915pbs");
3437 #endif
3438                 goto retry;
3439         }
3440         m->valid = VM_PAGE_BITS_ALL;
3441         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
3442 have_page:
3443         *mres = m;
3444         vm_page_busy_try(m, false);
3445
3446         DRM_UNLOCK(dev);
3447         if (oldm != NULL) {
3448                 vm_page_free(oldm);
3449         }
3450         vm_object_pip_wakeup(vm_obj);
3451         return (VM_PAGER_OK);
3452
3453 unlock:
3454         DRM_UNLOCK(dev);
3455 out:
3456         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
3457         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
3458                 goto unlocked_vmobj;
3459         }
3460         VM_OBJECT_LOCK(vm_obj);
3461         vm_object_pip_wakeup(vm_obj);
3462         return (VM_PAGER_ERROR);
3463 }
3464
3465 static void
3466 i915_gem_pager_dtor(void *handle)
3467 {
3468         struct drm_gem_object *obj;
3469         struct drm_device *dev;
3470
3471         obj = handle;
3472         dev = obj->dev;
3473
3474         DRM_LOCK(dev);
3475         drm_gem_free_mmap_offset(obj);
3476         i915_gem_release_mmap(to_intel_bo(obj));
3477         drm_gem_object_unreference(obj);
3478         DRM_UNLOCK(dev);
3479 }
3480
3481 struct cdev_pager_ops i915_gem_pager_ops = {
3482         .cdev_pg_fault  = i915_gem_pager_fault,
3483         .cdev_pg_ctor   = i915_gem_pager_ctor,
3484         .cdev_pg_dtor   = i915_gem_pager_dtor
3485 };
3486
3487 static int
3488 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3489     uint64_t offset, uint64_t size)
3490 {
3491         uint32_t old_read_domains;
3492         int i, ret;
3493
3494         if (offset == 0 && size == obj->base.size)
3495                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
3496
3497         ret = i915_gem_object_flush_gpu_write_domain(obj);
3498         if (ret != 0)
3499                 return (ret);
3500         ret = i915_gem_object_wait_rendering(obj);
3501         if (ret != 0)
3502                 return (ret);
3503
3504         i915_gem_object_flush_gtt_write_domain(obj);
3505
3506         if (obj->page_cpu_valid == NULL &&
3507             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3508                 return (0);
3509
3510         if (obj->page_cpu_valid == NULL) {
3511                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
3512                     DRM_I915_GEM, M_WAITOK | M_ZERO);
3513         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3514                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3515
3516         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3517              i++) {
3518                 if (obj->page_cpu_valid[i])
3519                         continue;
3520                 drm_clflush_pages(obj->pages + i, 1);
3521                 obj->page_cpu_valid[i] = 1;
3522         }
3523
3524         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
3525             ("In gpu write domain"));
3526
3527         old_read_domains = obj->base.read_domains;
3528         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3529
3530         return (0);
3531 }
3532
3533 #define GEM_PARANOID_CHECK_GTT 0
3534 #if GEM_PARANOID_CHECK_GTT
3535 static void
3536 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
3537     int page_count)
3538 {
3539         struct drm_i915_private *dev_priv;
3540         vm_paddr_t pa;
3541         unsigned long start, end;
3542         u_int i;
3543         int j;
3544
3545         dev_priv = dev->dev_private;
3546         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
3547         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
3548         for (i = start; i < end; i++) {
3549                 pa = intel_gtt_read_pte_paddr(i);
3550                 for (j = 0; j < page_count; j++) {
3551                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
3552                                 panic("Page %p in GTT pte index %d pte %x",
3553                                     ma[i], i, intel_gtt_read_pte(i));
3554                         }
3555                 }
3556         }
3557 }
3558 #endif
3559
3560 static void
3561 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
3562     uint32_t flush_domains)
3563 {
3564         struct drm_i915_gem_object *obj, *next;
3565         uint32_t old_write_domain;
3566
3567         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
3568             gpu_write_list) {
3569                 if (obj->base.write_domain & flush_domains) {
3570                         old_write_domain = obj->base.write_domain;
3571                         obj->base.write_domain = 0;
3572                         list_del_init(&obj->gpu_write_list);
3573                         i915_gem_object_move_to_active(obj, ring,
3574                             i915_gem_next_request_seqno(ring));
3575                 }
3576         }
3577 }
3578
3579 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3580
3581 static vm_page_t
3582 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3583 {
3584         vm_page_t m;
3585         int rv;
3586
3587         VM_OBJECT_LOCK_ASSERT_OWNED(object);
3588         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
3589         if (m->valid != VM_PAGE_BITS_ALL) {
3590                 if (vm_pager_has_page(object, pindex)) {
3591                         rv = vm_pager_get_page(object, &m, 1);
3592                         m = vm_page_lookup(object, pindex);
3593                         if (m == NULL)
3594                                 return (NULL);
3595                         if (rv != VM_PAGER_OK) {
3596                                 vm_page_free(m);
3597                                 return (NULL);
3598                         }
3599                 } else {
3600                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
3601                         m->valid = VM_PAGE_BITS_ALL;
3602                         m->dirty = 0;
3603                 }
3604         }
3605         vm_page_wire(m);
3606         vm_page_wakeup(m);
3607         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3608         return (m);
3609 }
3610
3611 int
3612 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
3613     uint32_t flush_domains)
3614 {
3615         int ret;
3616
3617         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
3618                 return 0;
3619
3620         ret = ring->flush(ring, invalidate_domains, flush_domains);
3621         if (ret)
3622                 return ret;
3623
3624         if (flush_domains & I915_GEM_GPU_DOMAINS)
3625                 i915_gem_process_flushing_list(ring, flush_domains);
3626         return 0;
3627 }
3628
3629 u32
3630 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
3631 {
3632         if (ring->outstanding_lazy_request == 0)
3633                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
3634
3635         return ring->outstanding_lazy_request;
3636 }
3637
3638 static void
3639 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3640 {
3641         drm_i915_private_t *dev_priv = dev->dev_private;
3642         uint32_t fence_reg = reg - dev_priv->fence_regs;
3643
3644         switch (INTEL_INFO(dev)->gen) {
3645         case 7:
3646         case 6:
3647                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3648                 break;
3649         case 5:
3650         case 4:
3651                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3652                 break;
3653         case 3:
3654                 if (fence_reg >= 8)
3655                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3656                 else
3657         case 2:
3658                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3659
3660                 I915_WRITE(fence_reg, 0);
3661                 break;
3662         }
3663
3664         list_del_init(&reg->lru_list);
3665         reg->obj = NULL;
3666         reg->setup_seqno = 0;
3667         reg->pin_count = 0;
3668 }
3669
3670 static int
3671 i915_gpu_is_active(struct drm_device *dev)
3672 {
3673         drm_i915_private_t *dev_priv;
3674
3675         dev_priv = dev->dev_private;
3676         return (!list_empty(&dev_priv->mm.flushing_list) ||
3677             !list_empty(&dev_priv->mm.active_list));
3678 }
3679
3680 static void
3681 i915_gem_lowmem(void *arg)
3682 {
3683         struct drm_device *dev;
3684         struct drm_i915_private *dev_priv;
3685         struct drm_i915_gem_object *obj, *next;
3686         int cnt, cnt_fail, cnt_total;
3687
3688         dev = arg;
3689         dev_priv = dev->dev_private;
3690
3691         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3692                 return;
3693
3694 rescan:
3695         /* first scan for clean buffers */
3696         i915_gem_retire_requests(dev);
3697
3698         cnt_total = cnt_fail = cnt = 0;
3699
3700         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3701             mm_list) {
3702                 if (i915_gem_object_is_purgeable(obj)) {
3703                         if (i915_gem_object_unbind(obj) != 0)
3704                                 cnt_total++;
3705                 } else
3706                         cnt_total++;
3707         }
3708
3709         /* second pass, evict/count anything still on the inactive list */
3710         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3711             mm_list) {
3712                 if (i915_gem_object_unbind(obj) == 0)
3713                         cnt++;
3714                 else
3715                         cnt_fail++;
3716         }
3717
3718         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3719                 /*
3720                  * We are desperate for pages, so as a last resort, wait
3721                  * for the GPU to finish and discard whatever we can.
3722                  * This has a dramatic impact to reduce the number of
3723                  * OOM-killer events whilst running the GPU aggressively.
3724                  */
3725                 if (i915_gpu_idle(dev) == 0)
3726                         goto rescan;
3727         }
3728         DRM_UNLOCK(dev);
3729 }
3730
3731 void
3732 i915_gem_unload(struct drm_device *dev)
3733 {
3734         struct drm_i915_private *dev_priv;
3735
3736         dev_priv = dev->dev_private;
3737         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3738 }