sys/dev/drm/i915/i915_gem.c

   1 /*-
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  * Copyright (c) 2011 The FreeBSD Foundation
  27  * All rights reserved.
  28  *
  29  * This software was developed by Konstantin Belousov under sponsorship from
  30  * the FreeBSD Foundation.
  31  *
  32  * Redistribution and use in source and binary forms, with or without
  33  * modification, are permitted provided that the following conditions
  34  * are met:
  35  * 1. Redistributions of source code must retain the above copyright
  36  *    notice, this list of conditions and the following disclaimer.
  37  * 2. Redistributions in binary form must reproduce the above copyright
  38  *    notice, this list of conditions and the following disclaimer in the
  39  *    documentation and/or other materials provided with the distribution.
  40  *
  41  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  44  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  51  * SUCH DAMAGE.
  52  *
  53  * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $
  54  */
  55
  56 #include <sys/resourcevar.h>
  57 #include <sys/sfbuf.h>
  58
  59 #include <drm/drmP.h>
  60 #include <drm/i915_drm.h>
  61 #include "i915_drv.h"
  62 #include "intel_drv.h"
  63 #include "intel_ringbuffer.h"
  64 #include <linux/completion.h>
  65
  66 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  67 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  68 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  69     unsigned alignment, bool map_and_fenceable);
  70
  71 static int i915_gem_phys_pwrite(struct drm_device *dev,
  72     struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
  73     uint64_t size, struct drm_file *file_priv);
  74
  75 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
  76     int tiling_mode);
  77 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
  78     uint32_t size, int tiling_mode);
  79 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
  80     int flags);
  81 static int i915_gem_object_set_cpu_read_domain_range(
  82     struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
  83 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
  84 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  85 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
  86 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
  87 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
  88 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
  89 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
  90     uint32_t flush_domains);
  91 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  92     struct drm_i915_fence_reg *reg);
  93 static void i915_gem_reset_fences(struct drm_device *dev);
  94 static void i915_gem_lowmem(void *arg);
  95
  96 static int i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
  97     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file);
  98
  99 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
 100 long i915_gem_wired_pages_cnt;
 101
 102 /* some bookkeeping */
 103 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 104                                   size_t size)
 105 {
 106
 107         dev_priv->mm.object_count++;
 108         dev_priv->mm.object_memory += size;
 109 }
 110
 111 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 112                                      size_t size)
 113 {
 114
 115         dev_priv->mm.object_count--;
 116         dev_priv->mm.object_memory -= size;
 117 }
 118
 119 static int
 120 i915_gem_wait_for_error(struct drm_device *dev)
 121 {
 122         struct drm_i915_private *dev_priv = dev->dev_private;
 123         struct completion *x = &dev_priv->error_completion;
 124         int ret;
 125
 126         if (!atomic_read(&dev_priv->mm.wedged))
 127                 return 0;
 128
 129         /*
 130          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 131          * userspace. If it takes that long something really bad is going on and
 132          * we should simply try to bail out and fail as gracefully as possible.
 133          */
 134         ret = wait_for_completion_interruptible_timeout(x, 10*hz);
 135         if (ret == 0) {
 136                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 137                 return -EIO;
 138         } else if (ret < 0) {
 139                 return ret;
 140         }
 141
 142         if (atomic_read(&dev_priv->mm.wedged)) {
 143                 /* GPU is hung, bump the completion count to account for
 144                  * the token we just consumed so that we never hit zero and
 145                  * end up waiting upon a subsequent completion event that
 146                  * will never happen.
 147                  */
 148                 spin_lock(&x->wait.lock);
 149                 x->done++;
 150                 spin_unlock(&x->wait.lock);
 151         }
 152         return 0;
 153 }
 154
 155 int i915_mutex_lock_interruptible(struct drm_device *dev)
 156 {
 157         int ret;
 158
 159         ret = i915_gem_wait_for_error(dev);
 160         if (ret != 0)
 161                 return (ret);
 162
 163         ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL);
 164         if (ret)
 165                 return -EINTR;
 166
 167 #if 0
 168         WARN_ON(i915_verify_lists(dev));
 169 #endif
 170         return 0;
 171 }
 172
 173 static inline bool
 174 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 175 {
 176         return (obj->gtt_space && !obj->active && obj->pin_count == 0);
 177 }
 178
 179 int
 180 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 181     struct drm_file *file)
 182 {
 183         struct drm_i915_gem_init *args;
 184         drm_i915_private_t *dev_priv;
 185
 186         dev_priv = dev->dev_private;
 187         args = data;
 188
 189         if (args->gtt_start >= args->gtt_end ||
 190             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 191                 return (-EINVAL);
 192
 193         /*
 194          * XXXKIB. The second-time initialization should be guarded
 195          * against.
 196          */
 197         lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE);
 198         i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 199         lockmgr(&dev->dev_lock, LK_RELEASE);
 200
 201         return 0;
 202 }
 203
 204 int
 205 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 206                             struct drm_file *file)
 207 {
 208         struct drm_i915_private *dev_priv;
 209         struct drm_i915_gem_get_aperture *args;
 210         struct drm_i915_gem_object *obj;
 211         size_t pinned;
 212
 213         dev_priv = dev->dev_private;
 214         args = data;
 215
 216         if (!(dev->driver->driver_features & DRIVER_GEM))
 217                 return (-ENODEV);
 218
 219         pinned = 0;
 220         DRM_LOCK(dev);
 221         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 222                 pinned += obj->gtt_space->size;
 223         DRM_UNLOCK(dev);
 224
 225         args->aper_size = dev_priv->mm.gtt_total;
 226         args->aper_available_size = args->aper_size - pinned;
 227
 228         return (0);
 229 }
 230
 231 int
 232 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size,
 233     uint32_t *handle_p)
 234 {
 235         struct drm_i915_gem_object *obj;
 236         uint32_t handle;
 237         int ret;
 238
 239         size = roundup(size, PAGE_SIZE);
 240         if (size == 0)
 241                 return (-EINVAL);
 242
 243         obj = i915_gem_alloc_object(dev, size);
 244         if (obj == NULL)
 245                 return (-ENOMEM);
 246
 247         handle = 0;
 248         ret = drm_gem_handle_create(file, &obj->base, &handle);
 249         if (ret != 0) {
 250                 drm_gem_object_release(&obj->base);
 251                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 252                 drm_free(obj, DRM_I915_GEM);
 253                 return (-ret);
 254         }
 255
 256         /* drop reference from allocate - handle holds it now */
 257         drm_gem_object_unreference(&obj->base);
 258         *handle_p = handle;
 259         return (0);
 260 }
 261
 262 int
 263 i915_gem_dumb_create(struct drm_file *file,
 264                      struct drm_device *dev,
 265                      struct drm_mode_create_dumb *args)
 266 {
 267
 268         /* have to work out size/pitch and return them */
 269         args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64);
 270         args->size = args->pitch * args->height;
 271         return (i915_gem_create(file, dev, args->size, &args->handle));
 272 }
 273
 274 int i915_gem_dumb_destroy(struct drm_file *file,
 275                           struct drm_device *dev,
 276                           uint32_t handle)
 277 {
 278
 279         return (drm_gem_handle_delete(file, handle));
 280 }
 281
 282 /**
 283  * Creates a new mm object and returns a handle to it.
 284  */
 285 int
 286 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 287                       struct drm_file *file)
 288 {
 289         struct drm_i915_gem_create *args = data;
 290
 291         return (i915_gem_create(file, dev, args->size, &args->handle));
 292 }
 293
 294 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 295 {
 296         drm_i915_private_t *dev_priv;
 297
 298         dev_priv = obj->base.dev->dev_private;
 299         return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 300             obj->tiling_mode != I915_TILING_NONE);
 301 }
 302
 303 /**
 304  * Reads data from the object referenced by handle.
 305  *
 306  * On error, the contents of *data are undefined.
 307  */
 308 int
 309 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 310                      struct drm_file *file)
 311 {
 312         struct drm_i915_gem_pread *args;
 313
 314         args = data;
 315         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 316             args->offset, UIO_READ, file));
 317 }
 318
 319 /**
 320  * Writes data to the object referenced by handle.
 321  *
 322  * On error, the contents of the buffer that were to be modified are undefined.
 323  */
 324 int
 325 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 326                       struct drm_file *file)
 327 {
 328         struct drm_i915_gem_pwrite *args;
 329
 330         args = data;
 331         return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
 332             args->offset, UIO_WRITE, file));
 333 }
 334
 335 /**
 336  * Waits for a sequence number to be signaled, and cleans up the
 337  * request and object lists appropriately for that event.
 338  */
 339 int
 340 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
 341 {
 342         drm_i915_private_t *dev_priv;
 343         struct drm_i915_gem_request *request;
 344         uint32_t ier;
 345         int flags, ret;
 346         bool recovery_complete;
 347         bool do_retire = true;
 348
 349         KASSERT(seqno != 0, ("Zero seqno"));
 350
 351         dev_priv = ring->dev->dev_private;
 352         ret = 0;
 353
 354         if (atomic_read(&dev_priv->mm.wedged) != 0) {
 355                 struct completion *x = &dev_priv->error_completion;
 356                 spin_lock(&x->wait.lock);
 357                 /* Give the error handler a chance to run. */
 358                 recovery_complete = x->done > 0;
 359                 spin_unlock(&x->wait.lock);
 360                 return (recovery_complete ? -EIO : -EAGAIN);
 361         }
 362
 363         if (seqno == ring->outstanding_lazy_request) {
 364                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
 365                     M_WAITOK | M_ZERO);
 366                 if (request == NULL)
 367                         return (-ENOMEM);
 368
 369                 ret = i915_add_request(ring, NULL, request);
 370                 if (ret != 0) {
 371                         drm_free(request, DRM_I915_GEM);
 372                         return (ret);
 373                 }
 374
 375                 seqno = request->seqno;
 376         }
 377
 378         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
 379                 if (HAS_PCH_SPLIT(ring->dev))
 380                         ier = I915_READ(DEIER) | I915_READ(GTIER);
 381                 else
 382                         ier = I915_READ(IER);
 383                 if (!ier) {
 384                         DRM_ERROR("something (likely vbetool) disabled "
 385                                   "interrupts, re-enabling\n");
 386                         ring->dev->driver->irq_preinstall(ring->dev);
 387                         ring->dev->driver->irq_postinstall(ring->dev);
 388                 }
 389
 390                 lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
 391                 if (ring->irq_get(ring)) {
 392                         flags = dev_priv->mm.interruptible ? PCATCH : 0;
 393                         while (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)
 394                             && !atomic_read(&dev_priv->mm.wedged) &&
 395                             ret == 0) {
 396                                 ret = -lksleep(ring, &ring->irq_lock, flags,
 397                                     "915gwr", 1*hz);
 398                         }
 399                         ring->irq_put(ring);
 400                         lockmgr(&ring->irq_lock, LK_RELEASE);
 401                 } else {
 402                         lockmgr(&ring->irq_lock, LK_RELEASE);
 403                         if (_intel_wait_for(ring->dev,
 404                             i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
 405                             atomic_read(&dev_priv->mm.wedged), 3000,
 406                             0, "i915wrq") != 0)
 407                                 ret = -EBUSY;
 408                 }
 409
 410         }
 411         if (atomic_read(&dev_priv->mm.wedged))
 412                 ret = -EAGAIN;
 413
 414         /* Directly dispatch request retiring.  While we have the work queue
 415          * to handle this, the waiter on a request often wants an associated
 416          * buffer to have made it to the inactive list, and we would need
 417          * a separate wait queue to handle that.
 418          */
 419         if (ret == 0 && do_retire)
 420                 i915_gem_retire_requests_ring(ring);
 421
 422         return (ret);
 423 }
 424
 425 /**
 426  * Ensures that all rendering to the object has completed and the object is
 427  * safe to unbind from the GTT or access from the CPU.
 428  */
 429 int
 430 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 431 {
 432         u32 seqno;
 433         int ret;
 434
 435
 436         seqno = obj->last_rendering_seqno;
 437         if (seqno == 0)
 438                 return 0;
 439
 440         if (obj->active) {
 441                 ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
 442                 if (ret != 0)
 443                         return (ret);
 444         }
 445
 446         /* Manually manage the write flush as we may have not yet
 447          * retired the buffer.
 448          */
 449         if (obj->last_rendering_seqno &&
 450             i915_seqno_passed(seqno, obj->last_rendering_seqno)) {
 451                 obj->last_rendering_seqno = 0;
 452                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
 453         }
 454
 455         return 0;
 456 }
 457
 458 /**
 459  * Called when user space prepares to use an object with the CPU, either
 460  * through the mmap ioctl's mapping or a GTT mapping.
 461  */
 462 int
 463 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 464                           struct drm_file *file)
 465 {
 466         struct drm_i915_gem_set_domain *args;
 467         struct drm_i915_gem_object *obj;
 468         uint32_t read_domains;
 469         uint32_t write_domain;
 470         int ret;
 471
 472         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 473                 return (-ENODEV);
 474
 475         args = data;
 476         read_domains = args->read_domains;
 477         write_domain = args->write_domain;
 478
 479         if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
 480             (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
 481             (write_domain != 0 && read_domains != write_domain))
 482                 return (-EINVAL);
 483
 484         ret = i915_mutex_lock_interruptible(dev);
 485         if (ret != 0)
 486                 return (ret);
 487
 488         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 489         if (&obj->base == NULL) {
 490                 ret = -ENOENT;
 491                 goto unlock;
 492         }
 493
 494         if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) {
 495                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 496                 if (ret == -EINVAL)
 497                         ret = 0;
 498         } else
 499                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 500
 501         drm_gem_object_unreference(&obj->base);
 502 unlock:
 503         DRM_UNLOCK(dev);
 504         return (ret);
 505 }
 506
 507 /**
 508  * Called when user space has done writes to this buffer
 509  */
 510 int
 511 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 512                          struct drm_file *file)
 513 {
 514         struct drm_i915_gem_sw_finish *args;
 515         struct drm_i915_gem_object *obj;
 516         int ret;
 517
 518         args = data;
 519         ret = 0;
 520         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 521                 return (ENODEV);
 522         ret = i915_mutex_lock_interruptible(dev);
 523         if (ret != 0)
 524                 return (ret);
 525         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 526         if (&obj->base == NULL) {
 527                 ret = -ENOENT;
 528                 goto unlock;
 529         }
 530         if (obj->pin_count != 0)
 531                 i915_gem_object_flush_cpu_write_domain(obj);
 532         drm_gem_object_unreference(&obj->base);
 533 unlock:
 534         DRM_UNLOCK(dev);
 535         return (ret);
 536 }
 537
 538 /**
 539  * Maps the contents of an object, returning the address it is mapped
 540  * into.
 541  *
 542  * While the mapping holds a reference on the contents of the object, it doesn't
 543  * imply a ref on the object itself.
 544  */
 545 int
 546 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 547                     struct drm_file *file)
 548 {
 549         struct drm_i915_gem_mmap *args;
 550         struct drm_gem_object *obj;
 551         struct proc *p;
 552         vm_map_t map;
 553         vm_offset_t addr;
 554         vm_size_t size;
 555         int error, rv;
 556
 557         args = data;
 558
 559         if ((dev->driver->driver_features & DRIVER_GEM) == 0)
 560                 return (-ENODEV);
 561
 562         obj = drm_gem_object_lookup(dev, file, args->handle);
 563         if (obj == NULL)
 564                 return (-ENOENT);
 565         error = 0;
 566         if (args->size == 0)
 567                 goto out;
 568         p = curproc;
 569         map = &p->p_vmspace->vm_map;
 570         size = round_page(args->size);
 571         PROC_LOCK(p);
 572         if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) {
 573                 PROC_UNLOCK(p);
 574                 error = ENOMEM;
 575                 goto out;
 576         }
 577         PROC_UNLOCK(p);
 578
 579         addr = 0;
 580         vm_object_hold(obj->vm_obj);
 581         vm_object_reference_locked(obj->vm_obj);
 582         vm_object_drop(obj->vm_obj);
 583         DRM_UNLOCK(dev);
 584         rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size,
 585             PAGE_SIZE, /* align */
 586             TRUE, /* fitit */
 587             VM_MAPTYPE_NORMAL, /* maptype */
 588             VM_PROT_READ | VM_PROT_WRITE, /* prot */
 589             VM_PROT_READ | VM_PROT_WRITE, /* max */
 590             MAP_SHARED /* cow */);
 591         if (rv != KERN_SUCCESS) {
 592                 vm_object_deallocate(obj->vm_obj);
 593                 error = -vm_mmap_to_errno(rv);
 594         } else {
 595                 args->addr_ptr = (uint64_t)addr;
 596         }
 597         DRM_LOCK(dev);
 598 out:
 599         drm_gem_object_unreference(obj);
 600         return (error);
 601 }
 602
 603 /**
 604  * i915_gem_release_mmap - remove physical page mappings
 605  * @obj: obj in question
 606  *
 607  * Preserve the reservation of the mmapping with the DRM core code, but
 608  * relinquish ownership of the pages back to the system.
 609  *
 610  * It is vital that we remove the page mapping if we have mapped a tiled
 611  * object through the GTT and then lose the fence register due to
 612  * resource pressure. Similarly if the object has been moved out of the
 613  * aperture, than pages mapped into userspace must be revoked. Removing the
 614  * mapping will then trigger a page fault on the next user access, allowing
 615  * fixup by i915_gem_fault().
 616  */
 617 void
 618 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 619 {
 620         vm_object_t devobj;
 621         vm_page_t m;
 622         int i, page_count;
 623
 624         if (!obj->fault_mappable)
 625                 return;
 626
 627         devobj = cdev_pager_lookup(obj);
 628         if (devobj != NULL) {
 629                 page_count = OFF_TO_IDX(obj->base.size);
 630
 631                 VM_OBJECT_LOCK(devobj);
 632                 for (i = 0; i < page_count; i++) {
 633                         m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm");
 634                         if (m == NULL)
 635                                 continue;
 636                         cdev_pager_free_page(devobj, m);
 637                 }
 638                 VM_OBJECT_UNLOCK(devobj);
 639                 vm_object_deallocate(devobj);
 640         }
 641
 642         obj->fault_mappable = false;
 643 }
 644
 645 static uint32_t
 646 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 647 {
 648         uint32_t gtt_size;
 649
 650         if (INTEL_INFO(dev)->gen >= 4 ||
 651             tiling_mode == I915_TILING_NONE)
 652                 return (size);
 653
 654         /* Previous chips need a power-of-two fence region when tiling */
 655         if (INTEL_INFO(dev)->gen == 3)
 656                 gtt_size = 1024*1024;
 657         else
 658                 gtt_size = 512*1024;
 659
 660         while (gtt_size < size)
 661                 gtt_size <<= 1;
 662
 663         return (gtt_size);
 664 }
 665
 666 /**
 667  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 668  * @obj: object to check
 669  *
 670  * Return the required GTT alignment for an object, taking into account
 671  * potential fence register mapping.
 672  */
 673 static uint32_t
 674 i915_gem_get_gtt_alignment(struct drm_device *dev,
 675                            uint32_t size,
 676                            int tiling_mode)
 677 {
 678
 679         /*
 680          * Minimum alignment is 4k (GTT page size), but might be greater
 681          * if a fence register is needed for the object.
 682          */
 683         if (INTEL_INFO(dev)->gen >= 4 ||
 684             tiling_mode == I915_TILING_NONE)
 685                 return (4096);
 686
 687         /*
 688          * Previous chips need to be aligned to the size of the smallest
 689          * fence register that can contain the object.
 690          */
 691         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 692 }
 693
 694 /**
 695  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
 696  *                                       unfenced object
 697  * @dev: the device
 698  * @size: size of the object
 699  * @tiling_mode: tiling mode of the object
 700  *
 701  * Return the required GTT alignment for an object, only taking into account
 702  * unfenced tiled surface requirements.
 703  */
 704 uint32_t
 705 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 706                                     uint32_t size,
 707                                     int tiling_mode)
 708 {
 709
 710         if (tiling_mode == I915_TILING_NONE)
 711                 return (4096);
 712
 713         /*
 714          * Minimum alignment is 4k (GTT page size) for sane hw.
 715          */
 716         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev))
 717                 return (4096);
 718
 719         /*
 720          * Previous hardware however needs to be aligned to a power-of-two
 721          * tile height. The simplest method for determining this is to reuse
 722          * the power-of-tile object size.
 723          */
 724         return (i915_gem_get_gtt_size(dev, size, tiling_mode));
 725 }
 726
 727 int
 728 i915_gem_mmap_gtt(struct drm_file *file,
 729                   struct drm_device *dev,
 730                   uint32_t handle,
 731                   uint64_t *offset)
 732 {
 733         struct drm_i915_private *dev_priv;
 734         struct drm_i915_gem_object *obj;
 735         int ret;
 736
 737         if (!(dev->driver->driver_features & DRIVER_GEM))
 738                 return (-ENODEV);
 739
 740         dev_priv = dev->dev_private;
 741
 742         ret = i915_mutex_lock_interruptible(dev);
 743         if (ret != 0)
 744                 return (ret);
 745
 746         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
 747         if (&obj->base == NULL) {
 748                 ret = -ENOENT;
 749                 goto unlock;
 750         }
 751
 752         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
 753                 ret = -E2BIG;
 754                 goto out;
 755         }
 756
 757         if (obj->madv != I915_MADV_WILLNEED) {
 758                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
 759                 ret = -EINVAL;
 760                 goto out;
 761         }
 762
 763         ret = drm_gem_create_mmap_offset(&obj->base);
 764         if (ret != 0)
 765                 goto out;
 766
 767         *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) |
 768             DRM_GEM_MAPPING_KEY;
 769 out:
 770         drm_gem_object_unreference(&obj->base);
 771 unlock:
 772         DRM_UNLOCK(dev);
 773         return (ret);
 774 }
 775
 776 /**
 777  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 778  * @dev: DRM device
 779  * @data: GTT mapping ioctl data
 780  * @file: GEM object info
 781  *
 782  * Simply returns the fake offset to userspace so it can mmap it.
 783  * The mmap call will end up in drm_gem_mmap(), which will set things
 784  * up so we can get faults in the handler above.
 785  *
 786  * The fault handler will take care of binding the object into the GTT
 787  * (since it may have been evicted to make room for something), allocating
 788  * a fence register, and mapping the appropriate aperture address into
 789  * userspace.
 790  */
 791 int
 792 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 793                         struct drm_file *file)
 794 {
 795         struct drm_i915_private *dev_priv;
 796         struct drm_i915_gem_mmap_gtt *args;
 797
 798         dev_priv = dev->dev_private;
 799         args = data;
 800
 801         return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset));
 802 }
 803
 804 /* Immediately discard the backing storage */
 805 static void
 806 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 807 {
 808         vm_object_t vm_obj;
 809
 810         vm_obj = obj->base.vm_obj;
 811         VM_OBJECT_LOCK(vm_obj);
 812         vm_object_page_remove(vm_obj, 0, 0, false);
 813         VM_OBJECT_UNLOCK(vm_obj);
 814         obj->madv = __I915_MADV_PURGED;
 815 }
 816
 817 static inline int
 818 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
 819 {
 820         return obj->madv == I915_MADV_DONTNEED;
 821 }
 822
 823 static inline void vm_page_reference(vm_page_t m)
 824 {
 825         vm_page_flag_set(m, PG_REFERENCED);
 826 }
 827
 828 static void
 829 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 830 {
 831         vm_page_t m;
 832         int page_count, i;
 833
 834         BUG_ON(obj->madv == __I915_MADV_PURGED);
 835
 836         if (obj->tiling_mode != I915_TILING_NONE)
 837                 i915_gem_object_save_bit_17_swizzle(obj);
 838         if (obj->madv == I915_MADV_DONTNEED)
 839                 obj->dirty = 0;
 840         page_count = obj->base.size / PAGE_SIZE;
 841         VM_OBJECT_LOCK(obj->base.vm_obj);
 842 #if GEM_PARANOID_CHECK_GTT
 843         i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count);
 844 #endif
 845         for (i = 0; i < page_count; i++) {
 846                 m = obj->pages[i];
 847                 if (obj->dirty)
 848                         vm_page_dirty(m);
 849                 if (obj->madv == I915_MADV_WILLNEED)
 850                         vm_page_reference(m);
 851                 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem");
 852                 vm_page_unwire(obj->pages[i], 1);
 853                 vm_page_wakeup(obj->pages[i]);
 854                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 855         }
 856         VM_OBJECT_UNLOCK(obj->base.vm_obj);
 857         obj->dirty = 0;
 858         drm_free(obj->pages, DRM_I915_GEM);
 859         obj->pages = NULL;
 860 }
 861
 862 static int
 863 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 864     int flags)
 865 {
 866         struct drm_device *dev;
 867         vm_object_t vm_obj;
 868         vm_page_t m;
 869         int page_count, i, j;
 870
 871         dev = obj->base.dev;
 872         KASSERT(obj->pages == NULL, ("Obj already has pages"));
 873         page_count = obj->base.size / PAGE_SIZE;
 874         obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 875             M_WAITOK);
 876         vm_obj = obj->base.vm_obj;
 877         VM_OBJECT_LOCK(vm_obj);
 878         for (i = 0; i < page_count; i++) {
 879                 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
 880                         goto failed;
 881         }
 882         VM_OBJECT_UNLOCK(vm_obj);
 883         if (i915_gem_object_needs_bit17_swizzle(obj))
 884                 i915_gem_object_do_bit_17_swizzle(obj);
 885         return (0);
 886
 887 failed:
 888         for (j = 0; j < i; j++) {
 889                 m = obj->pages[j];
 890                 vm_page_busy_wait(m, FALSE, "i915gem");
 891                 vm_page_unwire(m, 0);
 892                 vm_page_wakeup(m);
 893                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
 894         }
 895         VM_OBJECT_UNLOCK(vm_obj);
 896         drm_free(obj->pages, DRM_I915_GEM);
 897         obj->pages = NULL;
 898         return (-EIO);
 899 }
 900
 901 void
 902 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 903     struct intel_ring_buffer *ring, uint32_t seqno)
 904 {
 905         struct drm_device *dev = obj->base.dev;
 906         struct drm_i915_private *dev_priv = dev->dev_private;
 907         struct drm_i915_fence_reg *reg;
 908
 909         obj->ring = ring;
 910         KASSERT(ring != NULL, ("NULL ring"));
 911
 912         /* Add a reference if we're newly entering the active list. */
 913         if (!obj->active) {
 914                 drm_gem_object_reference(&obj->base);
 915                 obj->active = 1;
 916         }
 917
 918         /* Move from whatever list we were on to the tail of execution. */
 919         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 920         list_move_tail(&obj->ring_list, &ring->active_list);
 921
 922         obj->last_rendering_seqno = seqno;
 923         if (obj->fenced_gpu_access) {
 924                 obj->last_fenced_seqno = seqno;
 925                 obj->last_fenced_ring = ring;
 926
 927                 /* Bump MRU to take account of the delayed flush */
 928                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
 929                         reg = &dev_priv->fence_regs[obj->fence_reg];
 930                         list_move_tail(&reg->lru_list,
 931                                        &dev_priv->mm.fence_list);
 932                 }
 933         }
 934 }
 935
 936 static void
 937 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
 938 {
 939         list_del_init(&obj->ring_list);
 940         obj->last_rendering_seqno = 0;
 941         obj->last_fenced_seqno = 0;
 942 }
 943
 944 static void
 945 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 946 {
 947         struct drm_device *dev = obj->base.dev;
 948         struct drm_i915_private *dev_priv = dev->dev_private;
 949
 950         if (obj->pin_count != 0)
 951                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
 952         else
 953                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 954
 955         KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
 956         KASSERT(obj->active, ("Object not active"));
 957         obj->ring = NULL;
 958         obj->last_fenced_ring = NULL;
 959
 960         i915_gem_object_move_off_active(obj);
 961         obj->fenced_gpu_access = false;
 962
 963         obj->active = 0;
 964         obj->pending_gpu_write = false;
 965         drm_gem_object_unreference(&obj->base);
 966
 967 #if 1
 968         KIB_NOTYET();
 969 #else
 970         WARN_ON(i915_verify_lists(dev));
 971 #endif
 972 }
 973
 974 static u32
 975 i915_gem_get_seqno(struct drm_device *dev)
 976 {
 977         drm_i915_private_t *dev_priv = dev->dev_private;
 978         u32 seqno = dev_priv->next_seqno;
 979
 980         /* reserve 0 for non-seqno */
 981         if (++dev_priv->next_seqno == 0)
 982                 dev_priv->next_seqno = 1;
 983
 984         return seqno;
 985 }
 986
 987 int
 988 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
 989      struct drm_i915_gem_request *request)
 990 {
 991         drm_i915_private_t *dev_priv;
 992         struct drm_i915_file_private *file_priv;
 993         uint32_t seqno;
 994         u32 request_ring_position;
 995         int was_empty;
 996         int ret;
 997
 998         KASSERT(request != NULL, ("NULL request in add"));
 999         DRM_LOCK_ASSERT(ring->dev);
1000         dev_priv = ring->dev->dev_private;
1001
1002         seqno = i915_gem_next_request_seqno(ring);
1003         request_ring_position = intel_ring_get_tail(ring);
1004
1005         ret = ring->add_request(ring, &seqno);
1006         if (ret != 0)
1007             return ret;
1008
1009         request->seqno = seqno;
1010         request->ring = ring;
1011         request->tail = request_ring_position;
1012         request->emitted_jiffies = ticks;
1013         was_empty = list_empty(&ring->request_list);
1014         list_add_tail(&request->list, &ring->request_list);
1015
1016         if (file != NULL) {
1017                 file_priv = file->driver_priv;
1018
1019                 spin_lock(&file_priv->mm.lock);
1020                 request->file_priv = file_priv;
1021                 list_add_tail(&request->client_list,
1022                     &file_priv->mm.request_list);
1023                 spin_unlock(&file_priv->mm.lock);
1024         }
1025
1026         ring->outstanding_lazy_request = 0;
1027
1028         if (!dev_priv->mm.suspended) {
1029                 if (i915_enable_hangcheck) {
1030                         mod_timer(&dev_priv->hangcheck_timer,
1031                                   round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
1032                 }
1033                 if (was_empty) {
1034                         queue_delayed_work(dev_priv->wq,
1035                                            &dev_priv->mm.retire_work,
1036                                            round_jiffies_up_relative(hz));
1037                         intel_mark_busy(dev_priv->dev);
1038                 }
1039         }
1040         return (0);
1041 }
1042
1043 static inline void
1044 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1045 {
1046         struct drm_i915_file_private *file_priv = request->file_priv;
1047
1048         if (!file_priv)
1049                 return;
1050
1051         DRM_LOCK_ASSERT(request->ring->dev);
1052
1053         spin_lock(&file_priv->mm.lock);
1054         if (request->file_priv != NULL) {
1055                 list_del(&request->client_list);
1056                 request->file_priv = NULL;
1057         }
1058         spin_unlock(&file_priv->mm.lock);
1059 }
1060
1061 static void
1062 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1063     struct intel_ring_buffer *ring)
1064 {
1065
1066         if (ring->dev != NULL)
1067                 DRM_LOCK_ASSERT(ring->dev);
1068
1069         while (!list_empty(&ring->request_list)) {
1070                 struct drm_i915_gem_request *request;
1071
1072                 request = list_first_entry(&ring->request_list,
1073                     struct drm_i915_gem_request, list);
1074
1075                 list_del(&request->list);
1076                 i915_gem_request_remove_from_client(request);
1077                 drm_free(request, DRM_I915_GEM);
1078         }
1079
1080         while (!list_empty(&ring->active_list)) {
1081                 struct drm_i915_gem_object *obj;
1082
1083                 obj = list_first_entry(&ring->active_list,
1084                     struct drm_i915_gem_object, ring_list);
1085
1086                 obj->base.write_domain = 0;
1087                 list_del_init(&obj->gpu_write_list);
1088                 i915_gem_object_move_to_inactive(obj);
1089         }
1090 }
1091
1092 static void
1093 i915_gem_reset_fences(struct drm_device *dev)
1094 {
1095         struct drm_i915_private *dev_priv = dev->dev_private;
1096         int i;
1097
1098         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1099                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1100                 struct drm_i915_gem_object *obj = reg->obj;
1101
1102                 if (!obj)
1103                         continue;
1104
1105                 if (obj->tiling_mode)
1106                         i915_gem_release_mmap(obj);
1107
1108                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1109                 reg->obj->fenced_gpu_access = false;
1110                 reg->obj->last_fenced_seqno = 0;
1111                 reg->obj->last_fenced_ring = NULL;
1112                 i915_gem_clear_fence_reg(dev, reg);
1113         }
1114 }
1115
1116 void i915_gem_reset(struct drm_device *dev)
1117 {
1118         struct drm_i915_private *dev_priv = dev->dev_private;
1119         struct drm_i915_gem_object *obj;
1120         int i;
1121
1122         for (i = 0; i < I915_NUM_RINGS; i++)
1123                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1124
1125         /* Remove anything from the flushing lists. The GPU cache is likely
1126          * to be lost on reset along with the data, so simply move the
1127          * lost bo to the inactive list.
1128          */
1129         while (!list_empty(&dev_priv->mm.flushing_list)) {
1130                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1131                                       struct drm_i915_gem_object,
1132                                       mm_list);
1133
1134                 obj->base.write_domain = 0;
1135                 list_del_init(&obj->gpu_write_list);
1136                 i915_gem_object_move_to_inactive(obj);
1137         }
1138
1139         /* Move everything out of the GPU domains to ensure we do any
1140          * necessary invalidation upon reuse.
1141          */
1142         list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
1143                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1144         }
1145
1146         /* The fence registers are invalidated so clear them out */
1147         i915_gem_reset_fences(dev);
1148 }
1149
1150 static void
1151 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1152 {
1153         struct drm_device *dev = obj->base.dev;
1154         drm_i915_private_t *dev_priv = dev->dev_private;
1155
1156         KASSERT(obj->active, ("Object not active"));
1157         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1158
1159         i915_gem_object_move_off_active(obj);
1160 }
1161
1162 /**
1163  * This function clears the request list as sequence numbers are passed.
1164  */
1165 void
1166 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1167 {
1168         uint32_t seqno;
1169
1170         if (list_empty(&ring->request_list))
1171                 return;
1172
1173         seqno = ring->get_seqno(ring, true);
1174
1175         while (!list_empty(&ring->request_list)) {
1176                 struct drm_i915_gem_request *request;
1177
1178                 request = list_first_entry(&ring->request_list,
1179                                            struct drm_i915_gem_request,
1180                                            list);
1181
1182                 if (!i915_seqno_passed(seqno, request->seqno))
1183                         break;
1184
1185                 /* We know the GPU must have read the request to have
1186                  * sent us the seqno + interrupt, so use the position
1187                  * of tail of the request to update the last known position
1188                  * of the GPU head.
1189                  */
1190                 ring->last_retired_head = request->tail;
1191
1192                 list_del(&request->list);
1193                 i915_gem_request_remove_from_client(request);
1194                 drm_free(request, DRM_I915_GEM);
1195         }
1196
1197         /* Move any buffers on the active list that are no longer referenced
1198          * by the ringbuffer to the flushing/inactive lists as appropriate.
1199          */
1200         while (!list_empty(&ring->active_list)) {
1201                 struct drm_i915_gem_object *obj;
1202
1203                 obj = list_first_entry(&ring->active_list,
1204                                       struct drm_i915_gem_object,
1205                                       ring_list);
1206
1207                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1208                         break;
1209
1210                 if (obj->base.write_domain != 0)
1211                         i915_gem_object_move_to_flushing(obj);
1212                 else
1213                         i915_gem_object_move_to_inactive(obj);
1214         }
1215
1216         if (unlikely(ring->trace_irq_seqno &&
1217                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1218                 ring->irq_put(ring);
1219                 ring->trace_irq_seqno = 0;
1220         }
1221
1222 }
1223
1224 static void
1225 i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
1226
1227 void
1228 i915_gem_retire_requests(struct drm_device *dev)
1229 {
1230         drm_i915_private_t *dev_priv = dev->dev_private;
1231         struct drm_i915_gem_object *obj, *next;
1232         int i;
1233
1234         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1235                 list_for_each_entry_safe(obj, next,
1236                     &dev_priv->mm.deferred_free_list, mm_list)
1237                         i915_gem_free_object_tail(obj);
1238         }
1239
1240         for (i = 0; i < I915_NUM_RINGS; i++)
1241                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1242 }
1243
1244 static void
1245 i915_gem_retire_work_handler(struct work_struct *work)
1246 {
1247         drm_i915_private_t *dev_priv;
1248         struct drm_device *dev;
1249         struct intel_ring_buffer *ring;
1250         bool idle;
1251         int i;
1252
1253         dev_priv = container_of(work, drm_i915_private_t,
1254                                 mm.retire_work.work);
1255         dev = dev_priv->dev;
1256
1257         /* Come back later if the device is busy... */
1258         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) {
1259                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1260                                    round_jiffies_up_relative(hz));
1261                 return;
1262         }
1263
1264         i915_gem_retire_requests(dev);
1265
1266         /* Send a periodic flush down the ring so we don't hold onto GEM
1267          * objects indefinitely.
1268          */
1269         idle = true;
1270         for_each_ring(ring, dev_priv, i) {
1271                 if (ring->gpu_caches_dirty)
1272                         i915_add_request(ring, NULL, NULL);
1273
1274                 idle &= list_empty(&ring->request_list);
1275         }
1276
1277         if (!dev_priv->mm.suspended && !idle)
1278                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
1279                                    round_jiffies_up_relative(hz));
1280         if (idle)
1281                 intel_mark_idle(dev);
1282
1283         DRM_UNLOCK(dev);
1284 }
1285
1286 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1287 {
1288         u32 old_write_domain, old_read_domains;
1289
1290         /* Act a barrier for all accesses through the GTT */
1291         cpu_mfence();
1292
1293         /* Force a pagefault for domain tracking on next user access */
1294         i915_gem_release_mmap(obj);
1295
1296         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1297                 return;
1298
1299         old_read_domains = obj->base.read_domains;
1300         old_write_domain = obj->base.write_domain;
1301
1302         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1303         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1304
1305 }
1306
1307 int
1308 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1309 {
1310         drm_i915_private_t *dev_priv;
1311         int ret;
1312
1313         dev_priv = obj->base.dev->dev_private;
1314         ret = 0;
1315         if (obj->gtt_space == NULL)
1316                 return (0);
1317         if (obj->pin_count != 0) {
1318                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1319                 return (-EINVAL);
1320         }
1321
1322         ret = i915_gem_object_finish_gpu(obj);
1323         if (ret == -ERESTART || ret == -EINTR)
1324                 return (ret);
1325
1326         i915_gem_object_finish_gtt(obj);
1327
1328         if (ret == 0)
1329                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1330         if (ret == -ERESTART || ret == -EINTR)
1331                 return (ret);
1332         if (ret != 0) {
1333                 i915_gem_clflush_object(obj);
1334                 obj->base.read_domains = obj->base.write_domain =
1335                     I915_GEM_DOMAIN_CPU;
1336         }
1337
1338         ret = i915_gem_object_put_fence(obj);
1339         if (ret == -ERESTART)
1340                 return (ret);
1341
1342         i915_gem_gtt_unbind_object(obj);
1343         if (obj->has_aliasing_ppgtt_mapping) {
1344                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
1345                 obj->has_aliasing_ppgtt_mapping = 0;
1346         }
1347         i915_gem_object_put_pages_gtt(obj);
1348
1349         list_del_init(&obj->gtt_list);
1350         list_del_init(&obj->mm_list);
1351         obj->map_and_fenceable = true;
1352
1353         drm_mm_put_block(obj->gtt_space);
1354         obj->gtt_space = NULL;
1355         obj->gtt_offset = 0;
1356
1357         if (i915_gem_object_is_purgeable(obj))
1358                 i915_gem_object_truncate(obj);
1359
1360         return (ret);
1361 }
1362
1363 int i915_gpu_idle(struct drm_device *dev)
1364 {
1365         drm_i915_private_t *dev_priv = dev->dev_private;
1366         struct intel_ring_buffer *ring;
1367         int ret, i;
1368
1369         /* Flush everything onto the inactive list. */
1370         for_each_ring(ring, dev_priv, i) {
1371                 ret = intel_ring_idle(ring);
1372                 if (ret)
1373                         return ret;
1374         }
1375
1376         return 0;
1377 }
1378
1379 static int
1380 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
1381     struct intel_ring_buffer *pipelined)
1382 {
1383         struct drm_device *dev = obj->base.dev;
1384         drm_i915_private_t *dev_priv = dev->dev_private;
1385         u32 size = obj->gtt_space->size;
1386         int regnum = obj->fence_reg;
1387         uint64_t val;
1388
1389         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1390                          0xfffff000) << 32;
1391         val |= obj->gtt_offset & 0xfffff000;
1392         val |= (uint64_t)((obj->stride / 128) - 1) <<
1393                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
1394
1395         if (obj->tiling_mode == I915_TILING_Y)
1396                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1397         val |= I965_FENCE_REG_VALID;
1398
1399         if (pipelined) {
1400                 int ret = intel_ring_begin(pipelined, 6);
1401                 if (ret)
1402                         return ret;
1403
1404                 intel_ring_emit(pipelined, MI_NOOP);
1405                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1406                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
1407                 intel_ring_emit(pipelined, (u32)val);
1408                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
1409                 intel_ring_emit(pipelined, (u32)(val >> 32));
1410                 intel_ring_advance(pipelined);
1411         } else
1412                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
1413
1414         return 0;
1415 }
1416
1417 static int
1418 i965_write_fence_reg(struct drm_i915_gem_object *obj,
1419     struct intel_ring_buffer *pipelined)
1420 {
1421         struct drm_device *dev = obj->base.dev;
1422         drm_i915_private_t *dev_priv = dev->dev_private;
1423         u32 size = obj->gtt_space->size;
1424         int regnum = obj->fence_reg;
1425         uint64_t val;
1426
1427         val = (uint64_t)((obj->gtt_offset + size - 4096) &
1428                     0xfffff000) << 32;
1429         val |= obj->gtt_offset & 0xfffff000;
1430         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1431         if (obj->tiling_mode == I915_TILING_Y)
1432                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1433         val |= I965_FENCE_REG_VALID;
1434
1435         if (pipelined) {
1436                 int ret = intel_ring_begin(pipelined, 6);
1437                 if (ret)
1438                         return ret;
1439
1440                 intel_ring_emit(pipelined, MI_NOOP);
1441                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
1442                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
1443                 intel_ring_emit(pipelined, (u32)val);
1444                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
1445                 intel_ring_emit(pipelined, (u32)(val >> 32));
1446                 intel_ring_advance(pipelined);
1447         } else
1448                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
1449
1450         return 0;
1451 }
1452
1453 static int
1454 i915_write_fence_reg(struct drm_i915_gem_object *obj,
1455     struct intel_ring_buffer *pipelined)
1456 {
1457         struct drm_device *dev = obj->base.dev;
1458         drm_i915_private_t *dev_priv = dev->dev_private;
1459         u32 size = obj->gtt_space->size;
1460         u32 fence_reg, val, pitch_val;
1461         int tile_width;
1462
1463         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
1464             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1465                 kprintf(
1466 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
1467                  obj->gtt_offset, obj->map_and_fenceable, size);
1468                 return -EINVAL;
1469         }
1470
1471         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
1472                 tile_width = 128;
1473         else
1474                 tile_width = 512;
1475
1476         /* Note: pitch better be a power of two tile widths */
1477         pitch_val = obj->stride / tile_width;
1478         pitch_val = ffs(pitch_val) - 1;
1479
1480         val = obj->gtt_offset;
1481         if (obj->tiling_mode == I915_TILING_Y)
1482                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1483         val |= I915_FENCE_SIZE_BITS(size);
1484         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1485         val |= I830_FENCE_REG_VALID;
1486
1487         fence_reg = obj->fence_reg;
1488         if (fence_reg < 8)
1489                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
1490         else
1491                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
1492
1493         if (pipelined) {
1494                 int ret = intel_ring_begin(pipelined, 4);
1495                 if (ret)
1496                         return ret;
1497
1498                 intel_ring_emit(pipelined, MI_NOOP);
1499                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1500                 intel_ring_emit(pipelined, fence_reg);
1501                 intel_ring_emit(pipelined, val);
1502                 intel_ring_advance(pipelined);
1503         } else
1504                 I915_WRITE(fence_reg, val);
1505
1506         return 0;
1507 }
1508
1509 static int
1510 i830_write_fence_reg(struct drm_i915_gem_object *obj,
1511     struct intel_ring_buffer *pipelined)
1512 {
1513         struct drm_device *dev = obj->base.dev;
1514         drm_i915_private_t *dev_priv = dev->dev_private;
1515         u32 size = obj->gtt_space->size;
1516         int regnum = obj->fence_reg;
1517         uint32_t val;
1518         uint32_t pitch_val;
1519
1520         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
1521             (size & -size) != size || (obj->gtt_offset & (size - 1))) {
1522                 kprintf(
1523 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
1524                     obj->gtt_offset, size);
1525                 return -EINVAL;
1526         }
1527
1528         pitch_val = obj->stride / 128;
1529         pitch_val = ffs(pitch_val) - 1;
1530
1531         val = obj->gtt_offset;
1532         if (obj->tiling_mode == I915_TILING_Y)
1533                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1534         val |= I830_FENCE_SIZE_BITS(size);
1535         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1536         val |= I830_FENCE_REG_VALID;
1537
1538         if (pipelined) {
1539                 int ret = intel_ring_begin(pipelined, 4);
1540                 if (ret)
1541                         return ret;
1542
1543                 intel_ring_emit(pipelined, MI_NOOP);
1544                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
1545                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
1546                 intel_ring_emit(pipelined, val);
1547                 intel_ring_advance(pipelined);
1548         } else
1549                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
1550
1551         return 0;
1552 }
1553
1554 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
1555 {
1556         return i915_seqno_passed(ring->get_seqno(ring,false), seqno);
1557 }
1558
1559 static int
1560 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
1561     struct intel_ring_buffer *pipelined)
1562 {
1563         int ret;
1564
1565         if (obj->fenced_gpu_access) {
1566                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
1567                         ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
1568                             obj->base.write_domain);
1569                         if (ret)
1570                                 return ret;
1571                 }
1572
1573                 obj->fenced_gpu_access = false;
1574         }
1575
1576         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
1577                 if (!ring_passed_seqno(obj->last_fenced_ring,
1578                                        obj->last_fenced_seqno)) {
1579                         ret = i915_wait_seqno(obj->last_fenced_ring,
1580                                                 obj->last_fenced_seqno);
1581                         if (ret)
1582                                 return ret;
1583                 }
1584
1585                 obj->last_fenced_seqno = 0;
1586                 obj->last_fenced_ring = NULL;
1587         }
1588
1589         /* Ensure that all CPU reads are completed before installing a fence
1590          * and all writes before removing the fence.
1591          */
1592         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
1593                 cpu_mfence();
1594
1595         return 0;
1596 }
1597
1598 int
1599 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
1600 {
1601         int ret;
1602
1603         if (obj->tiling_mode)
1604                 i915_gem_release_mmap(obj);
1605
1606         ret = i915_gem_object_flush_fence(obj, NULL);
1607         if (ret)
1608                 return ret;
1609
1610         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1611                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1612
1613                 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
1614                         kprintf("%s: pin_count %d\n", __func__,
1615                             dev_priv->fence_regs[obj->fence_reg].pin_count);
1616                 i915_gem_clear_fence_reg(obj->base.dev,
1617                                          &dev_priv->fence_regs[obj->fence_reg]);
1618
1619                 obj->fence_reg = I915_FENCE_REG_NONE;
1620         }
1621
1622         return 0;
1623 }
1624
1625 static struct drm_i915_fence_reg *
1626 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
1627 {
1628         struct drm_i915_private *dev_priv = dev->dev_private;
1629         struct drm_i915_fence_reg *reg, *first, *avail;
1630         int i;
1631
1632         /* First try to find a free reg */
1633         avail = NULL;
1634         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
1635                 reg = &dev_priv->fence_regs[i];
1636                 if (!reg->obj)
1637                         return reg;
1638
1639                 if (!reg->pin_count)
1640                         avail = reg;
1641         }
1642
1643         if (avail == NULL)
1644                 return NULL;
1645
1646         /* None available, try to steal one or wait for a user to finish */
1647         avail = first = NULL;
1648         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
1649                 if (reg->pin_count)
1650                         continue;
1651
1652                 if (first == NULL)
1653                         first = reg;
1654
1655                 if (!pipelined ||
1656                     !reg->obj->last_fenced_ring ||
1657                     reg->obj->last_fenced_ring == pipelined) {
1658                         avail = reg;
1659                         break;
1660                 }
1661         }
1662
1663         if (avail == NULL)
1664                 avail = first;
1665
1666         return avail;
1667 }
1668
1669 int
1670 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1671     struct intel_ring_buffer *pipelined)
1672 {
1673         struct drm_device *dev = obj->base.dev;
1674         struct drm_i915_private *dev_priv = dev->dev_private;
1675         struct drm_i915_fence_reg *reg;
1676         int ret;
1677
1678         pipelined = NULL;
1679         ret = 0;
1680
1681         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1682                 reg = &dev_priv->fence_regs[obj->fence_reg];
1683                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1684
1685                 if (obj->tiling_changed) {
1686                         ret = i915_gem_object_flush_fence(obj, pipelined);
1687                         if (ret)
1688                                 return ret;
1689
1690                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1691                                 pipelined = NULL;
1692
1693                         if (pipelined) {
1694                                 reg->setup_seqno =
1695                                         i915_gem_next_request_seqno(pipelined);
1696                                 obj->last_fenced_seqno = reg->setup_seqno;
1697                                 obj->last_fenced_ring = pipelined;
1698                         }
1699
1700                         goto update;
1701                 }
1702
1703                 if (!pipelined) {
1704                         if (reg->setup_seqno) {
1705                                 if (!ring_passed_seqno(obj->last_fenced_ring,
1706                                     reg->setup_seqno)) {
1707                                         ret = i915_wait_seqno(
1708                                             obj->last_fenced_ring,
1709                                             reg->setup_seqno);
1710                                         if (ret)
1711                                                 return ret;
1712                                 }
1713
1714                                 reg->setup_seqno = 0;
1715                         }
1716                 } else if (obj->last_fenced_ring &&
1717                            obj->last_fenced_ring != pipelined) {
1718                         ret = i915_gem_object_flush_fence(obj, pipelined);
1719                         if (ret)
1720                                 return ret;
1721                 }
1722
1723                 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
1724                         pipelined = NULL;
1725                 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
1726
1727                 if (obj->tiling_changed) {
1728                         if (pipelined) {
1729                                 reg->setup_seqno =
1730                                         i915_gem_next_request_seqno(pipelined);
1731                                 obj->last_fenced_seqno = reg->setup_seqno;
1732                                 obj->last_fenced_ring = pipelined;
1733                         }
1734                         goto update;
1735                 }
1736
1737                 return 0;
1738         }
1739
1740         reg = i915_find_fence_reg(dev, pipelined);
1741         if (reg == NULL)
1742                 return -EDEADLK;
1743
1744         ret = i915_gem_object_flush_fence(obj, pipelined);
1745         if (ret)
1746                 return ret;
1747
1748         if (reg->obj) {
1749                 struct drm_i915_gem_object *old = reg->obj;
1750
1751                 drm_gem_object_reference(&old->base);
1752
1753                 if (old->tiling_mode)
1754                         i915_gem_release_mmap(old);
1755
1756                 ret = i915_gem_object_flush_fence(old, pipelined);
1757                 if (ret) {
1758                         drm_gem_object_unreference(&old->base);
1759                         return ret;
1760                 }
1761
1762                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
1763                         pipelined = NULL;
1764
1765                 old->fence_reg = I915_FENCE_REG_NONE;
1766                 old->last_fenced_ring = pipelined;
1767                 old->last_fenced_seqno =
1768                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1769
1770                 drm_gem_object_unreference(&old->base);
1771         } else if (obj->last_fenced_seqno == 0)
1772                 pipelined = NULL;
1773
1774         reg->obj = obj;
1775         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1776         obj->fence_reg = reg - dev_priv->fence_regs;
1777         obj->last_fenced_ring = pipelined;
1778
1779         reg->setup_seqno =
1780                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
1781         obj->last_fenced_seqno = reg->setup_seqno;
1782
1783 update:
1784         obj->tiling_changed = false;
1785         switch (INTEL_INFO(dev)->gen) {
1786         case 7:
1787         case 6:
1788                 ret = sandybridge_write_fence_reg(obj, pipelined);
1789                 break;
1790         case 5:
1791         case 4:
1792                 ret = i965_write_fence_reg(obj, pipelined);
1793                 break;
1794         case 3:
1795                 ret = i915_write_fence_reg(obj, pipelined);
1796                 break;
1797         case 2:
1798                 ret = i830_write_fence_reg(obj, pipelined);
1799                 break;
1800         }
1801
1802         return ret;
1803 }
1804
1805 static int
1806 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
1807     unsigned alignment, bool map_and_fenceable)
1808 {
1809         struct drm_device *dev;
1810         struct drm_i915_private *dev_priv;
1811         struct drm_mm_node *free_space;
1812         uint32_t size, fence_size, fence_alignment, unfenced_alignment;
1813         bool mappable, fenceable;
1814         int ret;
1815
1816         dev = obj->base.dev;
1817         dev_priv = dev->dev_private;
1818
1819         if (obj->madv != I915_MADV_WILLNEED) {
1820                 DRM_ERROR("Attempting to bind a purgeable object\n");
1821                 return (-EINVAL);
1822         }
1823
1824         fence_size = i915_gem_get_gtt_size(dev, obj->base.size,
1825             obj->tiling_mode);
1826         fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size,
1827             obj->tiling_mode);
1828         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev,
1829             obj->base.size, obj->tiling_mode);
1830         if (alignment == 0)
1831                 alignment = map_and_fenceable ? fence_alignment :
1832                     unfenced_alignment;
1833         if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) {
1834                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1835                 return (-EINVAL);
1836         }
1837
1838         size = map_and_fenceable ? fence_size : obj->base.size;
1839
1840         /* If the object is bigger than the entire aperture, reject it early
1841          * before evicting everything in a vain attempt to find space.
1842          */
1843         if (obj->base.size > (map_and_fenceable ?
1844             dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
1845                 DRM_ERROR(
1846 "Attempting to bind an object larger than the aperture\n");
1847                 return (-E2BIG);
1848         }
1849
1850  search_free:
1851         if (map_and_fenceable)
1852                 free_space = drm_mm_search_free_in_range(
1853                     &dev_priv->mm.gtt_space, size, alignment, 0,
1854                     dev_priv->mm.gtt_mappable_end, 0);
1855         else
1856                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1857                     size, alignment, 0);
1858         if (free_space != NULL) {
1859                 int color = 0;
1860                 if (map_and_fenceable)
1861                         obj->gtt_space = drm_mm_get_block_range_generic(
1862                             free_space, size, alignment, color, 0,
1863                             dev_priv->mm.gtt_mappable_end, 1);
1864                 else
1865                         obj->gtt_space = drm_mm_get_block_generic(free_space,
1866                             size, alignment, color, 1);
1867         }
1868         if (obj->gtt_space == NULL) {
1869                 ret = i915_gem_evict_something(dev, size, alignment,
1870                     map_and_fenceable);
1871                 if (ret != 0)
1872                         return (ret);
1873                 goto search_free;
1874         }
1875
1876         /*
1877          * NOTE: i915_gem_object_get_pages_gtt() cannot
1878          *       return ENOMEM, since we used VM_ALLOC_RETRY.
1879          */
1880         ret = i915_gem_object_get_pages_gtt(obj, 0);
1881         if (ret != 0) {
1882                 drm_mm_put_block(obj->gtt_space);
1883                 obj->gtt_space = NULL;
1884                 return (ret);
1885         }
1886
1887         i915_gem_gtt_bind_object(obj, obj->cache_level);
1888         if (ret != 0) {
1889                 i915_gem_object_put_pages_gtt(obj);
1890                 drm_mm_put_block(obj->gtt_space);
1891                 obj->gtt_space = NULL;
1892                 if (i915_gem_evict_everything(dev, false))
1893                         return (ret);
1894                 goto search_free;
1895         }
1896
1897         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
1898         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1899
1900         obj->gtt_offset = obj->gtt_space->start;
1901
1902         fenceable =
1903                 obj->gtt_space->size == fence_size &&
1904                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
1905
1906         mappable =
1907                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
1908         obj->map_and_fenceable = mappable && fenceable;
1909
1910         return (0);
1911 }
1912
1913 void
1914 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
1915 {
1916
1917         /* If we don't have a page list set up, then we're not pinned
1918          * to GPU, and we can ignore the cache flush because it'll happen
1919          * again at bind time.
1920          */
1921         if (obj->pages == NULL)
1922                 return;
1923
1924         /* If the GPU is snooping the contents of the CPU cache,
1925          * we do not need to manually clear the CPU cache lines.  However,
1926          * the caches are only snooped when the render cache is
1927          * flushed/invalidated.  As we always have to emit invalidations
1928          * and flushes when moving into and out of the RENDER domain, correct
1929          * snooping behaviour occurs naturally as the result of our domain
1930          * tracking.
1931          */
1932         if (obj->cache_level != I915_CACHE_NONE)
1933                 return;
1934
1935         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
1936 }
1937
1938 /** Flushes the GTT write domain for the object if it's dirty. */
1939 static void
1940 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
1941 {
1942         uint32_t old_write_domain;
1943
1944         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
1945                 return;
1946
1947         /* No actual flushing is required for the GTT write domain.  Writes
1948          * to it immediately go to main memory as far as we know, so there's
1949          * no chipset flush.  It also doesn't land in render cache.
1950          *
1951          * However, we do have to enforce the order so that all writes through
1952          * the GTT land before any writes to the device, such as updates to
1953          * the GATT itself.
1954          */
1955         cpu_sfence();
1956
1957         old_write_domain = obj->base.write_domain;
1958         obj->base.write_domain = 0;
1959 }
1960
1961 /** Flushes the CPU write domain for the object if it's dirty. */
1962 static void
1963 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
1964 {
1965         uint32_t old_write_domain;
1966
1967         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
1968                 return;
1969
1970         i915_gem_clflush_object(obj);
1971         intel_gtt_chipset_flush();
1972         old_write_domain = obj->base.write_domain;
1973         obj->base.write_domain = 0;
1974 }
1975
1976 static int
1977 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
1978 {
1979
1980         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
1981                 return (0);
1982         return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain));
1983 }
1984
1985 /**
1986  * Moves a single object to the GTT read, and possibly write domain.
1987  *
1988  * This function returns when the move is complete, including waiting on
1989  * flushes to occur.
1990  */
1991 int
1992 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
1993 {
1994         uint32_t old_write_domain, old_read_domains;
1995         int ret;
1996
1997         if (obj->gtt_space == NULL)
1998                 return (-EINVAL);
1999
2000         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2001                 return 0;
2002
2003         ret = i915_gem_object_flush_gpu_write_domain(obj);
2004         if (ret != 0)
2005                 return (ret);
2006
2007         if (obj->pending_gpu_write || write) {
2008                 ret = i915_gem_object_wait_rendering(obj);
2009                 if (ret != 0)
2010                         return (ret);
2011         }
2012
2013         i915_gem_object_flush_cpu_write_domain(obj);
2014
2015         old_write_domain = obj->base.write_domain;
2016         old_read_domains = obj->base.read_domains;
2017
2018         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2019             ("In GTT write domain"));
2020         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2021         if (write) {
2022                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2023                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2024                 obj->dirty = 1;
2025         }
2026
2027         return (0);
2028 }
2029
2030 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2031                                     enum i915_cache_level cache_level)
2032 {
2033         struct drm_device *dev = obj->base.dev;
2034         drm_i915_private_t *dev_priv = dev->dev_private;
2035         int ret;
2036
2037         if (obj->cache_level == cache_level)
2038                 return 0;
2039
2040         if (obj->pin_count) {
2041                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2042                 return -EBUSY;
2043         }
2044
2045         if (obj->gtt_space) {
2046                 ret = i915_gem_object_finish_gpu(obj);
2047                 if (ret != 0)
2048                         return (ret);
2049
2050                 i915_gem_object_finish_gtt(obj);
2051
2052                 /* Before SandyBridge, you could not use tiling or fence
2053                  * registers with snooped memory, so relinquish any fences
2054                  * currently pointing to our region in the aperture.
2055                  */
2056                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2057                         ret = i915_gem_object_put_fence(obj);
2058                         if (ret)
2059                                 return ret;
2060                 }
2061
2062                 if (obj->has_global_gtt_mapping)
2063                         i915_gem_gtt_bind_object(obj, cache_level);
2064                 if (obj->has_aliasing_ppgtt_mapping)
2065                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2066                                                obj, cache_level);
2067         }
2068
2069         if (cache_level == I915_CACHE_NONE) {
2070                 u32 old_read_domains, old_write_domain;
2071
2072                 /* If we're coming from LLC cached, then we haven't
2073                  * actually been tracking whether the data is in the
2074                  * CPU cache or not, since we only allow one bit set
2075                  * in obj->write_domain and have been skipping the clflushes.
2076                  * Just set it to the CPU cache for now.
2077                  */
2078                 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
2079                     ("obj %p in CPU write domain", obj));
2080                 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0,
2081                     ("obj %p in CPU read domain", obj));
2082
2083                 old_read_domains = obj->base.read_domains;
2084                 old_write_domain = obj->base.write_domain;
2085
2086                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2087                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2088
2089         }
2090
2091         obj->cache_level = cache_level;
2092         return 0;
2093 }
2094
2095 /*
2096  * Prepare buffer for display plane (scanout, cursors, etc).
2097  * Can be called from an uninterruptible phase (modesetting) and allows
2098  * any flushes to be pipelined (for pageflips).
2099  */
2100 int
2101 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2102                                      u32 alignment,
2103                                      struct intel_ring_buffer *pipelined)
2104 {
2105         u32 old_read_domains, old_write_domain;
2106         int ret;
2107
2108         ret = i915_gem_object_flush_gpu_write_domain(obj);
2109         if (ret != 0)
2110                 return (ret);
2111
2112         if (pipelined != obj->ring) {
2113                 ret = i915_gem_object_wait_rendering(obj);
2114                 if (ret == -ERESTART || ret == -EINTR)
2115                         return (ret);
2116         }
2117
2118         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2119         if (ret != 0)
2120                 return (ret);
2121
2122         ret = i915_gem_object_pin(obj, alignment, true);
2123         if (ret != 0)
2124                 return (ret);
2125
2126         i915_gem_object_flush_cpu_write_domain(obj);
2127
2128         old_write_domain = obj->base.write_domain;
2129         old_read_domains = obj->base.read_domains;
2130
2131         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2132             ("obj %p in GTT write domain", obj));
2133         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2134
2135         return (0);
2136 }
2137
2138 int
2139 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2140 {
2141         int ret;
2142
2143         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2144                 return (0);
2145
2146         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2147                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2148                 if (ret != 0)
2149                         return (ret);
2150         }
2151
2152         ret = i915_gem_object_wait_rendering(obj);
2153         if (ret != 0)
2154                 return (ret);
2155
2156         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2157
2158         return (0);
2159 }
2160
2161 /**
2162  * Moves a single object to the CPU read, and possibly write domain.
2163  *
2164  * This function returns when the move is complete, including waiting on
2165  * flushes to occur.
2166  */
2167 int
2168 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2169 {
2170         uint32_t old_write_domain, old_read_domains;
2171         int ret;
2172
2173         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2174                 return 0;
2175
2176         ret = i915_gem_object_flush_gpu_write_domain(obj);
2177         if (ret != 0)
2178                 return (ret);
2179
2180         ret = i915_gem_object_wait_rendering(obj);
2181         if (ret)
2182                 return ret;
2183
2184         i915_gem_object_flush_gtt_write_domain(obj);
2185
2186         old_write_domain = obj->base.write_domain;
2187         old_read_domains = obj->base.read_domains;
2188
2189         /* Flush the CPU cache if it's still invalid. */
2190         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2191                 i915_gem_clflush_object(obj);
2192
2193                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2194         }
2195
2196         /* It should now be out of any other write domains, and we can update
2197          * the domain values for our changes.
2198          */
2199         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2200
2201         /* If we're writing through the CPU, then the GPU read domains will
2202          * need to be invalidated at next use.
2203          */
2204         if (write) {
2205                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2206                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2207         }
2208
2209         return 0;
2210 }
2211
2212 /* Throttle our rendering by waiting until the ring has completed our requests
2213  * emitted over 20 msec ago.
2214  *
2215  * Note that if we were to use the current jiffies each time around the loop,
2216  * we wouldn't escape the function with any frames outstanding if the time to
2217  * render a frame was over 20ms.
2218  *
2219  * This should get us reasonable parallelism between CPU and GPU but also
2220  * relatively low latency when blocking on a particular request to finish.
2221  */
2222 static int
2223 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
2224 {
2225         struct drm_i915_private *dev_priv = dev->dev_private;
2226         struct drm_i915_file_private *file_priv = file->driver_priv;
2227         unsigned long recent_enough = ticks - (20 * hz / 1000);
2228         struct drm_i915_gem_request *request;
2229         struct intel_ring_buffer *ring = NULL;
2230         u32 seqno = 0;
2231         int ret;
2232
2233         dev_priv = dev->dev_private;
2234         if (atomic_read(&dev_priv->mm.wedged))
2235                 return -EIO;
2236
2237         recent_enough = ticks - (20 * hz / 1000);
2238         ring = NULL;
2239         seqno = 0;
2240
2241         spin_lock(&file_priv->mm.lock);
2242         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
2243                 if (time_after_eq(request->emitted_jiffies, recent_enough))
2244                         break;
2245
2246                 ring = request->ring;
2247                 seqno = request->seqno;
2248         }
2249         spin_unlock(&file_priv->mm.lock);
2250
2251         if (seqno == 0)
2252                 return 0;
2253
2254         ret = 0;
2255         lockmgr(&ring->irq_lock, LK_EXCLUSIVE);
2256         if (!i915_seqno_passed(ring->get_seqno(ring,false), seqno)) {
2257                 if (ring->irq_get(ring)) {
2258                         while (ret == 0 &&
2259                             !(i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2260                             atomic_read(&dev_priv->mm.wedged)))
2261                                 ret = -lksleep(ring, &ring->irq_lock, PCATCH,
2262                                     "915thr", 1*hz);
2263                         ring->irq_put(ring);
2264                         if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
2265                                 ret = -EIO;
2266                 } else if (_intel_wait_for(dev,
2267                     i915_seqno_passed(ring->get_seqno(ring,false), seqno) ||
2268                     atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
2269                         ret = -EBUSY;
2270                 }
2271         }
2272         lockmgr(&ring->irq_lock, LK_RELEASE);
2273
2274         if (ret == 0)
2275                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
2276
2277         return ret;
2278 }
2279
2280 int
2281 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
2282      bool map_and_fenceable)
2283 {
2284         struct drm_device *dev;
2285         struct drm_i915_private *dev_priv;
2286         int ret;
2287
2288         dev = obj->base.dev;
2289         dev_priv = dev->dev_private;
2290
2291         KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
2292             ("Max pin count"));
2293
2294         if (obj->gtt_space != NULL) {
2295                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
2296                     (map_and_fenceable && !obj->map_and_fenceable)) {
2297                         DRM_DEBUG("bo is already pinned with incorrect alignment:"
2298                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
2299                              " obj->map_and_fenceable=%d\n",
2300                              obj->gtt_offset, alignment,
2301                              map_and_fenceable,
2302                              obj->map_and_fenceable);
2303                         ret = i915_gem_object_unbind(obj);
2304                         if (ret != 0)
2305                                 return (ret);
2306                 }
2307         }
2308
2309         if (obj->gtt_space == NULL) {
2310                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
2311                     map_and_fenceable);
2312                 if (ret)
2313                         return (ret);
2314         }
2315
2316         if (obj->pin_count++ == 0 && !obj->active)
2317                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2318         obj->pin_mappable |= map_and_fenceable;
2319
2320 #if 1
2321         KIB_NOTYET();
2322 #else
2323         WARN_ON(i915_verify_lists(dev));
2324 #endif
2325         return (0);
2326 }
2327
2328 void
2329 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
2330 {
2331         struct drm_device *dev;
2332         drm_i915_private_t *dev_priv;
2333
2334         dev = obj->base.dev;
2335         dev_priv = dev->dev_private;
2336
2337 #if 1
2338         KIB_NOTYET();
2339 #else
2340         WARN_ON(i915_verify_lists(dev));
2341 #endif
2342
2343         KASSERT(obj->pin_count != 0, ("zero pin count"));
2344         KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
2345
2346         if (--obj->pin_count == 0) {
2347                 if (!obj->active)
2348                         list_move_tail(&obj->mm_list,
2349                             &dev_priv->mm.inactive_list);
2350                 obj->pin_mappable = false;
2351         }
2352 #if 1
2353         KIB_NOTYET();
2354 #else
2355         WARN_ON(i915_verify_lists(dev));
2356 #endif
2357 }
2358
2359 int
2360 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2361     struct drm_file *file)
2362 {
2363         struct drm_i915_gem_pin *args;
2364         struct drm_i915_gem_object *obj;
2365         struct drm_gem_object *gobj;
2366         int ret;
2367
2368         args = data;
2369
2370         ret = i915_mutex_lock_interruptible(dev);
2371         if (ret != 0)
2372                 return ret;
2373
2374         gobj = drm_gem_object_lookup(dev, file, args->handle);
2375         if (gobj == NULL) {
2376                 ret = -ENOENT;
2377                 goto unlock;
2378         }
2379         obj = to_intel_bo(gobj);
2380
2381         if (obj->madv != I915_MADV_WILLNEED) {
2382                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
2383                 ret = -EINVAL;
2384                 goto out;
2385         }
2386
2387         if (obj->pin_filp != NULL && obj->pin_filp != file) {
2388                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2389                     args->handle);
2390                 ret = -EINVAL;
2391                 goto out;
2392         }
2393
2394         obj->user_pin_count++;
2395         obj->pin_filp = file;
2396         if (obj->user_pin_count == 1) {
2397                 ret = i915_gem_object_pin(obj, args->alignment, true);
2398                 if (ret != 0)
2399                         goto out;
2400         }
2401
2402         /* XXX - flush the CPU caches for pinned objects
2403          * as the X server doesn't manage domains yet
2404          */
2405         i915_gem_object_flush_cpu_write_domain(obj);
2406         args->offset = obj->gtt_offset;
2407 out:
2408         drm_gem_object_unreference(&obj->base);
2409 unlock:
2410         DRM_UNLOCK(dev);
2411         return (ret);
2412 }
2413
2414 int
2415 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2416     struct drm_file *file)
2417 {
2418         struct drm_i915_gem_pin *args;
2419         struct drm_i915_gem_object *obj;
2420         int ret;
2421
2422         args = data;
2423         ret = i915_mutex_lock_interruptible(dev);
2424         if (ret != 0)
2425                 return (ret);
2426
2427         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2428         if (&obj->base == NULL) {
2429                 ret = -ENOENT;
2430                 goto unlock;
2431         }
2432
2433         if (obj->pin_filp != file) {
2434                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2435                     args->handle);
2436                 ret = -EINVAL;
2437                 goto out;
2438         }
2439         obj->user_pin_count--;
2440         if (obj->user_pin_count == 0) {
2441                 obj->pin_filp = NULL;
2442                 i915_gem_object_unpin(obj);
2443         }
2444
2445 out:
2446         drm_gem_object_unreference(&obj->base);
2447 unlock:
2448         DRM_UNLOCK(dev);
2449         return (ret);
2450 }
2451
2452 int
2453 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2454                     struct drm_file *file)
2455 {
2456         struct drm_i915_gem_busy *args;
2457         struct drm_i915_gem_object *obj;
2458         struct drm_i915_gem_request *request;
2459         int ret;
2460
2461         args = data;
2462
2463         ret = i915_mutex_lock_interruptible(dev);
2464         if (ret != 0)
2465                 return ret;
2466
2467         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
2468         if (&obj->base == NULL) {
2469                 ret = -ENOENT;
2470                 goto unlock;
2471         }
2472
2473         args->busy = obj->active;
2474         if (args->busy) {
2475                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2476                         ret = i915_gem_flush_ring(obj->ring,
2477                             0, obj->base.write_domain);
2478                 } else if (obj->ring->outstanding_lazy_request ==
2479                     obj->last_rendering_seqno) {
2480                         request = kmalloc(sizeof(*request), DRM_I915_GEM,
2481                             M_WAITOK | M_ZERO);
2482                         ret = i915_add_request(obj->ring, NULL, request);
2483                         if (ret != 0)
2484                                 drm_free(request, DRM_I915_GEM);
2485                 }
2486
2487                 i915_gem_retire_requests_ring(obj->ring);
2488                 args->busy = obj->active;
2489         }
2490
2491         drm_gem_object_unreference(&obj->base);
2492 unlock:
2493         DRM_UNLOCK(dev);
2494         return (ret);
2495 }
2496
2497 int
2498 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2499     struct drm_file *file_priv)
2500 {
2501
2502         return (i915_gem_ring_throttle(dev, file_priv));
2503 }
2504
2505 int
2506 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
2507                        struct drm_file *file_priv)
2508 {
2509         struct drm_i915_gem_madvise *args = data;
2510         struct drm_i915_gem_object *obj;
2511         int ret;
2512
2513         switch (args->madv) {
2514         case I915_MADV_DONTNEED:
2515         case I915_MADV_WILLNEED:
2516             break;
2517         default:
2518             return -EINVAL;
2519         }
2520
2521         ret = i915_mutex_lock_interruptible(dev);
2522         if (ret)
2523                 return ret;
2524
2525         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
2526         if (&obj->base == NULL) {
2527                 ret = -ENOENT;
2528                 goto unlock;
2529         }
2530
2531         if (obj->pin_count) {
2532                 ret = -EINVAL;
2533                 goto out;
2534         }
2535
2536         if (obj->madv != __I915_MADV_PURGED)
2537                 obj->madv = args->madv;
2538
2539         /* if the object is no longer attached, discard its backing storage */
2540         if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
2541                 i915_gem_object_truncate(obj);
2542
2543         args->retained = obj->madv != __I915_MADV_PURGED;
2544
2545 out:
2546         drm_gem_object_unreference(&obj->base);
2547 unlock:
2548         DRM_UNLOCK(dev);
2549         return ret;
2550 }
2551
2552 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
2553                                                   size_t size)
2554 {
2555         struct drm_i915_private *dev_priv;
2556         struct drm_i915_gem_object *obj;
2557
2558         dev_priv = dev->dev_private;
2559
2560         obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO);
2561
2562         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
2563                 drm_free(obj, DRM_I915_GEM);
2564                 return (NULL);
2565         }
2566
2567         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2568         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2569
2570         if (HAS_LLC(dev))
2571                 obj->cache_level = I915_CACHE_LLC;
2572         else
2573                 obj->cache_level = I915_CACHE_NONE;
2574         obj->base.driver_private = NULL;
2575         obj->fence_reg = I915_FENCE_REG_NONE;
2576         INIT_LIST_HEAD(&obj->mm_list);
2577         INIT_LIST_HEAD(&obj->gtt_list);
2578         INIT_LIST_HEAD(&obj->ring_list);
2579         INIT_LIST_HEAD(&obj->exec_list);
2580         INIT_LIST_HEAD(&obj->gpu_write_list);
2581         obj->madv = I915_MADV_WILLNEED;
2582         /* Avoid an unnecessary call to unbind on the first bind. */
2583         obj->map_and_fenceable = true;
2584
2585         i915_gem_info_add_obj(dev_priv, size);
2586
2587         return (obj);
2588 }
2589
2590 int i915_gem_init_object(struct drm_gem_object *obj)
2591 {
2592
2593         kprintf("i915_gem_init_object called\n");
2594         return (0);
2595 }
2596
2597 static void
2598 i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
2599 {
2600         struct drm_device *dev;
2601         drm_i915_private_t *dev_priv;
2602         int ret;
2603
2604         dev = obj->base.dev;
2605         dev_priv = dev->dev_private;
2606
2607         ret = i915_gem_object_unbind(obj);
2608         if (ret == -ERESTART) {
2609                 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
2610                 return;
2611         }
2612
2613         drm_gem_free_mmap_offset(&obj->base);
2614         drm_gem_object_release(&obj->base);
2615         i915_gem_info_remove_obj(dev_priv, obj->base.size);
2616
2617         drm_free(obj->page_cpu_valid, DRM_I915_GEM);
2618         drm_free(obj->bit_17, DRM_I915_GEM);
2619         drm_free(obj, DRM_I915_GEM);
2620 }
2621
2622 void
2623 i915_gem_free_object(struct drm_gem_object *gem_obj)
2624 {
2625         struct drm_i915_gem_object *obj;
2626         struct drm_device *dev;
2627
2628         obj = to_intel_bo(gem_obj);
2629         dev = obj->base.dev;
2630
2631         while (obj->pin_count > 0)
2632                 i915_gem_object_unpin(obj);
2633
2634         if (obj->phys_obj != NULL)
2635                 i915_gem_detach_phys_object(dev, obj);
2636
2637         i915_gem_free_object_tail(obj);
2638 }
2639
2640 int
2641 i915_gem_do_init(struct drm_device *dev, unsigned long start,
2642     unsigned long mappable_end, unsigned long end)
2643 {
2644         drm_i915_private_t *dev_priv;
2645         unsigned long mappable;
2646         int error;
2647
2648         dev_priv = dev->dev_private;
2649         mappable = min(end, mappable_end) - start;
2650
2651         drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
2652
2653         dev_priv->mm.gtt_start = start;
2654         dev_priv->mm.gtt_mappable_end = mappable_end;
2655         dev_priv->mm.gtt_end = end;
2656         dev_priv->mm.gtt_total = end - start;
2657         dev_priv->mm.mappable_gtt_total = mappable;
2658
2659         /* Take over this portion of the GTT */
2660         intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
2661         device_printf(dev->dev,
2662             "taking over the fictitious range 0x%lx-0x%lx\n",
2663             dev->agp->base + start, dev->agp->base + start + mappable);
2664         error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
2665             dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
2666         return (error);
2667 }
2668
2669 int
2670 i915_gem_idle(struct drm_device *dev)
2671 {
2672         drm_i915_private_t *dev_priv;
2673         int ret;
2674
2675         dev_priv = dev->dev_private;
2676         if (dev_priv->mm.suspended)
2677                 return (0);
2678
2679         ret = i915_gpu_idle(dev);
2680         if (ret != 0)
2681                 return (ret);
2682
2683         /* Under UMS, be paranoid and evict. */
2684         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
2685                 ret = i915_gem_evict_inactive(dev, false);
2686                 if (ret != 0)
2687                         return ret;
2688         }
2689
2690         i915_gem_reset_fences(dev);
2691
2692         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
2693          * We need to replace this with a semaphore, or something.
2694          * And not confound mm.suspended!
2695          */
2696         dev_priv->mm.suspended = 1;
2697         del_timer_sync(&dev_priv->hangcheck_timer);
2698
2699         i915_kernel_lost_context(dev);
2700         i915_gem_cleanup_ringbuffer(dev);
2701
2702         /* Cancel the retire work handler, which should be idle now. */
2703         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2704
2705         return (ret);
2706 }
2707
2708 void i915_gem_l3_remap(struct drm_device *dev)
2709 {
2710         drm_i915_private_t *dev_priv = dev->dev_private;
2711         u32 misccpctl;
2712         int i;
2713
2714         if (!HAS_L3_GPU_CACHE(dev))
2715                 return;
2716
2717         if (!dev_priv->l3_parity.remap_info)
2718                 return;
2719
2720         misccpctl = I915_READ(GEN7_MISCCPCTL);
2721         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
2722         POSTING_READ(GEN7_MISCCPCTL);
2723
2724         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
2725                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
2726                 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
2727                         DRM_DEBUG("0x%x was already programmed to %x\n",
2728                                   GEN7_L3LOG_BASE + i, remap);
2729                 if (remap && !dev_priv->l3_parity.remap_info[i/4])
2730                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
2731                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
2732         }
2733
2734         /* Make sure all the writes land before disabling dop clock gating */
2735         POSTING_READ(GEN7_L3LOG_BASE);
2736
2737         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
2738 }
2739
2740 void
2741 i915_gem_init_swizzling(struct drm_device *dev)
2742 {
2743         drm_i915_private_t *dev_priv;
2744
2745         dev_priv = dev->dev_private;
2746
2747         if (INTEL_INFO(dev)->gen < 5 ||
2748             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
2749                 return;
2750
2751         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
2752                                  DISP_TILE_SURFACE_SWIZZLING);
2753
2754         if (IS_GEN5(dev))
2755                 return;
2756
2757         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
2758         if (IS_GEN6(dev))
2759                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
2760         else
2761                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
2762 }
2763
2764 static bool
2765 intel_enable_blt(struct drm_device *dev)
2766 {
2767         int revision;
2768
2769         if (!HAS_BLT(dev))
2770                 return false;
2771
2772         /* The blitter was dysfunctional on early prototypes */
2773         revision = pci_read_config(dev->dev, PCIR_REVID, 1);
2774         if (IS_GEN6(dev) && revision < 8) {
2775                 DRM_INFO("BLT not supported on this pre-production hardware;"
2776                          " graphics performance will be degraded.\n");
2777                 return false;
2778         }
2779
2780         return true;
2781 }
2782
2783 int
2784 i915_gem_init_hw(struct drm_device *dev)
2785 {
2786         drm_i915_private_t *dev_priv = dev->dev_private;
2787         int ret;
2788
2789         if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
2790                 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
2791
2792         i915_gem_l3_remap(dev);
2793
2794         i915_gem_init_swizzling(dev);
2795
2796         ret = intel_init_render_ring_buffer(dev);
2797         if (ret)
2798                 return ret;
2799
2800         if (HAS_BSD(dev)) {
2801                 ret = intel_init_bsd_ring_buffer(dev);
2802                 if (ret)
2803                         goto cleanup_render_ring;
2804         }
2805
2806         if (intel_enable_blt(dev)) {
2807                 ret = intel_init_blt_ring_buffer(dev);
2808                 if (ret)
2809                         goto cleanup_bsd_ring;
2810         }
2811
2812         dev_priv->next_seqno = 1;
2813
2814         /*
2815          * XXX: There was some w/a described somewhere suggesting loading
2816          * contexts before PPGTT.
2817          */
2818 #if 0   /* XXX: HW context support */
2819         i915_gem_context_init(dev);
2820 #endif
2821         i915_gem_init_ppgtt(dev);
2822
2823         return 0;
2824
2825 cleanup_bsd_ring:
2826         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
2827 cleanup_render_ring:
2828         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
2829         return ret;
2830 }
2831
2832 void
2833 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2834 {
2835         drm_i915_private_t *dev_priv;
2836         int i;
2837
2838         dev_priv = dev->dev_private;
2839         for (i = 0; i < I915_NUM_RINGS; i++)
2840                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
2841 }
2842
2843 int
2844 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2845                        struct drm_file *file_priv)
2846 {
2847         drm_i915_private_t *dev_priv = dev->dev_private;
2848         int ret;
2849
2850         if (drm_core_check_feature(dev, DRIVER_MODESET))
2851                 return 0;
2852
2853         if (atomic_read(&dev_priv->mm.wedged)) {
2854                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2855                 atomic_set(&dev_priv->mm.wedged, 0);
2856         }
2857
2858         DRM_LOCK(dev);
2859         dev_priv->mm.suspended = 0;
2860
2861         ret = i915_gem_init_hw(dev);
2862         if (ret != 0) {
2863                 DRM_UNLOCK(dev);
2864                 return ret;
2865         }
2866
2867         KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
2868         DRM_UNLOCK(dev);
2869
2870         ret = drm_irq_install(dev);
2871         if (ret)
2872                 goto cleanup_ringbuffer;
2873
2874         return 0;
2875
2876 cleanup_ringbuffer:
2877         DRM_LOCK(dev);
2878         i915_gem_cleanup_ringbuffer(dev);
2879         dev_priv->mm.suspended = 1;
2880         DRM_UNLOCK(dev);
2881
2882         return ret;
2883 }
2884
2885 int
2886 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2887     struct drm_file *file_priv)
2888 {
2889
2890         if (drm_core_check_feature(dev, DRIVER_MODESET))
2891                 return 0;
2892
2893         drm_irq_uninstall(dev);
2894         return (i915_gem_idle(dev));
2895 }
2896
2897 void
2898 i915_gem_lastclose(struct drm_device *dev)
2899 {
2900         int ret;
2901
2902         if (drm_core_check_feature(dev, DRIVER_MODESET))
2903                 return;
2904
2905         ret = i915_gem_idle(dev);
2906         if (ret != 0)
2907                 DRM_ERROR("failed to idle hardware: %d\n", ret);
2908 }
2909
2910 static void
2911 init_ring_lists(struct intel_ring_buffer *ring)
2912 {
2913
2914         INIT_LIST_HEAD(&ring->active_list);
2915         INIT_LIST_HEAD(&ring->request_list);
2916         INIT_LIST_HEAD(&ring->gpu_write_list);
2917 }
2918
2919 void
2920 i915_gem_load(struct drm_device *dev)
2921 {
2922         int i;
2923         drm_i915_private_t *dev_priv = dev->dev_private;
2924
2925         INIT_LIST_HEAD(&dev_priv->mm.active_list);
2926         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2927         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2928         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
2929         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2930         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
2931         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
2932         for (i = 0; i < I915_NUM_RINGS; i++)
2933                 init_ring_lists(&dev_priv->ring[i]);
2934         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
2935                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
2936         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2937                           i915_gem_retire_work_handler);
2938         init_completion(&dev_priv->error_completion);
2939
2940         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
2941         if (IS_GEN3(dev)) {
2942                 I915_WRITE(MI_ARB_STATE,
2943                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
2944         }
2945
2946         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
2947
2948         /* Old X drivers will take 0-2 for front, back, depth buffers */
2949         if (!drm_core_check_feature(dev, DRIVER_MODESET))
2950                 dev_priv->fence_reg_start = 3;
2951
2952         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
2953                 dev_priv->num_fence_regs = 16;
2954         else
2955                 dev_priv->num_fence_regs = 8;
2956
2957         /* Initialize fence registers to zero */
2958         i915_gem_reset_fences(dev);
2959
2960         i915_gem_detect_bit_6_swizzle(dev);
2961
2962         dev_priv->mm.interruptible = true;
2963
2964         dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
2965             i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
2966 }
2967
2968 static int
2969 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align)
2970 {
2971         drm_i915_private_t *dev_priv;
2972         struct drm_i915_gem_phys_object *phys_obj;
2973         int ret;
2974
2975         dev_priv = dev->dev_private;
2976         if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0)
2977                 return (0);
2978
2979         phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM,
2980             M_WAITOK | M_ZERO);
2981
2982         phys_obj->id = id;
2983
2984         phys_obj->handle = drm_pci_alloc(dev, size, align, ~0);
2985         if (phys_obj->handle == NULL) {
2986                 ret = -ENOMEM;
2987                 goto free_obj;
2988         }
2989         pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr,
2990             size / PAGE_SIZE, PAT_WRITE_COMBINING);
2991
2992         dev_priv->mm.phys_objs[id - 1] = phys_obj;
2993
2994         return (0);
2995
2996 free_obj:
2997         drm_free(phys_obj, DRM_I915_GEM);
2998         return (ret);
2999 }
3000
3001 static void
3002 i915_gem_free_phys_object(struct drm_device *dev, int id)
3003 {
3004         drm_i915_private_t *dev_priv;
3005         struct drm_i915_gem_phys_object *phys_obj;
3006
3007         dev_priv = dev->dev_private;
3008         if (dev_priv->mm.phys_objs[id - 1] == NULL)
3009                 return;
3010
3011         phys_obj = dev_priv->mm.phys_objs[id - 1];
3012         if (phys_obj->cur_obj != NULL)
3013                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3014
3015         drm_pci_free(dev, phys_obj->handle);
3016         drm_free(phys_obj, DRM_I915_GEM);
3017         dev_priv->mm.phys_objs[id - 1] = NULL;
3018 }
3019
3020 void
3021 i915_gem_free_all_phys_object(struct drm_device *dev)
3022 {
3023         int i;
3024
3025         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3026                 i915_gem_free_phys_object(dev, i);
3027 }
3028
3029 void
3030 i915_gem_detach_phys_object(struct drm_device *dev,
3031     struct drm_i915_gem_object *obj)
3032 {
3033         vm_page_t m;
3034         struct sf_buf *sf;
3035         char *vaddr, *dst;
3036         int i, page_count;
3037
3038         if (obj->phys_obj == NULL)
3039                 return;
3040         vaddr = obj->phys_obj->handle->vaddr;
3041
3042         page_count = obj->base.size / PAGE_SIZE;
3043         VM_OBJECT_LOCK(obj->base.vm_obj);
3044         for (i = 0; i < page_count; i++) {
3045                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3046                 if (m == NULL)
3047                         continue; /* XXX */
3048
3049                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3050                 sf = sf_buf_alloc(m);
3051                 if (sf != NULL) {
3052                         dst = (char *)sf_buf_kva(sf);
3053                         memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);
3054                         sf_buf_free(sf);
3055                 }
3056                 drm_clflush_pages(&m, 1);
3057
3058                 VM_OBJECT_LOCK(obj->base.vm_obj);
3059                 vm_page_reference(m);
3060                 vm_page_dirty(m);
3061                 vm_page_busy_wait(m, FALSE, "i915gem");
3062                 vm_page_unwire(m, 0);
3063                 vm_page_wakeup(m);
3064                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3065         }
3066         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3067         intel_gtt_chipset_flush();
3068
3069         obj->phys_obj->cur_obj = NULL;
3070         obj->phys_obj = NULL;
3071 }
3072
3073 int
3074 i915_gem_attach_phys_object(struct drm_device *dev,
3075                             struct drm_i915_gem_object *obj,
3076                             int id,
3077                             int align)
3078 {
3079         drm_i915_private_t *dev_priv;
3080         vm_page_t m;
3081         struct sf_buf *sf;
3082         char *dst, *src;
3083         int i, page_count, ret;
3084
3085         if (id > I915_MAX_PHYS_OBJECT)
3086                 return (-EINVAL);
3087
3088         if (obj->phys_obj != NULL) {
3089                 if (obj->phys_obj->id == id)
3090                         return (0);
3091                 i915_gem_detach_phys_object(dev, obj);
3092         }
3093
3094         dev_priv = dev->dev_private;
3095         if (dev_priv->mm.phys_objs[id - 1] == NULL) {
3096                 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align);
3097                 if (ret != 0) {
3098                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3099                                   id, obj->base.size);
3100                         return (ret);
3101                 }
3102         }
3103
3104         /* bind to the object */
3105         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3106         obj->phys_obj->cur_obj = obj;
3107
3108         page_count = obj->base.size / PAGE_SIZE;
3109
3110         VM_OBJECT_LOCK(obj->base.vm_obj);
3111         ret = 0;
3112         for (i = 0; i < page_count; i++) {
3113                 m = i915_gem_wire_page(obj->base.vm_obj, i);
3114                 if (m == NULL) {
3115                         ret = -EIO;
3116                         break;
3117                 }
3118                 VM_OBJECT_UNLOCK(obj->base.vm_obj);
3119                 sf = sf_buf_alloc(m);
3120                 src = (char *)sf_buf_kva(sf);
3121                 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);
3122                 memcpy(dst, src, PAGE_SIZE);
3123                 sf_buf_free(sf);
3124
3125                 VM_OBJECT_LOCK(obj->base.vm_obj);
3126
3127                 vm_page_reference(m);
3128                 vm_page_busy_wait(m, FALSE, "i915gem");
3129                 vm_page_unwire(m, 0);
3130                 vm_page_wakeup(m);
3131                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3132         }
3133         VM_OBJECT_UNLOCK(obj->base.vm_obj);
3134
3135         return (0);
3136 }
3137
3138 static int
3139 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3140     uint64_t data_ptr, uint64_t offset, uint64_t size,
3141     struct drm_file *file_priv)
3142 {
3143         char *user_data, *vaddr;
3144         int ret;
3145
3146         vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3147         user_data = (char *)(uintptr_t)data_ptr;
3148
3149         if (copyin_nofault(user_data, vaddr, size) != 0) {
3150                 /* The physical object once assigned is fixed for the lifetime
3151                  * of the obj, so we can safely drop the lock and continue
3152                  * to access vaddr.
3153                  */
3154                 DRM_UNLOCK(dev);
3155                 ret = -copyin(user_data, vaddr, size);
3156                 DRM_LOCK(dev);
3157                 if (ret != 0)
3158                         return (ret);
3159         }
3160
3161         intel_gtt_chipset_flush();
3162         return (0);
3163 }
3164
3165 void
3166 i915_gem_release(struct drm_device *dev, struct drm_file *file)
3167 {
3168         struct drm_i915_file_private *file_priv;
3169         struct drm_i915_gem_request *request;
3170
3171         file_priv = file->driver_priv;
3172
3173         /* Clean up our request list when the client is going away, so that
3174          * later retire_requests won't dereference our soon-to-be-gone
3175          * file_priv.
3176          */
3177         spin_lock(&file_priv->mm.lock);
3178         while (!list_empty(&file_priv->mm.request_list)) {
3179                 request = list_first_entry(&file_priv->mm.request_list,
3180                                            struct drm_i915_gem_request,
3181                                            client_list);
3182                 list_del(&request->client_list);
3183                 request->file_priv = NULL;
3184         }
3185         spin_unlock(&file_priv->mm.lock);
3186 }
3187
3188 static int
3189 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
3190     uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
3191     struct drm_file *file)
3192 {
3193         vm_object_t vm_obj;
3194         vm_page_t m;
3195         struct sf_buf *sf;
3196         vm_offset_t mkva;
3197         vm_pindex_t obj_pi;
3198         int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
3199
3200         if (obj->gtt_offset != 0 && rw == UIO_READ)
3201                 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
3202         else
3203                 do_bit17_swizzling = 0;
3204
3205         obj->dirty = 1;
3206         vm_obj = obj->base.vm_obj;
3207         ret = 0;
3208
3209         VM_OBJECT_LOCK(vm_obj);
3210         vm_object_pip_add(vm_obj, 1);
3211         while (size > 0) {
3212                 obj_pi = OFF_TO_IDX(offset);
3213                 obj_po = offset & PAGE_MASK;
3214
3215                 m = i915_gem_wire_page(vm_obj, obj_pi);
3216                 VM_OBJECT_UNLOCK(vm_obj);
3217
3218                 sf = sf_buf_alloc(m);
3219                 mkva = sf_buf_kva(sf);
3220                 length = min(size, PAGE_SIZE - obj_po);
3221                 while (length > 0) {
3222                         if (do_bit17_swizzling &&
3223                             (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
3224                                 cnt = roundup2(obj_po + 1, 64);
3225                                 cnt = min(cnt - obj_po, length);
3226                                 swizzled_po = obj_po ^ 64;
3227                         } else {
3228                                 cnt = length;
3229                                 swizzled_po = obj_po;
3230                         }
3231                         if (rw == UIO_READ)
3232                                 ret = -copyout_nofault(
3233                                     (char *)mkva + swizzled_po,
3234                                     (void *)(uintptr_t)data_ptr, cnt);
3235                         else
3236                                 ret = -copyin_nofault(
3237                                     (void *)(uintptr_t)data_ptr,
3238                                     (char *)mkva + swizzled_po, cnt);
3239                         if (ret != 0)
3240                                 break;
3241                         data_ptr += cnt;
3242                         size -= cnt;
3243                         length -= cnt;
3244                         offset += cnt;
3245                         obj_po += cnt;
3246                 }
3247                 sf_buf_free(sf);
3248                 VM_OBJECT_LOCK(vm_obj);
3249                 if (rw == UIO_WRITE)
3250                         vm_page_dirty(m);
3251                 vm_page_reference(m);
3252                 vm_page_busy_wait(m, FALSE, "i915gem");
3253                 vm_page_unwire(m, 1);
3254                 vm_page_wakeup(m);
3255                 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3256
3257                 if (ret != 0)
3258                         break;
3259         }
3260         vm_object_pip_wakeup(vm_obj);
3261         VM_OBJECT_UNLOCK(vm_obj);
3262
3263         return (ret);
3264 }
3265
3266 static int
3267 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
3268     uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
3269 {
3270         vm_offset_t mkva;
3271         int ret;
3272
3273         /*
3274          * Pass the unaligned physical address and size to pmap_mapdev_attr()
3275          * so it can properly calculate whether an extra page needs to be
3276          * mapped or not to cover the requested range.  The function will
3277          * add the page offset into the returned mkva for us.
3278          */
3279         mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
3280             offset, size, PAT_WRITE_COMBINING);
3281         ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size);
3282         pmap_unmapdev(mkva, size);
3283         return (ret);
3284 }
3285
3286 static int
3287 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
3288     uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
3289 {
3290         struct drm_i915_gem_object *obj;
3291         vm_page_t *ma;
3292         vm_offset_t start, end;
3293         int npages, ret;
3294
3295         if (size == 0)
3296                 return (0);
3297         start = trunc_page(data_ptr);
3298         end = round_page(data_ptr + size);
3299         npages = howmany(end - start, PAGE_SIZE);
3300         ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
3301             M_ZERO);
3302         npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
3303             (vm_offset_t)data_ptr, size,
3304             (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
3305         if (npages == -1) {
3306                 ret = -EFAULT;
3307                 goto free_ma;
3308         }
3309
3310         ret = i915_mutex_lock_interruptible(dev);
3311         if (ret != 0)
3312                 goto unlocked;
3313
3314         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
3315         if (&obj->base == NULL) {
3316                 ret = -ENOENT;
3317                 goto unlock;
3318         }
3319         if (offset > obj->base.size || size > obj->base.size - offset) {
3320                 ret = -EINVAL;
3321                 goto out;
3322         }
3323
3324         if (rw == UIO_READ) {
3325                 ret = i915_gem_object_set_cpu_read_domain_range(obj,
3326                     offset, size);
3327                 if (ret != 0)
3328                         goto out;
3329                 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3330                     UIO_READ, file);
3331         } else {
3332                 if (obj->phys_obj) {
3333                         ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
3334                             size, file);
3335                 } else if (obj->gtt_space &&
3336                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
3337                         ret = i915_gem_object_pin(obj, 0, true);
3338                         if (ret != 0)
3339                                 goto out;
3340                         ret = i915_gem_object_set_to_gtt_domain(obj, true);
3341                         if (ret != 0)
3342                                 goto out_unpin;
3343                         ret = i915_gem_object_put_fence(obj);
3344                         if (ret != 0)
3345                                 goto out_unpin;
3346                         ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
3347                             offset, file);
3348 out_unpin:
3349                         i915_gem_object_unpin(obj);
3350                 } else {
3351                         ret = i915_gem_object_set_to_cpu_domain(obj, true);
3352                         if (ret != 0)
3353                                 goto out;
3354                         ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
3355                             UIO_WRITE, file);
3356                 }
3357         }
3358 out:
3359         drm_gem_object_unreference(&obj->base);
3360 unlock:
3361         DRM_UNLOCK(dev);
3362 unlocked:
3363         vm_page_unhold_pages(ma, npages);
3364 free_ma:
3365         drm_free(ma, DRM_I915_GEM);
3366         return (ret);
3367 }
3368
3369 static int
3370 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
3371     vm_ooffset_t foff, struct ucred *cred, u_short *color)
3372 {
3373
3374         *color = 0; /* XXXKIB */
3375         return (0);
3376 }
3377
3378 int i915_intr_pf;
3379
3380 static int
3381 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot,
3382     vm_page_t *mres)
3383 {
3384         struct drm_gem_object *gem_obj;
3385         struct drm_i915_gem_object *obj;
3386         struct drm_device *dev;
3387         drm_i915_private_t *dev_priv;
3388         vm_page_t m, oldm;
3389         int cause, ret;
3390         bool write;
3391
3392         gem_obj = vm_obj->handle;
3393         obj = to_intel_bo(gem_obj);
3394         dev = obj->base.dev;
3395         dev_priv = dev->dev_private;
3396 #if 0
3397         write = (prot & VM_PROT_WRITE) != 0;
3398 #else
3399         write = true;
3400 #endif
3401         vm_object_pip_add(vm_obj, 1);
3402
3403         /*
3404          * Remove the placeholder page inserted by vm_fault() from the
3405          * object before dropping the object lock. If
3406          * i915_gem_release_mmap() is active in parallel on this gem
3407          * object, then it owns the drm device sx and might find the
3408          * placeholder already. Then, since the page is busy,
3409          * i915_gem_release_mmap() sleeps waiting for the busy state
3410          * of the page cleared. We will be not able to acquire drm
3411          * device lock until i915_gem_release_mmap() is able to make a
3412          * progress.
3413          */
3414         if (*mres != NULL) {
3415                 oldm = *mres;
3416                 vm_page_remove(oldm);
3417                 *mres = NULL;
3418         } else
3419                 oldm = NULL;
3420 retry:
3421         VM_OBJECT_UNLOCK(vm_obj);
3422 unlocked_vmobj:
3423         cause = ret = 0;
3424         m = NULL;
3425
3426         if (i915_intr_pf) {
3427                 ret = i915_mutex_lock_interruptible(dev);
3428                 if (ret != 0) {
3429                         cause = 10;
3430                         goto out;
3431                 }
3432         } else
3433                 DRM_LOCK(dev);
3434
3435         /*
3436          * Since the object lock was dropped, other thread might have
3437          * faulted on the same GTT address and instantiated the
3438          * mapping for the page.  Recheck.
3439          */
3440         VM_OBJECT_LOCK(vm_obj);
3441         m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
3442         if (m != NULL) {
3443                 if ((m->flags & PG_BUSY) != 0) {
3444                         DRM_UNLOCK(dev);
3445 #if 0 /* XXX */
3446                         vm_page_sleep(m, "915pee");
3447 #endif
3448                         goto retry;
3449                 }
3450                 goto have_page;
3451         } else
3452                 VM_OBJECT_UNLOCK(vm_obj);
3453
3454         /* Now bind it into the GTT if needed */
3455         if (!obj->map_and_fenceable) {
3456                 ret = i915_gem_object_unbind(obj);
3457                 if (ret != 0) {
3458                         cause = 20;
3459                         goto unlock;
3460                 }
3461         }
3462         if (!obj->gtt_space) {
3463                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
3464                 if (ret != 0) {
3465                         cause = 30;
3466                         goto unlock;
3467                 }
3468
3469                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
3470                 if (ret != 0) {
3471                         cause = 40;
3472                         goto unlock;
3473                 }
3474         }
3475
3476         if (obj->tiling_mode == I915_TILING_NONE)
3477                 ret = i915_gem_object_put_fence(obj);
3478         else
3479                 ret = i915_gem_object_get_fence(obj, NULL);
3480         if (ret != 0) {
3481                 cause = 50;
3482                 goto unlock;
3483         }
3484
3485         if (i915_gem_object_is_inactive(obj))
3486                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3487
3488         obj->fault_mappable = true;
3489         VM_OBJECT_LOCK(vm_obj);
3490         m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset +
3491             offset);
3492         if (m == NULL) {
3493                 cause = 60;
3494                 ret = -EFAULT;
3495                 goto unlock;
3496         }
3497         KASSERT((m->flags & PG_FICTITIOUS) != 0,
3498             ("not fictitious %p", m));
3499         KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m));
3500
3501         if ((m->flags & PG_BUSY) != 0) {
3502                 DRM_UNLOCK(dev);
3503 #if 0 /* XXX */
3504                 vm_page_sleep(m, "915pbs");
3505 #endif
3506                 goto retry;
3507         }
3508         m->valid = VM_PAGE_BITS_ALL;
3509         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
3510 have_page:
3511         *mres = m;
3512         vm_page_busy_try(m, false);
3513
3514         DRM_UNLOCK(dev);
3515         if (oldm != NULL) {
3516                 vm_page_free(oldm);
3517         }
3518         vm_object_pip_wakeup(vm_obj);
3519         return (VM_PAGER_OK);
3520
3521 unlock:
3522         DRM_UNLOCK(dev);
3523 out:
3524         KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return"));
3525         if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) {
3526                 goto unlocked_vmobj;
3527         }
3528         VM_OBJECT_LOCK(vm_obj);
3529         vm_object_pip_wakeup(vm_obj);
3530         return (VM_PAGER_ERROR);
3531 }
3532
3533 static void
3534 i915_gem_pager_dtor(void *handle)
3535 {
3536         struct drm_gem_object *obj;
3537         struct drm_device *dev;
3538
3539         obj = handle;
3540         dev = obj->dev;
3541
3542         DRM_LOCK(dev);
3543         drm_gem_free_mmap_offset(obj);
3544         i915_gem_release_mmap(to_intel_bo(obj));
3545         drm_gem_object_unreference(obj);
3546         DRM_UNLOCK(dev);
3547 }
3548
3549 struct cdev_pager_ops i915_gem_pager_ops = {
3550         .cdev_pg_fault  = i915_gem_pager_fault,
3551         .cdev_pg_ctor   = i915_gem_pager_ctor,
3552         .cdev_pg_dtor   = i915_gem_pager_dtor
3553 };
3554
3555 static int
3556 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3557     uint64_t offset, uint64_t size)
3558 {
3559         uint32_t old_read_domains;
3560         int i, ret;
3561
3562         if (offset == 0 && size == obj->base.size)
3563                 return (i915_gem_object_set_to_cpu_domain(obj, 0));
3564
3565         ret = i915_gem_object_flush_gpu_write_domain(obj);
3566         if (ret != 0)
3567                 return (ret);
3568         ret = i915_gem_object_wait_rendering(obj);
3569         if (ret != 0)
3570                 return (ret);
3571
3572         i915_gem_object_flush_gtt_write_domain(obj);
3573
3574         if (obj->page_cpu_valid == NULL &&
3575             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3576                 return (0);
3577
3578         if (obj->page_cpu_valid == NULL) {
3579                 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE,
3580                     DRM_I915_GEM, M_WAITOK | M_ZERO);
3581         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3582                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3583
3584         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3585              i++) {
3586                 if (obj->page_cpu_valid[i])
3587                         continue;
3588                 drm_clflush_pages(obj->pages + i, 1);
3589                 obj->page_cpu_valid[i] = 1;
3590         }
3591
3592         KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
3593             ("In gpu write domain"));
3594
3595         old_read_domains = obj->base.read_domains;
3596         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3597
3598         return (0);
3599 }
3600
3601 #define GEM_PARANOID_CHECK_GTT 0
3602 #if GEM_PARANOID_CHECK_GTT
3603 static void
3604 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
3605     int page_count)
3606 {
3607         struct drm_i915_private *dev_priv;
3608         vm_paddr_t pa;
3609         unsigned long start, end;
3610         u_int i;
3611         int j;
3612
3613         dev_priv = dev->dev_private;
3614         start = OFF_TO_IDX(dev_priv->mm.gtt_start);
3615         end = OFF_TO_IDX(dev_priv->mm.gtt_end);
3616         for (i = start; i < end; i++) {
3617                 pa = intel_gtt_read_pte_paddr(i);
3618                 for (j = 0; j < page_count; j++) {
3619                         if (pa == VM_PAGE_TO_PHYS(ma[j])) {
3620                                 panic("Page %p in GTT pte index %d pte %x",
3621                                     ma[i], i, intel_gtt_read_pte(i));
3622                         }
3623                 }
3624         }
3625 }
3626 #endif
3627
3628 static void
3629 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
3630     uint32_t flush_domains)
3631 {
3632         struct drm_i915_gem_object *obj, *next;
3633         uint32_t old_write_domain;
3634
3635         list_for_each_entry_safe(obj, next, &ring->gpu_write_list,
3636             gpu_write_list) {
3637                 if (obj->base.write_domain & flush_domains) {
3638                         old_write_domain = obj->base.write_domain;
3639                         obj->base.write_domain = 0;
3640                         list_del_init(&obj->gpu_write_list);
3641                         i915_gem_object_move_to_active(obj, ring,
3642                             i915_gem_next_request_seqno(ring));
3643                 }
3644         }
3645 }
3646
3647 #define VM_OBJECT_LOCK_ASSERT_OWNED(object)
3648
3649 static vm_page_t
3650 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3651 {
3652         vm_page_t m;
3653         int rv;
3654
3655         VM_OBJECT_LOCK_ASSERT_OWNED(object);
3656         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
3657         if (m->valid != VM_PAGE_BITS_ALL) {
3658                 if (vm_pager_has_page(object, pindex)) {
3659                         rv = vm_pager_get_page(object, &m, 1);
3660                         m = vm_page_lookup(object, pindex);
3661                         if (m == NULL)
3662                                 return (NULL);
3663                         if (rv != VM_PAGER_OK) {
3664                                 vm_page_free(m);
3665                                 return (NULL);
3666                         }
3667                 } else {
3668                         pmap_zero_page(VM_PAGE_TO_PHYS(m));
3669                         m->valid = VM_PAGE_BITS_ALL;
3670                         m->dirty = 0;
3671                 }
3672         }
3673         vm_page_wire(m);
3674         vm_page_wakeup(m);
3675         atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3676         return (m);
3677 }
3678
3679 int
3680 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
3681     uint32_t flush_domains)
3682 {
3683         int ret;
3684
3685         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
3686                 return 0;
3687
3688         ret = ring->flush(ring, invalidate_domains, flush_domains);
3689         if (ret)
3690                 return ret;
3691
3692         if (flush_domains & I915_GEM_GPU_DOMAINS)
3693                 i915_gem_process_flushing_list(ring, flush_domains);
3694         return 0;
3695 }
3696
3697 u32
3698 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
3699 {
3700         if (ring->outstanding_lazy_request == 0)
3701                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
3702
3703         return ring->outstanding_lazy_request;
3704 }
3705
3706 static void
3707 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3708 {
3709         drm_i915_private_t *dev_priv = dev->dev_private;
3710         uint32_t fence_reg = reg - dev_priv->fence_regs;
3711
3712         switch (INTEL_INFO(dev)->gen) {
3713         case 7:
3714         case 6:
3715                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3716                 break;
3717         case 5:
3718         case 4:
3719                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3720                 break;
3721         case 3:
3722                 if (fence_reg >= 8)
3723                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3724                 else
3725         case 2:
3726                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3727
3728                 I915_WRITE(fence_reg, 0);
3729                 break;
3730         }
3731
3732         list_del_init(&reg->lru_list);
3733         reg->obj = NULL;
3734         reg->setup_seqno = 0;
3735         reg->pin_count = 0;
3736 }
3737
3738 static int
3739 i915_gpu_is_active(struct drm_device *dev)
3740 {
3741         drm_i915_private_t *dev_priv;
3742
3743         dev_priv = dev->dev_private;
3744         return (!list_empty(&dev_priv->mm.flushing_list) ||
3745             !list_empty(&dev_priv->mm.active_list));
3746 }
3747
3748 static void
3749 i915_gem_lowmem(void *arg)
3750 {
3751         struct drm_device *dev;
3752         struct drm_i915_private *dev_priv;
3753         struct drm_i915_gem_object *obj, *next;
3754         int cnt, cnt_fail, cnt_total;
3755
3756         dev = arg;
3757         dev_priv = dev->dev_private;
3758
3759         if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT))
3760                 return;
3761
3762 rescan:
3763         /* first scan for clean buffers */
3764         i915_gem_retire_requests(dev);
3765
3766         cnt_total = cnt_fail = cnt = 0;
3767
3768         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3769             mm_list) {
3770                 if (i915_gem_object_is_purgeable(obj)) {
3771                         if (i915_gem_object_unbind(obj) != 0)
3772                                 cnt_total++;
3773                 } else
3774                         cnt_total++;
3775         }
3776
3777         /* second pass, evict/count anything still on the inactive list */
3778         list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list,
3779             mm_list) {
3780                 if (i915_gem_object_unbind(obj) == 0)
3781                         cnt++;
3782                 else
3783                         cnt_fail++;
3784         }
3785
3786         if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3787                 /*
3788                  * We are desperate for pages, so as a last resort, wait
3789                  * for the GPU to finish and discard whatever we can.
3790                  * This has a dramatic impact to reduce the number of
3791                  * OOM-killer events whilst running the GPU aggressively.
3792                  */
3793                 if (i915_gpu_idle(dev) == 0)
3794                         goto rescan;
3795         }
3796         DRM_UNLOCK(dev);
3797 }
3798
3799 void
3800 i915_gem_unload(struct drm_device *dev)
3801 {
3802         struct drm_i915_private *dev_priv;
3803
3804         dev_priv = dev->dev_private;
3805         EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3806 }