sys/dev/drm/i915/i915_gem_execbuffer.c

   1 /*
   2  * Copyright © 2008,2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  * $FreeBSD: src/sys/dev/drm2/i915/i915_gem_execbuffer.c,v 1.3 2012/05/28 13:58:08 kib Exp $
  28  */
  29
  30 #include <sys/limits.h>
  31 #include <sys/sfbuf.h>
  32
  33 #include <drm/drmP.h>
  34 #include <drm/i915_drm.h>
  35 #include "i915_drv.h"
  36 #include "intel_drv.h"
  37
  38 struct change_domains {
  39         uint32_t invalidate_domains;
  40         uint32_t flush_domains;
  41         uint32_t flush_rings;
  42         uint32_t flips;
  43 };
  44
  45 /*
  46  * Set the next domain for the specified object. This
  47  * may not actually perform the necessary flushing/invaliding though,
  48  * as that may want to be batched with other set_domain operations
  49  *
  50  * This is (we hope) the only really tricky part of gem. The goal
  51  * is fairly simple -- track which caches hold bits of the object
  52  * and make sure they remain coherent. A few concrete examples may
  53  * help to explain how it works. For shorthand, we use the notation
  54  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
  55  * a pair of read and write domain masks.
  56  *
  57  * Case 1: the batch buffer
  58  *
  59  *      1. Allocated
  60  *      2. Written by CPU
  61  *      3. Mapped to GTT
  62  *      4. Read by GPU
  63  *      5. Unmapped from GTT
  64  *      6. Freed
  65  *
  66  *      Let's take these a step at a time
  67  *
  68  *      1. Allocated
  69  *              Pages allocated from the kernel may still have
  70  *              cache contents, so we set them to (CPU, CPU) always.
  71  *      2. Written by CPU (using pwrite)
  72  *              The pwrite function calls set_domain (CPU, CPU) and
  73  *              this function does nothing (as nothing changes)
  74  *      3. Mapped by GTT
  75  *              This function asserts that the object is not
  76  *              currently in any GPU-based read or write domains
  77  *      4. Read by GPU
  78  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
  79  *              As write_domain is zero, this function adds in the
  80  *              current read domains (CPU+COMMAND, 0).
  81  *              flush_domains is set to CPU.
  82  *              invalidate_domains is set to COMMAND
  83  *              clflush is run to get data out of the CPU caches
  84  *              then i915_dev_set_domain calls i915_gem_flush to
  85  *              emit an MI_FLUSH and drm_agp_chipset_flush
  86  *      5. Unmapped from GTT
  87  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
  88  *              flush_domains and invalidate_domains end up both zero
  89  *              so no flushing/invalidating happens
  90  *      6. Freed
  91  *              yay, done
  92  *
  93  * Case 2: The shared render buffer
  94  *
  95  *      1. Allocated
  96  *      2. Mapped to GTT
  97  *      3. Read/written by GPU
  98  *      4. set_domain to (CPU,CPU)
  99  *      5. Read/written by CPU
 100  *      6. Read/written by GPU
 101  *
 102  *      1. Allocated
 103  *              Same as last example, (CPU, CPU)
 104  *      2. Mapped to GTT
 105  *              Nothing changes (assertions find that it is not in the GPU)
 106  *      3. Read/written by GPU
 107  *              execbuffer calls set_domain (RENDER, RENDER)
 108  *              flush_domains gets CPU
 109  *              invalidate_domains gets GPU
 110  *              clflush (obj)
 111  *              MI_FLUSH and drm_agp_chipset_flush
 112  *      4. set_domain (CPU, CPU)
 113  *              flush_domains gets GPU
 114  *              invalidate_domains gets CPU
 115  *              wait_rendering (obj) to make sure all drawing is complete.
 116  *              This will include an MI_FLUSH to get the data from GPU
 117  *              to memory
 118  *              clflush (obj) to invalidate the CPU cache
 119  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
 120  *      5. Read/written by CPU
 121  *              cache lines are loaded and dirtied
 122  *      6. Read written by GPU
 123  *              Same as last GPU access
 124  *
 125  * Case 3: The constant buffer
 126  *
 127  *      1. Allocated
 128  *      2. Written by CPU
 129  *      3. Read by GPU
 130  *      4. Updated (written) by CPU again
 131  *      5. Read by GPU
 132  *
 133  *      1. Allocated
 134  *              (CPU, CPU)
 135  *      2. Written by CPU
 136  *              (CPU, CPU)
 137  *      3. Read by GPU
 138  *              (CPU+RENDER, 0)
 139  *              flush_domains = CPU
 140  *              invalidate_domains = RENDER
 141  *              clflush (obj)
 142  *              MI_FLUSH
 143  *              drm_agp_chipset_flush
 144  *      4. Updated (written) by CPU again
 145  *              (CPU, CPU)
 146  *              flush_domains = 0 (no previous write domain)
 147  *              invalidate_domains = 0 (no new read domains)
 148  *      5. Read by GPU
 149  *              (CPU+RENDER, 0)
 150  *              flush_domains = CPU
 151  *              invalidate_domains = RENDER
 152  *              clflush (obj)
 153  *              MI_FLUSH
 154  *              drm_agp_chipset_flush
 155  */
 156 static void
 157 i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
 158                                   struct intel_ring_buffer *ring,
 159                                   struct change_domains *cd)
 160 {
 161         uint32_t invalidate_domains = 0, flush_domains = 0;
 162
 163         /*
 164          * If the object isn't moving to a new write domain,
 165          * let the object stay in multiple read domains
 166          */
 167         if (obj->base.pending_write_domain == 0)
 168                 obj->base.pending_read_domains |= obj->base.read_domains;
 169
 170         /*
 171          * Flush the current write domain if
 172          * the new read domains don't match. Invalidate
 173          * any read domains which differ from the old
 174          * write domain
 175          */
 176         if (obj->base.write_domain &&
 177             (((obj->base.write_domain != obj->base.pending_read_domains ||
 178                obj->ring != ring)) ||
 179              (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
 180                 flush_domains |= obj->base.write_domain;
 181                 invalidate_domains |=
 182                         obj->base.pending_read_domains & ~obj->base.write_domain;
 183         }
 184         /*
 185          * Invalidate any read caches which may have
 186          * stale data. That is, any new read domains.
 187          */
 188         invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
 189         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
 190                 i915_gem_clflush_object(obj);
 191
 192         if (obj->base.pending_write_domain)
 193                 cd->flips |= atomic_read(&obj->pending_flip);
 194
 195         /* The actual obj->write_domain will be updated with
 196          * pending_write_domain after we emit the accumulated flush for all
 197          * of our domain changes in execbuffers (which clears objects'
 198          * write_domains).  So if we have a current write domain that we
 199          * aren't changing, set pending_write_domain to that.
 200          */
 201         if (flush_domains == 0 && obj->base.pending_write_domain == 0)
 202                 obj->base.pending_write_domain = obj->base.write_domain;
 203
 204         cd->invalidate_domains |= invalidate_domains;
 205         cd->flush_domains |= flush_domains;
 206         if (flush_domains & I915_GEM_GPU_DOMAINS)
 207                 cd->flush_rings |= intel_ring_flag(obj->ring);
 208         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
 209                 cd->flush_rings |= intel_ring_flag(ring);
 210 }
 211
 212 struct eb_objects {
 213         int and;
 214         struct hlist_head buckets[0];
 215 };
 216
 217 static struct eb_objects *
 218 eb_create(int size)
 219 {
 220         struct eb_objects *eb;
 221         int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
 222         while (count > size)
 223                 count >>= 1;
 224 #if 0
 225         eb = kzalloc(count*sizeof(struct hlist_head) +
 226                      sizeof(struct eb_objects),
 227                      GFP_KERNEL);
 228 #else
 229         eb = kmalloc(count*sizeof(struct hlist_head) +
 230                      sizeof(struct eb_objects),
 231                      DRM_I915_GEM, M_WAITOK | M_ZERO);
 232 #endif
 233         if (eb == NULL)
 234                 return eb;
 235
 236         eb->and = count - 1;
 237         return eb;
 238 }
 239
 240 static void
 241 eb_reset(struct eb_objects *eb)
 242 {
 243         memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
 244 }
 245
 246 static void
 247 eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
 248 {
 249         hlist_add_head(&obj->exec_node,
 250                        &eb->buckets[obj->exec_handle & eb->and]);
 251 }
 252
 253 static struct drm_i915_gem_object *
 254 eb_get_object(struct eb_objects *eb, unsigned long handle)
 255 {
 256         struct hlist_head *head;
 257         struct hlist_node *node;
 258         struct drm_i915_gem_object *obj;
 259
 260         head = &eb->buckets[handle & eb->and];
 261         hlist_for_each(node, head) {
 262                 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
 263                 if (obj->exec_handle == handle)
 264                         return obj;
 265         }
 266
 267         return NULL;
 268 }
 269
 270 static void
 271 eb_destroy(struct eb_objects *eb)
 272 {
 273         drm_free(eb, DRM_I915_GEM);
 274 }
 275
 276 static int
 277 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 278                                    struct eb_objects *eb,
 279                                    struct drm_i915_gem_relocation_entry *reloc)
 280 {
 281         struct drm_device *dev = obj->base.dev;
 282         struct drm_gem_object *target_obj;
 283         uint32_t target_offset;
 284         int ret = -EINVAL;
 285
 286         /* we've already hold a reference to all valid objects */
 287         target_obj = &eb_get_object(eb, reloc->target_handle)->base;
 288         if (unlikely(target_obj == NULL))
 289                 return -ENOENT;
 290
 291         target_offset = to_intel_bo(target_obj)->gtt_offset;
 292
 293 #if WATCH_RELOC
 294         DRM_INFO("%s: obj %p offset %08x target %d "
 295                  "read %08x write %08x gtt %08x "
 296                  "presumed %08x delta %08x\n",
 297                  __func__,
 298                  obj,
 299                  (int) reloc->offset,
 300                  (int) reloc->target_handle,
 301                  (int) reloc->read_domains,
 302                  (int) reloc->write_domain,
 303                  (int) target_offset,
 304                  (int) reloc->presumed_offset,
 305                  reloc->delta);
 306 #endif
 307
 308         /* The target buffer should have appeared before us in the
 309          * exec_object list, so it should have a GTT space bound by now.
 310          */
 311         if (unlikely(target_offset == 0)) {
 312                 DRM_DEBUG("No GTT space found for object %d\n",
 313                           reloc->target_handle);
 314                 return ret;
 315         }
 316
 317         /* Validate that the target is in a valid r/w GPU domain */
 318         if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
 319                 DRM_DEBUG("reloc with multiple write domains: "
 320                           "obj %p target %d offset %d "
 321                           "read %08x write %08x",
 322                           obj, reloc->target_handle,
 323                           (int) reloc->offset,
 324                           reloc->read_domains,
 325                           reloc->write_domain);
 326                 return ret;
 327         }
 328         if (unlikely((reloc->write_domain | reloc->read_domains)
 329                      & ~I915_GEM_GPU_DOMAINS)) {
 330                 DRM_DEBUG("reloc with read/write non-GPU domains: "
 331                           "obj %p target %d offset %d "
 332                           "read %08x write %08x",
 333                           obj, reloc->target_handle,
 334                           (int) reloc->offset,
 335                           reloc->read_domains,
 336                           reloc->write_domain);
 337                 return ret;
 338         }
 339         if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
 340                      reloc->write_domain != target_obj->pending_write_domain)) {
 341                 DRM_DEBUG("Write domain conflict: "
 342                           "obj %p target %d offset %d "
 343                           "new %08x old %08x\n",
 344                           obj, reloc->target_handle,
 345                           (int) reloc->offset,
 346                           reloc->write_domain,
 347                           target_obj->pending_write_domain);
 348                 return ret;
 349         }
 350
 351         target_obj->pending_read_domains |= reloc->read_domains;
 352         target_obj->pending_write_domain |= reloc->write_domain;
 353
 354         /* If the relocation already has the right value in it, no
 355          * more work needs to be done.
 356          */
 357         if (target_offset == reloc->presumed_offset)
 358                 return 0;
 359
 360         /* Check that the relocation address is valid... */
 361         if (unlikely(reloc->offset > obj->base.size - 4)) {
 362                 DRM_DEBUG("Relocation beyond object bounds: "
 363                           "obj %p target %d offset %d size %d.\n",
 364                           obj, reloc->target_handle,
 365                           (int) reloc->offset,
 366                           (int) obj->base.size);
 367                 return ret;
 368         }
 369         if (unlikely(reloc->offset & 3)) {
 370                 DRM_DEBUG("Relocation not 4-byte aligned: "
 371                           "obj %p target %d offset %d.\n",
 372                           obj, reloc->target_handle,
 373                           (int) reloc->offset);
 374                 return ret;
 375         }
 376
 377         reloc->delta += target_offset;
 378         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
 379                 uint32_t page_offset = reloc->offset & PAGE_MASK;
 380                 char *vaddr;
 381                 struct sf_buf *sf;
 382
 383                 sf = sf_buf_alloc(obj->pages[OFF_TO_IDX(reloc->offset)]);
 384                 if (sf == NULL)
 385                         return (-ENOMEM);
 386                 vaddr = (void *)sf_buf_kva(sf);
 387                 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
 388                 sf_buf_free(sf);
 389         } else {
 390                 uint32_t *reloc_entry;
 391                 char *reloc_page;
 392
 393                 /* We can't wait for rendering with pagefaults disabled */
 394                 if (obj->active && (curthread->td_flags & TDF_NOFAULT))
 395                         return (-EFAULT);
 396                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 397                 if (ret)
 398                         return ret;
 399
 400                 /*
 401                  * Map the page containing the relocation we're going
 402                  * to perform.
 403                  */
 404                 reloc->offset += obj->gtt_offset;
 405                 reloc_page = pmap_mapdev_attr(dev->agp->base + (reloc->offset &
 406                     ~PAGE_MASK), PAGE_SIZE, PAT_WRITE_COMBINING);
 407                 reloc_entry = (uint32_t *)(reloc_page + (reloc->offset &
 408                     PAGE_MASK));
 409                 *(volatile uint32_t *)reloc_entry = reloc->delta;
 410                 pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE);
 411         }
 412
 413         /* and update the user's relocation entry */
 414         reloc->presumed_offset = target_offset;
 415
 416         return 0;
 417 }
 418
 419 static int
 420 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
 421     struct eb_objects *eb)
 422 {
 423         struct drm_i915_gem_relocation_entry *user_relocs;
 424         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 425         struct drm_i915_gem_relocation_entry reloc;
 426         int i, ret;
 427
 428         user_relocs = (void *)(uintptr_t)entry->relocs_ptr;
 429         for (i = 0; i < entry->relocation_count; i++) {
 430                 ret = -copyin_nofault(user_relocs + i, &reloc, sizeof(reloc));
 431                 if (ret != 0)
 432                         return (ret);
 433
 434                 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc);
 435                 if (ret != 0)
 436                         return (ret);
 437
 438                 ret = -copyout_nofault(&reloc.presumed_offset,
 439                     &user_relocs[i].presumed_offset,
 440                     sizeof(reloc.presumed_offset));
 441                 if (ret != 0)
 442                         return (ret);
 443         }
 444
 445         return (0);
 446 }
 447
 448 static int
 449 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
 450     struct eb_objects *eb, struct drm_i915_gem_relocation_entry *relocs)
 451 {
 452         const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 453         int i, ret;
 454
 455         for (i = 0; i < entry->relocation_count; i++) {
 456                 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
 457                 if (ret)
 458                         return ret;
 459         }
 460
 461         return 0;
 462 }
 463
 464 static int
 465 i915_gem_execbuffer_relocate(struct drm_device *dev,
 466                              struct eb_objects *eb,
 467                              struct list_head *objects)
 468 {
 469         struct drm_i915_gem_object *obj;
 470         thread_t td = curthread;
 471         int ret;
 472         int pflags;
 473
 474         /* Try to move as many of the relocation targets off the active list
 475          * to avoid unnecessary fallbacks to the slow path, as we cannot wait
 476          * for the retirement with pagefaults disabled.
 477          */
 478         i915_gem_retire_requests(dev);
 479
 480         ret = 0;
 481         pflags = td->td_flags & TDF_NOFAULT;
 482         atomic_set_int(&td->td_flags, TDF_NOFAULT);
 483
 484         /* This is the fast path and we cannot handle a pagefault whilst
 485          * holding the device lock lest the user pass in the relocations
 486          * contained within a mmaped bo. For in such a case we, the page
 487          * fault handler would call i915_gem_fault() and we would try to
 488          * acquire the device lock again. Obviously this is bad.
 489          */
 490
 491         list_for_each_entry(obj, objects, exec_list) {
 492                 ret = i915_gem_execbuffer_relocate_object(obj, eb);
 493                 if (ret != 0)
 494                         break;
 495         }
 496
 497         if ((pflags & TDF_NOFAULT) == 0)
 498                 atomic_clear_int(&td->td_flags, TDF_NOFAULT);
 499
 500         return (ret);
 501 }
 502
 503 #define  __EXEC_OBJECT_HAS_FENCE (1<<31)
 504
 505 static int
 506 pin_and_fence_object(struct drm_i915_gem_object *obj,
 507                      struct intel_ring_buffer *ring)
 508 {
 509         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 510         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 511         bool need_fence, need_mappable;
 512         int ret;
 513
 514         need_fence =
 515                 has_fenced_gpu_access &&
 516                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 517                 obj->tiling_mode != I915_TILING_NONE;
 518         need_mappable =
 519                 entry->relocation_count ? true : need_fence;
 520
 521         ret = i915_gem_object_pin(obj, entry->alignment, need_mappable);
 522         if (ret)
 523                 return ret;
 524
 525         if (has_fenced_gpu_access) {
 526                 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
 527                         if (obj->tiling_mode) {
 528                                 ret = i915_gem_object_get_fence(obj, ring);
 529                                 if (ret)
 530                                         goto err_unpin;
 531
 532                                 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
 533                                 i915_gem_object_pin_fence(obj);
 534                         } else {
 535                                 ret = i915_gem_object_put_fence(obj);
 536                                 if (ret)
 537                                         goto err_unpin;
 538                         }
 539                         obj->pending_fenced_gpu_access = true;
 540                 }
 541         }
 542
 543         entry->offset = obj->gtt_offset;
 544         return 0;
 545
 546 err_unpin:
 547         i915_gem_object_unpin(obj);
 548         return ret;
 549 }
 550
 551 static int
 552 i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 553                             struct drm_file *file,
 554                             struct list_head *objects)
 555 {
 556         drm_i915_private_t *dev_priv;
 557         struct drm_i915_gem_object *obj;
 558         int ret, retry;
 559         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 560         struct list_head ordered_objects;
 561
 562         dev_priv = ring->dev->dev_private;
 563         INIT_LIST_HEAD(&ordered_objects);
 564         while (!list_empty(objects)) {
 565                 struct drm_i915_gem_exec_object2 *entry;
 566                 bool need_fence, need_mappable;
 567
 568                 obj = list_first_entry(objects,
 569                                        struct drm_i915_gem_object,
 570                                        exec_list);
 571                 entry = obj->exec_entry;
 572
 573                 need_fence =
 574                         has_fenced_gpu_access &&
 575                         entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 576                         obj->tiling_mode != I915_TILING_NONE;
 577                 need_mappable =
 578                         entry->relocation_count ? true : need_fence;
 579
 580                 if (need_mappable)
 581                         list_move(&obj->exec_list, &ordered_objects);
 582                 else
 583                         list_move_tail(&obj->exec_list, &ordered_objects);
 584
 585                 obj->base.pending_read_domains = 0;
 586                 obj->base.pending_write_domain = 0;
 587         }
 588         list_splice(&ordered_objects, objects);
 589
 590         /* Attempt to pin all of the buffers into the GTT.
 591          * This is done in 3 phases:
 592          *
 593          * 1a. Unbind all objects that do not match the GTT constraints for
 594          *     the execbuffer (fenceable, mappable, alignment etc).
 595          * 1b. Increment pin count for already bound objects and obtain
 596          *     a fence register if required.
 597          * 2.  Bind new objects.
 598          * 3.  Decrement pin count.
 599          *
 600          * This avoid unnecessary unbinding of later objects in order to makr
 601          * room for the earlier objects *unless* we need to defragment.
 602          */
 603         retry = 0;
 604         do {
 605                 ret = 0;
 606
 607                 /* Unbind any ill-fitting objects or pin. */
 608                 list_for_each_entry(obj, objects, exec_list) {
 609                         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 610                         bool need_fence, need_mappable;
 611
 612                         if (!obj->gtt_space)
 613                                 continue;
 614
 615                         need_fence =
 616                                 has_fenced_gpu_access &&
 617                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 618                                 obj->tiling_mode != I915_TILING_NONE;
 619                         need_mappable =
 620                                 entry->relocation_count ? true : need_fence;
 621
 622                         if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
 623                             (need_mappable && !obj->map_and_fenceable))
 624                                 ret = i915_gem_object_unbind(obj);
 625                         else
 626                                 ret = pin_and_fence_object(obj, ring);
 627                         if (ret)
 628                                 goto err;
 629                 }
 630
 631                 /* Bind fresh objects */
 632                 list_for_each_entry(obj, objects, exec_list) {
 633                         if (obj->gtt_space)
 634                                 continue;
 635
 636                         ret = pin_and_fence_object(obj, ring);
 637                         if (ret) {
 638                                 int ret_ignore;
 639
 640                                 /* This can potentially raise a harmless
 641                                  * -EINVAL if we failed to bind in the above
 642                                  * call. It cannot raise -EINTR since we know
 643                                  * that the bo is freshly bound and so will
 644                                  * not need to be flushed or waited upon.
 645                                  */
 646                                 ret_ignore = i915_gem_object_unbind(obj);
 647                                 (void)ret_ignore;
 648                                 if (obj->gtt_space != NULL)
 649                                         kprintf("%s: gtt_space\n", __func__);
 650                                 break;
 651                         }
 652                 }
 653
 654                 /* Decrement pin count for bound objects */
 655                 list_for_each_entry(obj, objects, exec_list) {
 656                         struct drm_i915_gem_exec_object2 *entry;
 657
 658                         if (!obj->gtt_space)
 659                                 continue;
 660
 661                         entry = obj->exec_entry;
 662                         if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
 663                                 i915_gem_object_unpin_fence(obj);
 664                                 entry->flags &= ~__EXEC_OBJECT_HAS_FENCE;
 665                         }
 666
 667                         i915_gem_object_unpin(obj);
 668
 669                         /* ... and ensure ppgtt mapping exist if needed. */
 670                         if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
 671                                 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
 672                                                        obj, obj->cache_level);
 673
 674                                 obj->has_aliasing_ppgtt_mapping = 1;
 675                         }
 676                 }
 677
 678                 if (ret != -ENOSPC || retry > 1)
 679                         return ret;
 680
 681                 /* First attempt, just clear anything that is purgeable.
 682                  * Second attempt, clear the entire GTT.
 683                  */
 684                 ret = i915_gem_evict_everything(ring->dev, retry == 0);
 685                 if (ret)
 686                         return ret;
 687
 688                 retry++;
 689         } while (1);
 690
 691 err:
 692         list_for_each_entry_continue_reverse(obj, objects, exec_list) {
 693                 struct drm_i915_gem_exec_object2 *entry;
 694
 695                 if (!obj->gtt_space)
 696                         continue;
 697
 698                 entry = obj->exec_entry;
 699                 if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
 700                         i915_gem_object_unpin_fence(obj);
 701                         entry->flags &= ~__EXEC_OBJECT_HAS_FENCE;
 702                 }
 703
 704                 i915_gem_object_unpin(obj);
 705         }
 706
 707         return ret;
 708 }
 709
 710 static int
 711 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 712     struct drm_file *file, struct intel_ring_buffer *ring,
 713     struct list_head *objects, struct eb_objects *eb,
 714     struct drm_i915_gem_exec_object2 *exec, int count)
 715 {
 716         struct drm_i915_gem_relocation_entry *reloc;
 717         struct drm_i915_gem_object *obj;
 718         int *reloc_offset;
 719         int i, total, ret;
 720
 721         /* We may process another execbuffer during the unlock... */
 722         while (!list_empty(objects)) {
 723                 obj = list_first_entry(objects,
 724                                        struct drm_i915_gem_object,
 725                                        exec_list);
 726                 list_del_init(&obj->exec_list);
 727                 drm_gem_object_unreference(&obj->base);
 728         }
 729
 730         DRM_UNLOCK(dev);
 731
 732         total = 0;
 733         for (i = 0; i < count; i++)
 734                 total += exec[i].relocation_count;
 735
 736         reloc_offset = kmalloc(count * sizeof(*reloc_offset), DRM_I915_GEM,
 737             M_WAITOK | M_ZERO);
 738         reloc = kmalloc(total * sizeof(*reloc), DRM_I915_GEM, M_WAITOK | M_ZERO);
 739
 740         total = 0;
 741         for (i = 0; i < count; i++) {
 742                 struct drm_i915_gem_relocation_entry *user_relocs;
 743
 744                 user_relocs = (void *)(uintptr_t)exec[i].relocs_ptr;
 745                 ret = -copyin(user_relocs, reloc + total,
 746                     exec[i].relocation_count * sizeof(*reloc));
 747                 if (ret != 0) {
 748                         DRM_LOCK(dev);
 749                         goto err;
 750                 }
 751
 752                 reloc_offset[i] = total;
 753                 total += exec[i].relocation_count;
 754         }
 755
 756         ret = i915_mutex_lock_interruptible(dev);
 757         if (ret) {
 758                 DRM_LOCK(dev);
 759                 goto err;
 760         }
 761
 762         /* reacquire the objects */
 763         eb_reset(eb);
 764         for (i = 0; i < count; i++) {
 765                 struct drm_i915_gem_object *obj;
 766
 767                 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
 768                                                         exec[i].handle));
 769                 if (&obj->base == NULL) {
 770                         DRM_DEBUG("Invalid object handle %d at index %d\n",
 771                                    exec[i].handle, i);
 772                         ret = -ENOENT;
 773                         goto err;
 774                 }
 775
 776                 list_add_tail(&obj->exec_list, objects);
 777                 obj->exec_handle = exec[i].handle;
 778                 obj->exec_entry = &exec[i];
 779                 eb_add_object(eb, obj);
 780         }
 781
 782         ret = i915_gem_execbuffer_reserve(ring, file, objects);
 783         if (ret)
 784                 goto err;
 785
 786         list_for_each_entry(obj, objects, exec_list) {
 787                 int offset = obj->exec_entry - exec;
 788                 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
 789                     reloc + reloc_offset[offset]);
 790                 if (ret)
 791                         goto err;
 792         }
 793
 794         /* Leave the user relocations as are, this is the painfully slow path,
 795          * and we want to avoid the complication of dropping the lock whilst
 796          * having buffers reserved in the aperture and so causing spurious
 797          * ENOSPC for random operations.
 798          */
 799
 800 err:
 801         drm_free(reloc, DRM_I915_GEM);
 802         drm_free(reloc_offset, DRM_I915_GEM);
 803         return ret;
 804 }
 805
 806 static int
 807 i915_gem_execbuffer_flush(struct drm_device *dev,
 808                           uint32_t invalidate_domains,
 809                           uint32_t flush_domains,
 810                           uint32_t flush_rings)
 811 {
 812         drm_i915_private_t *dev_priv = dev->dev_private;
 813         int i, ret;
 814
 815         if (flush_domains & I915_GEM_DOMAIN_CPU)
 816                 intel_gtt_chipset_flush();
 817
 818         if (flush_domains & I915_GEM_DOMAIN_GTT)
 819                 cpu_sfence();
 820
 821         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
 822                 for (i = 0; i < I915_NUM_RINGS; i++)
 823                         if (flush_rings & (1 << i)) {
 824                                 ret = i915_gem_flush_ring(&dev_priv->ring[i],
 825                                     invalidate_domains, flush_domains);
 826                                 if (ret)
 827                                         return ret;
 828                         }
 829         }
 830
 831         return 0;
 832 }
 833
 834 static bool
 835 intel_enable_semaphores(struct drm_device *dev)
 836 {
 837         if (INTEL_INFO(dev)->gen < 6)
 838                 return 0;
 839
 840         if (i915_semaphores >= 0)
 841                 return i915_semaphores;
 842
 843         /* Enable semaphores on SNB when IO remapping is off */
 844         if (INTEL_INFO(dev)->gen == 6)
 845                 return !intel_iommu_enabled;
 846
 847         return 1;
 848 }
 849
 850 static int
 851 i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
 852                                struct intel_ring_buffer *to)
 853 {
 854         struct intel_ring_buffer *from = obj->ring;
 855         u32 seqno;
 856         int ret, idx;
 857
 858         if (from == NULL || to == from)
 859                 return 0;
 860
 861         /* XXX gpu semaphores are implicated in various hard hangs on SNB */
 862         if (!intel_enable_semaphores(obj->base.dev))
 863                 return i915_gem_object_wait_rendering(obj);
 864
 865         idx = intel_ring_sync_index(from, to);
 866
 867         seqno = obj->last_rendering_seqno;
 868         if (seqno <= from->sync_seqno[idx])
 869                 return 0;
 870
 871         if (seqno == from->outstanding_lazy_request) {
 872                 struct drm_i915_gem_request *request;
 873
 874                 request = kmalloc(sizeof(*request), DRM_I915_GEM,
 875                     M_WAITOK | M_ZERO);
 876                 ret = i915_add_request(from, NULL, request);
 877                 if (ret) {
 878                         drm_free(request, DRM_I915_GEM);
 879                         return ret;
 880                 }
 881
 882                 seqno = request->seqno;
 883         }
 884
 885         from->sync_seqno[idx] = seqno;
 886
 887         return to->sync_to(to, from, seqno - 1);
 888 }
 889
 890 static int
 891 i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
 892 {
 893         u32 plane, flip_mask;
 894         int ret;
 895
 896         /* Check for any pending flips. As we only maintain a flip queue depth
 897          * of 1, we can simply insert a WAIT for the next display flip prior
 898          * to executing the batch and avoid stalling the CPU.
 899          */
 900
 901         for (plane = 0; flips >> plane; plane++) {
 902                 if (((flips >> plane) & 1) == 0)
 903                         continue;
 904
 905                 if (plane)
 906                         flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 907                 else
 908                         flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
 909
 910                 ret = intel_ring_begin(ring, 2);
 911                 if (ret)
 912                         return ret;
 913
 914                 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
 915                 intel_ring_emit(ring, MI_NOOP);
 916                 intel_ring_advance(ring);
 917         }
 918
 919         return 0;
 920 }
 921
 922 static int
 923 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 924                                 struct list_head *objects)
 925 {
 926         struct drm_i915_gem_object *obj;
 927         struct change_domains cd;
 928         int ret;
 929
 930         memset(&cd, 0, sizeof(cd));
 931         list_for_each_entry(obj, objects, exec_list)
 932                 i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
 933
 934         if (cd.invalidate_domains | cd.flush_domains) {
 935 #if WATCH_EXEC
 936                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
 937                           __func__,
 938                          cd.invalidate_domains,
 939                          cd.flush_domains);
 940 #endif
 941                 ret = i915_gem_execbuffer_flush(ring->dev,
 942                                                 cd.invalidate_domains,
 943                                                 cd.flush_domains,
 944                                                 cd.flush_rings);
 945                 if (ret)
 946                         return ret;
 947         }
 948
 949         if (cd.flips) {
 950                 ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips);
 951                 if (ret)
 952                         return ret;
 953         }
 954
 955         list_for_each_entry(obj, objects, exec_list) {
 956                 ret = i915_gem_execbuffer_sync_rings(obj, ring);
 957                 if (ret)
 958                         return ret;
 959         }
 960
 961         return 0;
 962 }
 963
 964 static bool
 965 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 966 {
 967         return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
 968 }
 969
 970 static int
 971 validate_exec_list(struct drm_i915_gem_exec_object2 *exec, int count,
 972     vm_page_t ***map)
 973 {
 974         vm_page_t *ma;
 975         int i, length, page_count;
 976
 977         /* XXXKIB various limits checking is missing there */
 978         *map = kmalloc(count * sizeof(*ma), DRM_I915_GEM, M_WAITOK | M_ZERO);
 979         for (i = 0; i < count; i++) {
 980                 /* First check for malicious input causing overflow */
 981                 if (exec[i].relocation_count >
 982                     INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
 983                         return -EINVAL;
 984
 985                 length = exec[i].relocation_count *
 986                     sizeof(struct drm_i915_gem_relocation_entry);
 987                 if (length == 0) {
 988                         (*map)[i] = NULL;
 989                         continue;
 990                 }
 991                 /*
 992                  * Since both start and end of the relocation region
 993                  * may be not aligned on the page boundary, be
 994                  * conservative and request a page slot for each
 995                  * partial page.  Thus +2.
 996                  */
 997                 page_count = howmany(length, PAGE_SIZE) + 2;
 998                 ma = (*map)[i] = kmalloc(page_count * sizeof(vm_page_t),
 999                     DRM_I915_GEM, M_WAITOK | M_ZERO);
1000                 if (vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
1001                     exec[i].relocs_ptr, length, VM_PROT_READ | VM_PROT_WRITE,
1002                     ma, page_count) == -1) {
1003                         drm_free(ma, DRM_I915_GEM);
1004                         (*map)[i] = NULL;
1005                         return (-EFAULT);
1006                 }
1007         }
1008
1009         return 0;
1010 }
1011
1012 static void
1013 i915_gem_execbuffer_move_to_active(struct list_head *objects,
1014                                    struct intel_ring_buffer *ring,
1015                                    u32 seqno)
1016 {
1017         struct drm_i915_gem_object *obj;
1018         uint32_t old_read, old_write;
1019
1020         list_for_each_entry(obj, objects, exec_list) {
1021                 old_read = obj->base.read_domains;
1022                 old_write = obj->base.write_domain;
1023
1024                 obj->base.read_domains = obj->base.pending_read_domains;
1025                 obj->base.write_domain = obj->base.pending_write_domain;
1026                 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
1027
1028                 i915_gem_object_move_to_active(obj, ring, seqno);
1029                 if (obj->base.write_domain) {
1030                         obj->dirty = 1;
1031                         obj->pending_gpu_write = true;
1032                         list_move_tail(&obj->gpu_write_list,
1033                                        &ring->gpu_write_list);
1034                         intel_mark_busy(ring->dev);
1035                 }
1036         }
1037 }
1038
1039 int i915_gem_sync_exec_requests;
1040
1041 static void
1042 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
1043                                     struct drm_file *file,
1044                                     struct intel_ring_buffer *ring)
1045 {
1046         struct drm_i915_gem_request *request;
1047         u32 invalidate;
1048
1049         /*
1050          * Ensure that the commands in the batch buffer are
1051          * finished before the interrupt fires.
1052          *
1053          * The sampler always gets flushed on i965 (sigh).
1054          */
1055         invalidate = I915_GEM_DOMAIN_COMMAND;
1056         if (INTEL_INFO(dev)->gen >= 4)
1057                 invalidate |= I915_GEM_DOMAIN_SAMPLER;
1058         if (ring->flush(ring, invalidate, 0)) {
1059                 i915_gem_next_request_seqno(ring);
1060                 return;
1061         }
1062
1063         /* Add a breadcrumb for the completion of the batch buffer */
1064         request = kmalloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO);
1065         if (request == NULL || i915_add_request(ring, file, request)) {
1066                 i915_gem_next_request_seqno(ring);
1067                 drm_free(request, DRM_I915_GEM);
1068         } else if (i915_gem_sync_exec_requests)
1069                 i915_wait_request(ring, request->seqno, true);
1070 }
1071
1072 static void
1073 i915_gem_fix_mi_batchbuffer_end(struct drm_i915_gem_object *batch_obj,
1074     uint32_t batch_start_offset, uint32_t batch_len)
1075 {
1076         char *mkva;
1077         uint64_t po_r, po_w;
1078         uint32_t cmd;
1079
1080         po_r = batch_obj->base.dev->agp->base + batch_obj->gtt_offset +
1081             batch_start_offset + batch_len;
1082         if (batch_len > 0)
1083                 po_r -= 4;
1084         mkva = pmap_mapdev_attr(trunc_page(po_r), 2 * PAGE_SIZE,
1085             PAT_WRITE_COMBINING);
1086         po_r &= PAGE_MASK;
1087         cmd = *(uint32_t *)(mkva + po_r);
1088
1089         if (cmd != MI_BATCH_BUFFER_END) {
1090                 /*
1091                  * batch_len != 0 due to the check at the start of
1092                  * i915_gem_do_execbuffer
1093                  */
1094                 if (batch_obj->base.size > batch_start_offset + batch_len) {
1095                         po_w = po_r + 4;
1096 /* DRM_DEBUG("batchbuffer does not end by MI_BATCH_BUFFER_END !\n"); */
1097                 } else {
1098                         po_w = po_r;
1099 DRM_DEBUG("batchbuffer does not end by MI_BATCH_BUFFER_END, overwriting last bo cmd !\n");
1100                 }
1101                 *(uint32_t *)(mkva + po_w) = MI_BATCH_BUFFER_END;
1102         }
1103
1104         pmap_unmapdev((vm_offset_t)mkva, 2 * PAGE_SIZE);
1105 }
1106
1107 int i915_fix_mi_batchbuffer_end = 0;
1108
1109  static int
1110 i915_reset_gen7_sol_offsets(struct drm_device *dev,
1111                             struct intel_ring_buffer *ring)
1112 {
1113         drm_i915_private_t *dev_priv = dev->dev_private;
1114         int ret, i;
1115
1116         if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
1117                 return 0;
1118
1119         ret = intel_ring_begin(ring, 4 * 3);
1120         if (ret)
1121                 return ret;
1122
1123         for (i = 0; i < 4; i++) {
1124                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1125                 intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
1126                 intel_ring_emit(ring, 0);
1127         }
1128
1129         intel_ring_advance(ring);
1130
1131         return 0;
1132 }
1133
1134 static int
1135 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1136                        struct drm_file *file,
1137                        struct drm_i915_gem_execbuffer2 *args,
1138                        struct drm_i915_gem_exec_object2 *exec)
1139 {
1140         drm_i915_private_t *dev_priv = dev->dev_private;
1141         struct list_head objects;
1142         struct eb_objects *eb;
1143         struct drm_i915_gem_object *batch_obj;
1144         struct drm_clip_rect *cliprects = NULL;
1145         struct intel_ring_buffer *ring;
1146         vm_page_t **relocs_ma;
1147         u32 exec_start, exec_len;
1148         u32 seqno;
1149         u32 mask;
1150         int ret, mode, i;
1151
1152         if (!i915_gem_check_execbuffer(args)) {
1153                 DRM_DEBUG("execbuf with invalid offset/length\n");
1154                 return -EINVAL;
1155         }
1156
1157         if (args->batch_len == 0)
1158                 return (0);
1159
1160         ret = validate_exec_list(exec, args->buffer_count, &relocs_ma);
1161         if (ret != 0)
1162                 goto pre_struct_lock_err;
1163
1164         switch (args->flags & I915_EXEC_RING_MASK) {
1165         case I915_EXEC_DEFAULT:
1166         case I915_EXEC_RENDER:
1167                 ring = &dev_priv->ring[RCS];
1168                 break;
1169         case I915_EXEC_BSD:
1170                 if (!HAS_BSD(dev)) {
1171                         DRM_DEBUG("execbuf with invalid ring (BSD)\n");
1172                         return -EINVAL;
1173                 }
1174                 ring = &dev_priv->ring[VCS];
1175                 break;
1176         case I915_EXEC_BLT:
1177                 if (!HAS_BLT(dev)) {
1178                         DRM_DEBUG("execbuf with invalid ring (BLT)\n");
1179                         return -EINVAL;
1180                 }
1181                 ring = &dev_priv->ring[BCS];
1182                 break;
1183         default:
1184                 DRM_DEBUG("execbuf with unknown ring: %d\n",
1185                           (int)(args->flags & I915_EXEC_RING_MASK));
1186                 ret = -EINVAL;
1187                 goto pre_struct_lock_err;
1188         }
1189
1190         mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1191         mask = I915_EXEC_CONSTANTS_MASK;
1192         switch (mode) {
1193         case I915_EXEC_CONSTANTS_REL_GENERAL:
1194         case I915_EXEC_CONSTANTS_ABSOLUTE:
1195         case I915_EXEC_CONSTANTS_REL_SURFACE:
1196                 if (ring == &dev_priv->ring[RCS] &&
1197                     mode != dev_priv->relative_constants_mode) {
1198                         if (INTEL_INFO(dev)->gen < 4) {
1199                                 ret = -EINVAL;
1200                                 goto pre_struct_lock_err;
1201                         }
1202
1203                         if (INTEL_INFO(dev)->gen > 5 &&
1204                             mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
1205                                 ret = -EINVAL;
1206                                 goto pre_struct_lock_err;
1207                         }
1208
1209                         /* The HW changed the meaning on this bit on gen6 */
1210                         if (INTEL_INFO(dev)->gen >= 6)
1211                                 mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
1212                 }
1213                 break;
1214         default:
1215                 DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
1216                 ret = -EINVAL;
1217                 goto pre_struct_lock_err;
1218         }
1219
1220         if (args->buffer_count < 1) {
1221                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1222                 ret = -EINVAL;
1223                 goto pre_struct_lock_err;
1224         }
1225
1226         if (args->num_cliprects != 0) {
1227                 if (ring != &dev_priv->ring[RCS]) {
1228         DRM_DEBUG("clip rectangles are only valid with the render ring\n");
1229                         ret = -EINVAL;
1230                         goto pre_struct_lock_err;
1231                 }
1232
1233                 if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
1234                         DRM_DEBUG("execbuf with %u cliprects\n",
1235                                   args->num_cliprects);
1236                         ret = -EINVAL;
1237                         goto pre_struct_lock_err;
1238                 }
1239                 cliprects = kmalloc( sizeof(*cliprects) * args->num_cliprects,
1240                     DRM_I915_GEM, M_WAITOK | M_ZERO);
1241                 ret = -copyin((void *)(uintptr_t)args->cliprects_ptr, cliprects,
1242                     sizeof(*cliprects) * args->num_cliprects);
1243                 if (ret != 0)
1244                         goto pre_struct_lock_err;
1245         }
1246
1247         ret = i915_mutex_lock_interruptible(dev);
1248         if (ret)
1249                 goto pre_struct_lock_err;
1250
1251         if (dev_priv->mm.suspended) {
1252                 ret = -EBUSY;
1253                 goto struct_lock_err;
1254         }
1255
1256         eb = eb_create(args->buffer_count);
1257         if (eb == NULL) {
1258                 ret = -ENOMEM;
1259                 goto struct_lock_err;
1260         }
1261
1262         /* Look up object handles */
1263         INIT_LIST_HEAD(&objects);
1264         for (i = 0; i < args->buffer_count; i++) {
1265                 struct drm_i915_gem_object *obj;
1266                 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
1267                                                         exec[i].handle));
1268                 if (&obj->base == NULL) {
1269                         DRM_DEBUG("Invalid object handle %d at index %d\n",
1270                                    exec[i].handle, i);
1271                         /* prevent error path from reading uninitialized data */
1272                         ret = -ENOENT;
1273                         goto err;
1274                 }
1275
1276                 if (!list_empty(&obj->exec_list)) {
1277                         DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
1278                                    obj, exec[i].handle, i);
1279                         ret = -EINVAL;
1280                         goto err;
1281                 }
1282
1283                 list_add_tail(&obj->exec_list, &objects);
1284                 obj->exec_handle = exec[i].handle;
1285                 obj->exec_entry = &exec[i];
1286                 eb_add_object(eb, obj);
1287         }
1288
1289         /* take note of the batch buffer before we might reorder the lists */
1290         batch_obj = list_entry(objects.prev,
1291                                struct drm_i915_gem_object,
1292                                exec_list);
1293
1294         /* Move the objects en-masse into the GTT, evicting if necessary. */
1295         ret = i915_gem_execbuffer_reserve(ring, file, &objects);
1296         if (ret)
1297                 goto err;
1298
1299         /* The objects are in their final locations, apply the relocations. */
1300         ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
1301         if (ret) {
1302                 if (ret == -EFAULT) {
1303                         ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
1304                             &objects, eb, exec, args->buffer_count);
1305                         DRM_LOCK_ASSERT(dev);
1306                 }
1307                 if (ret)
1308                         goto err;
1309         }
1310
1311         /* Set the pending read domains for the batch buffer to COMMAND */
1312         if (batch_obj->base.pending_write_domain) {
1313                 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1314                 ret = -EINVAL;
1315                 goto err;
1316         }
1317         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1318
1319         ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1320         if (ret)
1321                 goto err;
1322
1323         seqno = i915_gem_next_request_seqno(ring);
1324         for (i = 0; i < I915_NUM_RINGS - 1; i++) {
1325                 if (seqno < ring->sync_seqno[i]) {
1326                         /* The GPU can not handle its semaphore value wrapping,
1327                          * so every billion or so execbuffers, we need to stall
1328                          * the GPU in order to reset the counters.
1329                          */
1330                         ret = i915_gpu_idle(dev, true);
1331                         if (ret)
1332                                 goto err;
1333
1334                         KASSERT(ring->sync_seqno[i] == 0, ("Non-zero sync_seqno"));
1335                 }
1336         }
1337
1338         if (ring == &dev_priv->ring[RCS] &&
1339             mode != dev_priv->relative_constants_mode) {
1340                 ret = intel_ring_begin(ring, 4);
1341                 if (ret)
1342                         goto err;
1343
1344                 intel_ring_emit(ring, MI_NOOP);
1345                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1346                 intel_ring_emit(ring, INSTPM);
1347                 intel_ring_emit(ring, mask << 16 | mode);
1348                 intel_ring_advance(ring);
1349
1350                 dev_priv->relative_constants_mode = mode;
1351         }
1352
1353         if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1354                 ret = i915_reset_gen7_sol_offsets(dev, ring);
1355                 if (ret)
1356                         goto err;
1357         }
1358
1359         exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1360         exec_len = args->batch_len;
1361
1362         if (i915_fix_mi_batchbuffer_end) {
1363                 i915_gem_fix_mi_batchbuffer_end(batch_obj,
1364                     args->batch_start_offset, args->batch_len);
1365         }
1366
1367         if (cliprects) {
1368                 for (i = 0; i < args->num_cliprects; i++) {
1369                         ret = i915_emit_box_p(dev, &cliprects[i],
1370                             args->DR1, args->DR4);
1371                         if (ret)
1372                                 goto err;
1373
1374                         ret = ring->dispatch_execbuffer(ring, exec_start,
1375                             exec_len);
1376                         if (ret)
1377                                 goto err;
1378                 }
1379         } else {
1380                 ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
1381                 if (ret)
1382                         goto err;
1383         }
1384
1385         i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
1386         i915_gem_execbuffer_retire_commands(dev, file, ring);
1387
1388 err:
1389         eb_destroy(eb);
1390         while (!list_empty(&objects)) {
1391                 struct drm_i915_gem_object *obj;
1392
1393                 obj = list_first_entry(&objects, struct drm_i915_gem_object,
1394                     exec_list);
1395                 list_del_init(&obj->exec_list);
1396                 drm_gem_object_unreference(&obj->base);
1397         }
1398 struct_lock_err:
1399         DRM_UNLOCK(dev);
1400
1401 pre_struct_lock_err:
1402         for (i = 0; i < args->buffer_count; i++) {
1403                 if (relocs_ma[i] != NULL) {
1404                         vm_page_unhold_pages(relocs_ma[i], howmany(
1405                             exec[i].relocation_count *
1406                             sizeof(struct drm_i915_gem_relocation_entry),
1407                             PAGE_SIZE));
1408                         drm_free(relocs_ma[i], DRM_I915_GEM);
1409                 }
1410         }
1411         drm_free(relocs_ma, DRM_I915_GEM);
1412         drm_free(cliprects, DRM_I915_GEM);
1413         return ret;
1414 }
1415
1416 /*
1417  * Legacy execbuffer just creates an exec2 list from the original exec object
1418  * list array and passes it to the real function.
1419  */
1420 int
1421 i915_gem_execbuffer(struct drm_device *dev, void *data,
1422                     struct drm_file *file)
1423 {
1424         struct drm_i915_gem_execbuffer *args = data;
1425         struct drm_i915_gem_execbuffer2 exec2;
1426         struct drm_i915_gem_exec_object *exec_list = NULL;
1427         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1428         int ret, i;
1429
1430         DRM_DEBUG("buffers_ptr %d buffer_count %d len %08x\n",
1431             (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1432
1433         if (args->buffer_count < 1) {
1434                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1435                 return -EINVAL;
1436         }
1437
1438         /* Copy in the exec list from userland */
1439         /* XXXKIB user-controlled malloc size */
1440         exec_list = kmalloc(sizeof(*exec_list) * args->buffer_count,
1441             DRM_I915_GEM, M_WAITOK);
1442         exec2_list = kmalloc(sizeof(*exec2_list) * args->buffer_count,
1443             DRM_I915_GEM, M_WAITOK);
1444         ret = -copyin((void *)(uintptr_t)args->buffers_ptr, exec_list,
1445             sizeof(*exec_list) * args->buffer_count);
1446         if (ret != 0) {
1447                 DRM_DEBUG("copy %d exec entries failed %d\n",
1448                           args->buffer_count, ret);
1449                 drm_free(exec_list, DRM_I915_GEM);
1450                 drm_free(exec2_list, DRM_I915_GEM);
1451                 return (ret);
1452         }
1453
1454         for (i = 0; i < args->buffer_count; i++) {
1455                 exec2_list[i].handle = exec_list[i].handle;
1456                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1457                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1458                 exec2_list[i].alignment = exec_list[i].alignment;
1459                 exec2_list[i].offset = exec_list[i].offset;
1460                 if (INTEL_INFO(dev)->gen < 4)
1461                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1462                 else
1463                         exec2_list[i].flags = 0;
1464         }
1465
1466         exec2.buffers_ptr = args->buffers_ptr;
1467         exec2.buffer_count = args->buffer_count;
1468         exec2.batch_start_offset = args->batch_start_offset;
1469         exec2.batch_len = args->batch_len;
1470         exec2.DR1 = args->DR1;
1471         exec2.DR4 = args->DR4;
1472         exec2.num_cliprects = args->num_cliprects;
1473         exec2.cliprects_ptr = args->cliprects_ptr;
1474         exec2.flags = I915_EXEC_RENDER;
1475
1476         ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1477         if (!ret) {
1478                 /* Copy the new buffer offsets back to the user's exec list. */
1479                 for (i = 0; i < args->buffer_count; i++)
1480                         exec_list[i].offset = exec2_list[i].offset;
1481                 /* ... and back out to userspace */
1482                 ret = -copyout(exec_list, (void *)(uintptr_t)args->buffers_ptr,
1483                     sizeof(*exec_list) * args->buffer_count);
1484                 if (ret != 0) {
1485                         DRM_DEBUG("failed to copy %d exec entries "
1486                                   "back to user (%d)\n",
1487                                   args->buffer_count, ret);
1488                 }
1489         }
1490
1491         drm_free(exec_list, DRM_I915_GEM);
1492         drm_free(exec2_list, DRM_I915_GEM);
1493         return ret;
1494 }
1495
1496 int
1497 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1498                      struct drm_file *file)
1499 {
1500         struct drm_i915_gem_execbuffer2 *args = data;
1501         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1502         int ret;
1503
1504         DRM_DEBUG("buffers_ptr %jx buffer_count %d len %08x\n",
1505             (uintmax_t)args->buffers_ptr, args->buffer_count, args->batch_len);
1506
1507         if (args->buffer_count < 1 ||
1508             args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1509                 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1510                 return -EINVAL;
1511         }
1512
1513         /* XXXKIB user-controllable kmalloc size */
1514         exec2_list = kmalloc(sizeof(*exec2_list) * args->buffer_count,
1515             DRM_I915_GEM, M_WAITOK);
1516         ret = -copyin((void *)(uintptr_t)args->buffers_ptr, exec2_list,
1517             sizeof(*exec2_list) * args->buffer_count);
1518         if (ret != 0) {
1519                 DRM_DEBUG("copy %d exec entries failed %d\n",
1520                           args->buffer_count, ret);
1521                 drm_free(exec2_list, DRM_I915_GEM);
1522                 return (ret);
1523         }
1524
1525         ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1526         if (!ret) {
1527                 /* Copy the new buffer offsets back to the user's exec list. */
1528                 ret = -copyout(exec2_list, (void *)(uintptr_t)args->buffers_ptr,
1529                     sizeof(*exec2_list) * args->buffer_count);
1530                 if (ret) {
1531                         DRM_DEBUG("failed to copy %d exec entries "
1532                                   "back to user (%d)\n",
1533                                   args->buffer_count, ret);
1534                 }
1535         }
1536
1537         drm_free(exec2_list, DRM_I915_GEM);
1538         return ret;
1539 }