sys/dev/drm/i915/i915_gem_execbuffer.c

   1 /*
   2  * Copyright © 2008,2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include <drm/drmP.h>
  30 #include <drm/i915_drm.h>
  31 #include <linux/highmem.h>
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 struct eb_objects {
  36         int and;
  37         struct hlist_head buckets[0];
  38 };
  39
  40 static struct eb_objects *
  41 eb_create(int size)
  42 {
  43         struct eb_objects *eb;
  44         int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  45         while (count > size)
  46                 count >>= 1;
  47         eb = kmalloc(count*sizeof(struct hlist_head) +
  48                      sizeof(struct eb_objects),
  49                      M_DRM, M_WAITOK | M_ZERO);
  50         if (eb == NULL)
  51                 return eb;
  52
  53         eb->and = count - 1;
  54         return eb;
  55 }
  56
  57 static void
  58 eb_reset(struct eb_objects *eb)
  59 {
  60         memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
  61 }
  62
  63 static void
  64 eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
  65 {
  66         hlist_add_head(&obj->exec_node,
  67                        &eb->buckets[obj->exec_handle & eb->and]);
  68 }
  69
  70 static struct drm_i915_gem_object *
  71 eb_get_object(struct eb_objects *eb, unsigned long handle)
  72 {
  73         struct hlist_head *head;
  74         struct hlist_node *node;
  75         struct drm_i915_gem_object *obj;
  76
  77         head = &eb->buckets[handle & eb->and];
  78         hlist_for_each(node, head) {
  79                 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
  80                 if (obj->exec_handle == handle)
  81                         return obj;
  82         }
  83
  84         return NULL;
  85 }
  86
  87 static void
  88 eb_destroy(struct eb_objects *eb)
  89 {
  90         drm_free(eb, M_DRM);
  91 }
  92
  93 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
  94 {
  95         return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
  96                 !obj->map_and_fenceable ||
  97                 obj->cache_level != I915_CACHE_NONE);
  98 }
  99
 100 static int
 101 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 102                                    struct eb_objects *eb,
 103                                    struct drm_i915_gem_relocation_entry *reloc)
 104 {
 105         struct drm_device *dev = obj->base.dev;
 106         struct drm_gem_object *target_obj;
 107         struct drm_i915_gem_object *target_i915_obj;
 108         uint32_t target_offset;
 109         int ret = -EINVAL;
 110
 111         /* we've already hold a reference to all valid objects */
 112         target_obj = &eb_get_object(eb, reloc->target_handle)->base;
 113         if (unlikely(target_obj == NULL))
 114                 return -ENOENT;
 115
 116         target_i915_obj = to_intel_bo(target_obj);
 117         target_offset = target_i915_obj->gtt_offset;
 118
 119         /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
 120          * pipe_control writes because the gpu doesn't properly redirect them
 121          * through the ppgtt for non_secure batchbuffers. */
 122         if (unlikely(IS_GEN6(dev) &&
 123             reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
 124             !target_i915_obj->has_global_gtt_mapping)) {
 125                 i915_gem_gtt_bind_object(target_i915_obj,
 126                                          target_i915_obj->cache_level);
 127         }
 128
 129         /* Validate that the target is in a valid r/w GPU domain */
 130         if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
 131                 DRM_DEBUG("reloc with multiple write domains: "
 132                           "obj %p target %d offset %d "
 133                           "read %08x write %08x",
 134                           obj, reloc->target_handle,
 135                           (int) reloc->offset,
 136                           reloc->read_domains,
 137                           reloc->write_domain);
 138                 return ret;
 139         }
 140         if (unlikely((reloc->write_domain | reloc->read_domains)
 141                      & ~I915_GEM_GPU_DOMAINS)) {
 142                 DRM_DEBUG("reloc with read/write non-GPU domains: "
 143                           "obj %p target %d offset %d "
 144                           "read %08x write %08x",
 145                           obj, reloc->target_handle,
 146                           (int) reloc->offset,
 147                           reloc->read_domains,
 148                           reloc->write_domain);
 149                 return ret;
 150         }
 151         if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
 152                      reloc->write_domain != target_obj->pending_write_domain)) {
 153                 DRM_DEBUG("Write domain conflict: "
 154                           "obj %p target %d offset %d "
 155                           "new %08x old %08x\n",
 156                           obj, reloc->target_handle,
 157                           (int) reloc->offset,
 158                           reloc->write_domain,
 159                           target_obj->pending_write_domain);
 160                 return ret;
 161         }
 162
 163         target_obj->pending_read_domains |= reloc->read_domains;
 164         target_obj->pending_write_domain |= reloc->write_domain;
 165
 166         /* If the relocation already has the right value in it, no
 167          * more work needs to be done.
 168          */
 169         if (target_offset == reloc->presumed_offset)
 170                 return 0;
 171
 172         /* Check that the relocation address is valid... */
 173         if (unlikely(reloc->offset > obj->base.size - 4)) {
 174                 DRM_DEBUG("Relocation beyond object bounds: "
 175                           "obj %p target %d offset %d size %d.\n",
 176                           obj, reloc->target_handle,
 177                           (int) reloc->offset,
 178                           (int) obj->base.size);
 179                 return ret;
 180         }
 181         if (unlikely(reloc->offset & 3)) {
 182                 DRM_DEBUG("Relocation not 4-byte aligned: "
 183                           "obj %p target %d offset %d.\n",
 184                           obj, reloc->target_handle,
 185                           (int) reloc->offset);
 186                 return ret;
 187         }
 188
 189         /* We can't wait for rendering with pagefaults disabled */
 190         if (obj->active && (curthread->td_flags & TDF_NOFAULT))
 191                 return -EFAULT;
 192
 193         reloc->delta += target_offset;
 194         if (use_cpu_reloc(obj)) {
 195                 uint32_t page_offset = reloc->offset & PAGE_MASK;
 196                 char *vaddr;
 197
 198                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 199                 if (ret)
 200                         return ret;
 201
 202                 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
 203                 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
 204                 kunmap_atomic(vaddr);
 205         } else {
 206                 uint32_t __iomem *reloc_entry;
 207                 char __iomem *reloc_page;
 208
 209                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
 210                 if (ret)
 211                         return ret;
 212
 213                 ret = i915_gem_object_put_fence(obj);
 214                 if (ret)
 215                         return ret;
 216
 217                 /* Map the page containing the relocation we're going to perform.  */
 218                 reloc->offset += obj->gtt_offset;
 219                 reloc_page = pmap_mapdev_attr(dev->agp->base + (reloc->offset &
 220                     ~PAGE_MASK), PAGE_SIZE, PAT_WRITE_COMBINING);
 221                 reloc_entry = (uint32_t *)(reloc_page + (reloc->offset &
 222                     PAGE_MASK));
 223
 224                 iowrite32(reloc->delta, reloc_entry);
 225                 pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE);
 226         }
 227
 228         /* and update the user's relocation entry */
 229         reloc->presumed_offset = target_offset;
 230
 231         return 0;
 232 }
 233
 234 static int
 235 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
 236                                     struct eb_objects *eb)
 237 {
 238 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
 239         struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
 240         struct drm_i915_gem_relocation_entry __user *user_relocs;
 241         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 242         int remain, ret;
 243
 244         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
 245
 246         remain = entry->relocation_count;
 247         while (remain) {
 248                 struct drm_i915_gem_relocation_entry *r = stack_reloc;
 249                 int count = remain;
 250                 if (count > ARRAY_SIZE(stack_reloc))
 251                         count = ARRAY_SIZE(stack_reloc);
 252                 remain -= count;
 253
 254                 if (copyin_nofault(user_relocs, r, count*sizeof(r[0])))
 255                         return -EFAULT;
 256
 257                 do {
 258                         u64 offset = r->presumed_offset;
 259
 260                         ret = i915_gem_execbuffer_relocate_entry(obj, eb, r);
 261                         if (ret)
 262                                 return ret;
 263
 264                         if (r->presumed_offset != offset &&
 265                             copyout_nofault(&r->presumed_offset,
 266                                                     &user_relocs->presumed_offset,
 267                                                     sizeof(r->presumed_offset))) {
 268                                 return -EFAULT;
 269                         }
 270
 271                         user_relocs++;
 272                         r++;
 273                 } while (--count);
 274         }
 275
 276         return 0;
 277 #undef N_RELOC
 278 }
 279
 280 static int
 281 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
 282                                          struct eb_objects *eb,
 283                                          struct drm_i915_gem_relocation_entry *relocs)
 284 {
 285         const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 286         int i, ret;
 287
 288         for (i = 0; i < entry->relocation_count; i++) {
 289                 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
 290                 if (ret)
 291                         return ret;
 292         }
 293
 294         return 0;
 295 }
 296
 297 static int
 298 i915_gem_execbuffer_relocate(struct drm_device *dev,
 299                              struct eb_objects *eb,
 300                              struct list_head *objects)
 301 {
 302         struct drm_i915_gem_object *obj;
 303         int ret = 0;
 304
 305         /* This is the fast path and we cannot handle a pagefault whilst
 306          * holding the struct mutex lest the user pass in the relocations
 307          * contained within a mmaped bo. For in such a case we, the page
 308          * fault handler would call i915_gem_fault() and we would try to
 309          * acquire the struct mutex again. Obviously this is bad and so
 310          * lockdep complains vehemently.
 311          */
 312 #if 0
 313         pagefault_disable();
 314 #endif
 315         list_for_each_entry(obj, objects, exec_list) {
 316                 ret = i915_gem_execbuffer_relocate_object(obj, eb);
 317                 if (ret)
 318                         break;
 319         }
 320 #if 0
 321         pagefault_enable();
 322 #endif
 323
 324         return ret;
 325 }
 326
 327 #define  __EXEC_OBJECT_HAS_PIN (1<<31)
 328 #define  __EXEC_OBJECT_HAS_FENCE (1<<30)
 329
 330 static int
 331 need_reloc_mappable(struct drm_i915_gem_object *obj)
 332 {
 333         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 334         return entry->relocation_count && !use_cpu_reloc(obj);
 335 }
 336
 337 static int
 338 i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
 339                                    struct intel_ring_buffer *ring)
 340 {
 341         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 342         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 343         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 344         bool need_fence, need_mappable;
 345         int ret;
 346
 347         need_fence =
 348                 has_fenced_gpu_access &&
 349                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 350                 obj->tiling_mode != I915_TILING_NONE;
 351         need_mappable = need_fence || need_reloc_mappable(obj);
 352
 353         ret = i915_gem_object_pin(obj, entry->alignment, need_mappable, false);
 354         if (ret)
 355                 return ret;
 356
 357         entry->flags |= __EXEC_OBJECT_HAS_PIN;
 358
 359         if (has_fenced_gpu_access) {
 360                 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
 361                         ret = i915_gem_object_get_fence(obj);
 362                         if (ret)
 363                                 return ret;
 364
 365                         if (i915_gem_object_pin_fence(obj))
 366                                 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
 367
 368                         obj->pending_fenced_gpu_access = true;
 369                 }
 370         }
 371
 372         /* Ensure ppgtt mapping exists if needed */
 373         if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
 374                 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
 375                                        obj, obj->cache_level);
 376
 377                 obj->has_aliasing_ppgtt_mapping = 1;
 378         }
 379
 380         entry->offset = obj->gtt_offset;
 381         return 0;
 382 }
 383
 384 static void
 385 i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
 386 {
 387         struct drm_i915_gem_exec_object2 *entry;
 388
 389         if (!obj->gtt_space)
 390                 return;
 391
 392         entry = obj->exec_entry;
 393
 394         if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
 395                 i915_gem_object_unpin_fence(obj);
 396
 397         if (entry->flags & __EXEC_OBJECT_HAS_PIN)
 398                 i915_gem_object_unpin(obj);
 399
 400         entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 401 }
 402
 403 static int
 404 i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 405                             struct drm_file *file,
 406                             struct list_head *objects)
 407 {
 408         struct drm_i915_gem_object *obj;
 409         struct list_head ordered_objects;
 410         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 411         int retry;
 412
 413         INIT_LIST_HEAD(&ordered_objects);
 414         while (!list_empty(objects)) {
 415                 struct drm_i915_gem_exec_object2 *entry;
 416                 bool need_fence, need_mappable;
 417
 418                 obj = list_first_entry(objects,
 419                                        struct drm_i915_gem_object,
 420                                        exec_list);
 421                 entry = obj->exec_entry;
 422
 423                 need_fence =
 424                         has_fenced_gpu_access &&
 425                         entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 426                         obj->tiling_mode != I915_TILING_NONE;
 427                 need_mappable = need_fence || need_reloc_mappable(obj);
 428
 429                 if (need_mappable)
 430                         list_move(&obj->exec_list, &ordered_objects);
 431                 else
 432                         list_move_tail(&obj->exec_list, &ordered_objects);
 433
 434                 obj->base.pending_read_domains = 0;
 435                 obj->base.pending_write_domain = 0;
 436                 obj->pending_fenced_gpu_access = false;
 437         }
 438         list_splice(&ordered_objects, objects);
 439
 440         /* Attempt to pin all of the buffers into the GTT.
 441          * This is done in 3 phases:
 442          *
 443          * 1a. Unbind all objects that do not match the GTT constraints for
 444          *     the execbuffer (fenceable, mappable, alignment etc).
 445          * 1b. Increment pin count for already bound objects.
 446          * 2.  Bind new objects.
 447          * 3.  Decrement pin count.
 448          *
 449          * This avoid unnecessary unbinding of later objects in order to make
 450          * room for the earlier objects *unless* we need to defragment.
 451          */
 452         retry = 0;
 453         do {
 454                 int ret = 0;
 455
 456                 /* Unbind any ill-fitting objects or pin. */
 457                 list_for_each_entry(obj, objects, exec_list) {
 458                         struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 459                         bool need_fence, need_mappable;
 460
 461                         if (!obj->gtt_space)
 462                                 continue;
 463
 464                         need_fence =
 465                                 has_fenced_gpu_access &&
 466                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 467                                 obj->tiling_mode != I915_TILING_NONE;
 468                         need_mappable = need_fence || need_reloc_mappable(obj);
 469
 470                         if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
 471                             (need_mappable && !obj->map_and_fenceable))
 472                                 ret = i915_gem_object_unbind(obj);
 473                         else
 474                                 ret = i915_gem_execbuffer_reserve_object(obj, ring);
 475                         if (ret)
 476                                 goto err;
 477                 }
 478
 479                 /* Bind fresh objects */
 480                 list_for_each_entry(obj, objects, exec_list) {
 481                         if (obj->gtt_space)
 482                                 continue;
 483
 484                         ret = i915_gem_execbuffer_reserve_object(obj, ring);
 485                         if (ret)
 486                                 goto err;
 487                 }
 488
 489 err:            /* Decrement pin count for bound objects */
 490                 list_for_each_entry(obj, objects, exec_list)
 491                         i915_gem_execbuffer_unreserve_object(obj);
 492
 493                 if (ret != -ENOSPC || retry++)
 494                         return ret;
 495
 496                 ret = i915_gem_evict_everything(ring->dev);
 497                 if (ret)
 498                         return ret;
 499         } while (1);
 500 }
 501
 502 static int
 503 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 504                                   struct drm_file *file,
 505                                   struct intel_ring_buffer *ring,
 506                                   struct list_head *objects,
 507                                   struct eb_objects *eb,
 508                                   struct drm_i915_gem_exec_object2 *exec,
 509                                   int count)
 510 {
 511         struct drm_i915_gem_relocation_entry *reloc;
 512         struct drm_i915_gem_object *obj;
 513         int *reloc_offset;
 514         int i, total, ret;
 515
 516         /* We may process another execbuffer during the unlock... */
 517         while (!list_empty(objects)) {
 518                 obj = list_first_entry(objects,
 519                                        struct drm_i915_gem_object,
 520                                        exec_list);
 521                 list_del_init(&obj->exec_list);
 522                 drm_gem_object_unreference(&obj->base);
 523         }
 524
 525         DRM_UNLOCK(dev);
 526
 527         total = 0;
 528         for (i = 0; i < count; i++)
 529                 total += exec[i].relocation_count;
 530
 531         reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
 532         reloc = drm_malloc_ab(total, sizeof(*reloc));
 533         if (reloc == NULL || reloc_offset == NULL) {
 534                 drm_free_large(reloc);
 535                 drm_free_large(reloc_offset);
 536                 DRM_LOCK(dev);
 537                 return -ENOMEM;
 538         }
 539
 540         total = 0;
 541         for (i = 0; i < count; i++) {
 542                 struct drm_i915_gem_relocation_entry __user *user_relocs;
 543                 u64 invalid_offset = (u64)-1;
 544                 int j;
 545
 546                 user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
 547
 548                 if (copy_from_user(reloc+total, user_relocs,
 549                                    exec[i].relocation_count * sizeof(*reloc))) {
 550                         ret = -EFAULT;
 551                         DRM_LOCK(dev);
 552                         goto err;
 553                 }
 554
 555                 /* As we do not update the known relocation offsets after
 556                  * relocating (due to the complexities in lock handling),
 557                  * we need to mark them as invalid now so that we force the
 558                  * relocation processing next time. Just in case the target
 559                  * object is evicted and then rebound into its old
 560                  * presumed_offset before the next execbuffer - if that
 561                  * happened we would make the mistake of assuming that the
 562                  * relocations were valid.
 563                  */
 564                 for (j = 0; j < exec[i].relocation_count; j++) {
 565                         if (copy_to_user(&user_relocs[j].presumed_offset,
 566                                          &invalid_offset,
 567                                          sizeof(invalid_offset))) {
 568                                 ret = -EFAULT;
 569                                 DRM_LOCK(dev);
 570                                 goto err;
 571                         }
 572                 }
 573
 574                 reloc_offset[i] = total;
 575                 total += exec[i].relocation_count;
 576         }
 577
 578         ret = i915_mutex_lock_interruptible(dev);
 579         if (ret) {
 580                 DRM_LOCK(dev);
 581                 goto err;
 582         }
 583
 584         /* reacquire the objects */
 585         eb_reset(eb);
 586         for (i = 0; i < count; i++) {
 587                 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
 588                                                         exec[i].handle));
 589                 if (&obj->base == NULL) {
 590                         DRM_DEBUG("Invalid object handle %d at index %d\n",
 591                                    exec[i].handle, i);
 592                         ret = -ENOENT;
 593                         goto err;
 594                 }
 595
 596                 list_add_tail(&obj->exec_list, objects);
 597                 obj->exec_handle = exec[i].handle;
 598                 obj->exec_entry = &exec[i];
 599                 eb_add_object(eb, obj);
 600         }
 601
 602         ret = i915_gem_execbuffer_reserve(ring, file, objects);
 603         if (ret)
 604                 goto err;
 605
 606         list_for_each_entry(obj, objects, exec_list) {
 607                 int offset = obj->exec_entry - exec;
 608                 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
 609                                                                reloc + reloc_offset[offset]);
 610                 if (ret)
 611                         goto err;
 612         }
 613
 614         /* Leave the user relocations as are, this is the painfully slow path,
 615          * and we want to avoid the complication of dropping the lock whilst
 616          * having buffers reserved in the aperture and so causing spurious
 617          * ENOSPC for random operations.
 618          */
 619
 620 err:
 621         drm_free_large(reloc);
 622         drm_free_large(reloc_offset);
 623         return ret;
 624 }
 625
 626 static int
 627 i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
 628 {
 629         u32 plane, flip_mask;
 630         int ret;
 631
 632         /* Check for any pending flips. As we only maintain a flip queue depth
 633          * of 1, we can simply insert a WAIT for the next display flip prior
 634          * to executing the batch and avoid stalling the CPU.
 635          */
 636
 637         for (plane = 0; flips >> plane; plane++) {
 638                 if (((flips >> plane) & 1) == 0)
 639                         continue;
 640
 641                 if (plane)
 642                         flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 643                 else
 644                         flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
 645
 646                 ret = intel_ring_begin(ring, 2);
 647                 if (ret)
 648                         return ret;
 649
 650                 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
 651                 intel_ring_emit(ring, MI_NOOP);
 652                 intel_ring_advance(ring);
 653         }
 654
 655         return 0;
 656 }
 657
 658 static int
 659 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 660                                 struct list_head *objects)
 661 {
 662         struct drm_i915_gem_object *obj;
 663         uint32_t flush_domains = 0;
 664         uint32_t flips = 0;
 665         int ret;
 666
 667         list_for_each_entry(obj, objects, exec_list) {
 668                 ret = i915_gem_object_sync(obj, ring);
 669                 if (ret)
 670                         return ret;
 671
 672                 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
 673                         i915_gem_clflush_object(obj);
 674
 675                 if (obj->base.pending_write_domain)
 676                         flips |= atomic_read(&obj->pending_flip);
 677
 678                 flush_domains |= obj->base.write_domain;
 679         }
 680
 681         if (flips) {
 682                 ret = i915_gem_execbuffer_wait_for_flips(ring, flips);
 683                 if (ret)
 684                         return ret;
 685         }
 686
 687         if (flush_domains & I915_GEM_DOMAIN_CPU)
 688                 i915_gem_chipset_flush(ring->dev);
 689
 690         if (flush_domains & I915_GEM_DOMAIN_GTT)
 691                 cpu_sfence();
 692
 693         /* Unconditionally invalidate gpu caches and ensure that we do flush
 694          * any residual writes from the previous batch.
 695          */
 696         return intel_ring_invalidate_all_caches(ring);
 697 }
 698
 699 static bool
 700 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 701 {
 702         return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
 703 }
 704
 705 static int
 706 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
 707                    int count)
 708 {
 709         int i;
 710         int relocs_total = 0;
 711         int relocs_max = INT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
 712
 713         for (i = 0; i < count; i++) {
 714 #if 0
 715                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
 716 #endif
 717                 int length; /* limited by fault_in_pages_readable() */
 718
 719                 /* First check for malicious input causing overflow in
 720                  * the worst case where we need to allocate the entire
 721                  * relocation tree as a single array.
 722                  */
 723                 if (exec[i].relocation_count > relocs_max - relocs_total)
 724                         return -EINVAL;
 725                 relocs_total += exec[i].relocation_count;
 726
 727                 length = exec[i].relocation_count *
 728                         sizeof(struct drm_i915_gem_relocation_entry);
 729 #if 0
 730                 if (!access_ok(VERIFY_READ, ptr, length))
 731                         return -EFAULT;
 732
 733                 /* we may also need to update the presumed offsets */
 734                 if (!access_ok(VERIFY_WRITE, ptr, length))
 735                         return -EFAULT;
 736
 737                 if (fault_in_multipages_readable(ptr, length))
 738                         return -EFAULT;
 739 #endif
 740         }
 741
 742         return 0;
 743 }
 744
 745 static void
 746 i915_gem_execbuffer_move_to_active(struct list_head *objects,
 747                                    struct intel_ring_buffer *ring)
 748 {
 749         struct drm_i915_gem_object *obj;
 750
 751         list_for_each_entry(obj, objects, exec_list) {
 752                 obj->base.read_domains = obj->base.pending_read_domains;
 753                 obj->base.write_domain = obj->base.pending_write_domain;
 754                 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
 755
 756                 i915_gem_object_move_to_active(obj, ring);
 757                 if (obj->base.write_domain) {
 758                         obj->dirty = 1;
 759                         obj->last_write_seqno = intel_ring_get_seqno(ring);
 760                         if (obj->pin_count) /* check for potential scanout */
 761                                 intel_mark_fb_busy(obj);
 762                 }
 763         }
 764 }
 765
 766 static void
 767 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
 768                                     struct drm_file *file,
 769                                     struct intel_ring_buffer *ring)
 770 {
 771         /* Unconditionally force add_request to emit a full flush. */
 772         ring->gpu_caches_dirty = true;
 773
 774         /* Add a breadcrumb for the completion of the batch buffer */
 775         (void)i915_add_request(ring, file, NULL);
 776 }
 777
 778 static int
 779 i915_reset_gen7_sol_offsets(struct drm_device *dev,
 780                             struct intel_ring_buffer *ring)
 781 {
 782         drm_i915_private_t *dev_priv = dev->dev_private;
 783         int ret, i;
 784
 785         if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
 786                 return 0;
 787
 788         ret = intel_ring_begin(ring, 4 * 3);
 789         if (ret)
 790                 return ret;
 791
 792         for (i = 0; i < 4; i++) {
 793                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
 794                 intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
 795                 intel_ring_emit(ring, 0);
 796         }
 797
 798         intel_ring_advance(ring);
 799
 800         return 0;
 801 }
 802
 803 static int
 804 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 805                        struct drm_file *file,
 806                        struct drm_i915_gem_execbuffer2 *args,
 807                        struct drm_i915_gem_exec_object2 *exec)
 808 {
 809         drm_i915_private_t *dev_priv = dev->dev_private;
 810         struct list_head objects;
 811         struct eb_objects *eb;
 812         struct drm_i915_gem_object *batch_obj;
 813         struct drm_clip_rect *cliprects = NULL;
 814         struct intel_ring_buffer *ring;
 815         u32 ctx_id = i915_execbuffer2_get_context_id(*args);
 816         u32 exec_start, exec_len;
 817         u32 mask;
 818         u32 flags;
 819         int ret, mode, i;
 820
 821         if (!i915_gem_check_execbuffer(args)) {
 822                 DRM_DEBUG("execbuf with invalid offset/length\n");
 823                 return -EINVAL;
 824         }
 825
 826         ret = validate_exec_list(exec, args->buffer_count);
 827         if (ret)
 828                 return ret;
 829
 830         flags = 0;
 831         if (args->flags & I915_EXEC_SECURE) {
 832                 flags |= I915_DISPATCH_SECURE;
 833         }
 834         if (args->flags & I915_EXEC_IS_PINNED)
 835                 flags |= I915_DISPATCH_PINNED;
 836
 837         switch (args->flags & I915_EXEC_RING_MASK) {
 838         case I915_EXEC_DEFAULT:
 839         case I915_EXEC_RENDER:
 840                 ring = &dev_priv->ring[RCS];
 841                 break;
 842         case I915_EXEC_BSD:
 843                 ring = &dev_priv->ring[VCS];
 844                 if (ctx_id != 0) {
 845                         DRM_DEBUG("Ring %s doesn't support contexts\n",
 846                                   ring->name);
 847                         return -EPERM;
 848                 }
 849                 break;
 850         case I915_EXEC_BLT:
 851                 ring = &dev_priv->ring[BCS];
 852                 if (ctx_id != 0) {
 853                         DRM_DEBUG("Ring %s doesn't support contexts\n",
 854                                   ring->name);
 855                         return -EPERM;
 856                 }
 857                 break;
 858         default:
 859                 DRM_DEBUG("execbuf with unknown ring: %d\n",
 860                           (int)(args->flags & I915_EXEC_RING_MASK));
 861                 return -EINVAL;
 862         }
 863         if (!intel_ring_initialized(ring)) {
 864                 DRM_DEBUG("execbuf with invalid ring: %d\n",
 865                           (int)(args->flags & I915_EXEC_RING_MASK));
 866                 return -EINVAL;
 867         }
 868
 869         mode = args->flags & I915_EXEC_CONSTANTS_MASK;
 870         mask = I915_EXEC_CONSTANTS_MASK;
 871         switch (mode) {
 872         case I915_EXEC_CONSTANTS_REL_GENERAL:
 873         case I915_EXEC_CONSTANTS_ABSOLUTE:
 874         case I915_EXEC_CONSTANTS_REL_SURFACE:
 875                 if (ring == &dev_priv->ring[RCS] &&
 876                     mode != dev_priv->relative_constants_mode) {
 877                         if (INTEL_INFO(dev)->gen < 4)
 878                                 return -EINVAL;
 879
 880                         if (INTEL_INFO(dev)->gen > 5 &&
 881                             mode == I915_EXEC_CONSTANTS_REL_SURFACE)
 882                                 return -EINVAL;
 883
 884                         /* The HW changed the meaning on this bit on gen6 */
 885                         if (INTEL_INFO(dev)->gen >= 6)
 886                                 mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
 887                 }
 888                 break;
 889         default:
 890                 DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
 891                 return -EINVAL;
 892         }
 893
 894         if (args->buffer_count < 1) {
 895                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
 896                 return -EINVAL;
 897         }
 898
 899         if (args->num_cliprects != 0) {
 900                 if (ring != &dev_priv->ring[RCS]) {
 901                         DRM_DEBUG("clip rectangles are only valid with the render ring\n");
 902                         return -EINVAL;
 903                 }
 904
 905                 if (INTEL_INFO(dev)->gen >= 5) {
 906                         DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
 907                         return -EINVAL;
 908                 }
 909
 910                 if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
 911                         DRM_DEBUG("execbuf with %u cliprects\n",
 912                                   args->num_cliprects);
 913                         return -EINVAL;
 914                 }
 915                 cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
 916                                     M_DRM, M_WAITOK | M_ZERO);
 917                 if (cliprects == NULL) {
 918                         ret = -ENOMEM;
 919                         goto pre_mutex_err;
 920                 }
 921
 922                 if (copy_from_user(cliprects,
 923                                      (struct drm_clip_rect __user *)(uintptr_t)
 924                                      args->cliprects_ptr,
 925                                      sizeof(*cliprects)*args->num_cliprects)) {
 926                         ret = -EFAULT;
 927                         goto pre_mutex_err;
 928                 }
 929         }
 930
 931         ret = i915_mutex_lock_interruptible(dev);
 932         if (ret)
 933                 goto pre_mutex_err;
 934
 935         if (dev_priv->mm.suspended) {
 936                 DRM_UNLOCK(dev);
 937                 ret = -EBUSY;
 938                 goto pre_mutex_err;
 939         }
 940
 941         eb = eb_create(args->buffer_count);
 942         if (eb == NULL) {
 943                 DRM_UNLOCK(dev);
 944                 ret = -ENOMEM;
 945                 goto pre_mutex_err;
 946         }
 947
 948         /* Look up object handles */
 949         INIT_LIST_HEAD(&objects);
 950         for (i = 0; i < args->buffer_count; i++) {
 951                 struct drm_i915_gem_object *obj;
 952
 953                 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
 954                                                         exec[i].handle));
 955                 if (&obj->base == NULL) {
 956                         DRM_DEBUG("Invalid object handle %d at index %d\n",
 957                                    exec[i].handle, i);
 958                         /* prevent error path from reading uninitialized data */
 959                         ret = -ENOENT;
 960                         goto err;
 961                 }
 962
 963                 if (!list_empty(&obj->exec_list)) {
 964                         DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
 965                                    obj, exec[i].handle, i);
 966                         ret = -EINVAL;
 967                         goto err;
 968                 }
 969
 970                 list_add_tail(&obj->exec_list, &objects);
 971                 obj->exec_handle = exec[i].handle;
 972                 obj->exec_entry = &exec[i];
 973                 eb_add_object(eb, obj);
 974         }
 975
 976         /* take note of the batch buffer before we might reorder the lists */
 977         batch_obj = list_entry(objects.prev,
 978                                struct drm_i915_gem_object,
 979                                exec_list);
 980
 981         /* Move the objects en-masse into the GTT, evicting if necessary. */
 982         ret = i915_gem_execbuffer_reserve(ring, file, &objects);
 983         if (ret)
 984                 goto err;
 985
 986         /* The objects are in their final locations, apply the relocations. */
 987         ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
 988         if (ret) {
 989                 if (ret == -EFAULT) {
 990                         ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
 991                                                                 &objects, eb,
 992                                                                 exec,
 993                                                                 args->buffer_count);
 994                         DRM_LOCK_ASSERT(dev);
 995                 }
 996                 if (ret)
 997                         goto err;
 998         }
 999
1000         /* Set the pending read domains for the batch buffer to COMMAND */
1001         if (batch_obj->base.pending_write_domain) {
1002                 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1003                 ret = -EINVAL;
1004                 goto err;
1005         }
1006         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1007
1008         /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1009          * batch" bit. Hence we need to pin secure batches into the global gtt.
1010          * hsw should have this fixed, but let's be paranoid and do it
1011          * unconditionally for now. */
1012         if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
1013                 i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
1014
1015         ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1016         if (ret)
1017                 goto err;
1018
1019         ret = i915_switch_context(ring, file, ctx_id);
1020         if (ret)
1021                 goto err;
1022
1023         if (ring == &dev_priv->ring[RCS] &&
1024             mode != dev_priv->relative_constants_mode) {
1025                 ret = intel_ring_begin(ring, 4);
1026                 if (ret)
1027                                 goto err;
1028
1029                 intel_ring_emit(ring, MI_NOOP);
1030                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1031                 intel_ring_emit(ring, INSTPM);
1032                 intel_ring_emit(ring, mask << 16 | mode);
1033                 intel_ring_advance(ring);
1034
1035                 dev_priv->relative_constants_mode = mode;
1036         }
1037
1038         if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1039                 ret = i915_reset_gen7_sol_offsets(dev, ring);
1040                 if (ret)
1041                         goto err;
1042         }
1043
1044         exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1045         exec_len = args->batch_len;
1046         if (cliprects) {
1047                 for (i = 0; i < args->num_cliprects; i++) {
1048                         ret = i915_emit_box(dev, &cliprects[i],
1049                                             args->DR1, args->DR4);
1050                         if (ret)
1051                                 goto err;
1052
1053                         ret = ring->dispatch_execbuffer(ring,
1054                                                         exec_start, exec_len,
1055                                                         flags);
1056                         if (ret)
1057                                 goto err;
1058                 }
1059         } else {
1060                 ret = ring->dispatch_execbuffer(ring,
1061                                                 exec_start, exec_len,
1062                                                 flags);
1063                 if (ret)
1064                         goto err;
1065         }
1066
1067         i915_gem_execbuffer_move_to_active(&objects, ring);
1068         i915_gem_execbuffer_retire_commands(dev, file, ring);
1069
1070 err:
1071         eb_destroy(eb);
1072         while (!list_empty(&objects)) {
1073                 struct drm_i915_gem_object *obj;
1074
1075                 obj = list_first_entry(&objects,
1076                                        struct drm_i915_gem_object,
1077                                        exec_list);
1078                 list_del_init(&obj->exec_list);
1079                 drm_gem_object_unreference(&obj->base);
1080         }
1081
1082         DRM_UNLOCK(dev);
1083
1084 pre_mutex_err:
1085         drm_free(cliprects, M_DRM);
1086         return ret;
1087 }
1088
1089 /*
1090  * Legacy execbuffer just creates an exec2 list from the original exec object
1091  * list array and passes it to the real function.
1092  */
1093 int
1094 i915_gem_execbuffer(struct drm_device *dev, void *data,
1095                     struct drm_file *file)
1096 {
1097         struct drm_i915_gem_execbuffer *args = data;
1098         struct drm_i915_gem_execbuffer2 exec2;
1099         struct drm_i915_gem_exec_object *exec_list = NULL;
1100         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1101         int ret, i;
1102
1103         if (args->buffer_count < 1) {
1104                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1105                 return -EINVAL;
1106         }
1107
1108         /* Copy in the exec list from userland */
1109         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1110         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1111         if (exec_list == NULL || exec2_list == NULL) {
1112                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1113                           args->buffer_count);
1114                 drm_free_large(exec_list);
1115                 drm_free_large(exec2_list);
1116                 return -ENOMEM;
1117         }
1118
1119         ret = copy_from_user(exec_list,
1120                              (void __user *)(uintptr_t)args->buffers_ptr,
1121                              sizeof(*exec_list) * args->buffer_count);
1122         if (ret != 0) {
1123                 DRM_DEBUG("copy %d exec entries failed %d\n",
1124                           args->buffer_count, ret);
1125                 drm_free_large(exec_list);
1126                 drm_free_large(exec2_list);
1127                 return -EFAULT;
1128         }
1129
1130         for (i = 0; i < args->buffer_count; i++) {
1131                 exec2_list[i].handle = exec_list[i].handle;
1132                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1133                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1134                 exec2_list[i].alignment = exec_list[i].alignment;
1135                 exec2_list[i].offset = exec_list[i].offset;
1136                 if (INTEL_INFO(dev)->gen < 4)
1137                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1138                 else
1139                         exec2_list[i].flags = 0;
1140         }
1141
1142         exec2.buffers_ptr = args->buffers_ptr;
1143         exec2.buffer_count = args->buffer_count;
1144         exec2.batch_start_offset = args->batch_start_offset;
1145         exec2.batch_len = args->batch_len;
1146         exec2.DR1 = args->DR1;
1147         exec2.DR4 = args->DR4;
1148         exec2.num_cliprects = args->num_cliprects;
1149         exec2.cliprects_ptr = args->cliprects_ptr;
1150         exec2.flags = I915_EXEC_RENDER;
1151         i915_execbuffer2_set_context_id(exec2, 0);
1152
1153         ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1154         if (!ret) {
1155                 /* Copy the new buffer offsets back to the user's exec list. */
1156                 for (i = 0; i < args->buffer_count; i++)
1157                         exec_list[i].offset = exec2_list[i].offset;
1158                 /* ... and back out to userspace */
1159                 ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr,
1160                                    exec_list,
1161                                    sizeof(*exec_list) * args->buffer_count);
1162                 if (ret) {
1163                         ret = -EFAULT;
1164                         DRM_DEBUG("failed to copy %d exec entries "
1165                                   "back to user (%d)\n",
1166                                   args->buffer_count, ret);
1167                 }
1168         }
1169
1170         drm_free_large(exec_list);
1171         drm_free_large(exec2_list);
1172         return ret;
1173 }
1174
1175 int
1176 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1177                      struct drm_file *file)
1178 {
1179         struct drm_i915_gem_execbuffer2 *args = data;
1180         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1181         int ret;
1182
1183         if (args->buffer_count < 1 ||
1184             args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1185                 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1186                 return -EINVAL;
1187         }
1188
1189         exec2_list = kmalloc(sizeof(*exec2_list)*args->buffer_count,
1190                              M_DRM, M_WAITOK);
1191         if (exec2_list == NULL)
1192                 exec2_list = drm_malloc_ab(sizeof(*exec2_list),
1193                                            args->buffer_count);
1194         if (exec2_list == NULL) {
1195                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1196                           args->buffer_count);
1197                 return -ENOMEM;
1198         }
1199         ret = copy_from_user(exec2_list,
1200                              (struct drm_i915_relocation_entry __user *)
1201                              (uintptr_t) args->buffers_ptr,
1202                              sizeof(*exec2_list) * args->buffer_count);
1203         if (ret != 0) {
1204                 DRM_DEBUG("copy %d exec entries failed %d\n",
1205                           args->buffer_count, ret);
1206                 drm_free_large(exec2_list);
1207                 return -EFAULT;
1208         }
1209
1210         ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1211         if (!ret) {
1212                 /* Copy the new buffer offsets back to the user's exec list. */
1213                 ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr,
1214                                    exec2_list,
1215                                    sizeof(*exec2_list) * args->buffer_count);
1216                 if (ret) {
1217                         ret = -EFAULT;
1218                         DRM_DEBUG("failed to copy %d exec entries "
1219                                   "back to user (%d)\n",
1220                                   args->buffer_count, ret);
1221                 }
1222         }
1223
1224         drm_free_large(exec2_list);
1225         return ret;
1226 }