sys/dev/drm/ttm/ttm_bo.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  *
  30  * $FreeBSD: head/sys/dev/drm2/ttm/ttm_bo.c 248060 2013-03-08 18:11:02Z dumbbell $
  31  */
  32
  33 #include <dev/drm/drmP.h>
  34 #include <dev/drm/ttm/ttm_module.h>
  35 #include <dev/drm/ttm/ttm_bo_driver.h>
  36 #include <dev/drm/ttm/ttm_placement.h>
  37
  38 #define TTM_ASSERT_LOCKED(param)
  39 #define TTM_DEBUG(fmt, arg...)
  40 #define TTM_BO_HASH_ORDER 13
  41
  42 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
  43 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
  44 static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob);
  45
  46 MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects");
  47
  48 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
  49 {
  50         int i;
  51
  52         for (i = 0; i <= TTM_PL_PRIV5; i++)
  53                 if (flags & (1 << i)) {
  54                         *mem_type = i;
  55                         return 0;
  56                 }
  57         return -EINVAL;
  58 }
  59
  60 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
  61 {
  62         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  63
  64         kprintf("    has_type: %d\n", man->has_type);
  65         kprintf("    use_type: %d\n", man->use_type);
  66         kprintf("    flags: 0x%08X\n", man->flags);
  67         kprintf("    gpu_offset: 0x%08lX\n", man->gpu_offset);
  68         kprintf("    size: %ju\n", (uintmax_t)man->size);
  69         kprintf("    available_caching: 0x%08X\n", man->available_caching);
  70         kprintf("    default_caching: 0x%08X\n", man->default_caching);
  71         if (mem_type != TTM_PL_SYSTEM)
  72                 (*man->func->debug)(man, TTM_PFX);
  73 }
  74
  75 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  76                                         struct ttm_placement *placement)
  77 {
  78         int i, ret, mem_type;
  79
  80         kprintf("No space for %p (%lu pages, %luK, %luM)\n",
  81                bo, bo->mem.num_pages, bo->mem.size >> 10,
  82                bo->mem.size >> 20);
  83         for (i = 0; i < placement->num_placement; i++) {
  84                 ret = ttm_mem_type_from_flags(placement->placement[i],
  85                                                 &mem_type);
  86                 if (ret)
  87                         return;
  88                 kprintf("  placement[%d]=0x%08X (%d)\n",
  89                        i, placement->placement[i], mem_type);
  90                 ttm_mem_type_debug(bo->bdev, mem_type);
  91         }
  92 }
  93
  94 #if 0
  95 static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob,
  96     char *buffer)
  97 {
  98
  99         return snprintf(buffer, PAGE_SIZE, "%lu\n",
 100                         (unsigned long) atomic_read(&glob->bo_count));
 101 }
 102 #endif
 103
 104 static inline uint32_t ttm_bo_type_flags(unsigned type)
 105 {
 106         return 1 << (type);
 107 }
 108
 109 static void ttm_bo_release_list(struct ttm_buffer_object *bo)
 110 {
 111         struct ttm_bo_device *bdev = bo->bdev;
 112         size_t acc_size = bo->acc_size;
 113
 114         KKASSERT(atomic_read(&bo->list_kref) == 0);
 115         KKASSERT(atomic_read(&bo->kref) == 0);
 116         KKASSERT(atomic_read(&bo->cpu_writers) == 0);
 117         KKASSERT(bo->sync_obj == NULL);
 118         KKASSERT(bo->mem.mm_node == NULL);
 119         KKASSERT(list_empty(&bo->lru));
 120         KKASSERT(list_empty(&bo->ddestroy));
 121
 122         if (bo->ttm)
 123                 ttm_tt_destroy(bo->ttm);
 124         atomic_dec(&bo->glob->bo_count);
 125         if (bo->destroy)
 126                 bo->destroy(bo);
 127         else {
 128                 drm_free(bo, M_TTM_BO);
 129         }
 130         ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
 131 }
 132
 133 static int
 134 ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible)
 135 {
 136         const char *wmsg;
 137         int flags, ret;
 138
 139         ret = 0;
 140         if (interruptible) {
 141                 flags = PCATCH;
 142                 wmsg = "ttbowi";
 143         } else {
 144                 flags = 0;
 145                 wmsg = "ttbowu";
 146         }
 147         while (ttm_bo_is_reserved(bo)) {
 148                 ret = -lksleep(bo, &bo->glob->lru_lock, 0, wmsg, 0);
 149                 if (ret != 0)
 150                         break;
 151         }
 152         return (ret);
 153 }
 154
 155 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 156 {
 157         struct ttm_bo_device *bdev = bo->bdev;
 158         struct ttm_mem_type_manager *man;
 159
 160         KKASSERT(ttm_bo_is_reserved(bo));
 161
 162         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 163
 164                 KKASSERT(list_empty(&bo->lru));
 165
 166                 man = &bdev->man[bo->mem.mem_type];
 167                 list_add_tail(&bo->lru, &man->lru);
 168                 refcount_acquire(&bo->list_kref);
 169
 170                 if (bo->ttm != NULL) {
 171                         list_add_tail(&bo->swap, &bo->glob->swap_lru);
 172                         refcount_acquire(&bo->list_kref);
 173                 }
 174         }
 175 }
 176
 177 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 178 {
 179         int put_count = 0;
 180
 181         if (!list_empty(&bo->swap)) {
 182                 list_del_init(&bo->swap);
 183                 ++put_count;
 184         }
 185         if (!list_empty(&bo->lru)) {
 186                 list_del_init(&bo->lru);
 187                 ++put_count;
 188         }
 189
 190         /*
 191          * TODO: Add a driver hook to delete from
 192          * driver-specific LRU's here.
 193          */
 194
 195         return put_count;
 196 }
 197
 198 int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,
 199                           bool interruptible,
 200                           bool no_wait, bool use_sequence, uint32_t sequence)
 201 {
 202         int ret;
 203
 204         while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {
 205                 /**
 206                  * Deadlock avoidance for multi-bo reserving.
 207                  */
 208                 if (use_sequence && bo->seq_valid) {
 209                         /**
 210                          * We've already reserved this one.
 211                          */
 212                         if (unlikely(sequence == bo->val_seq))
 213                                 return -EDEADLK;
 214                         /**
 215                          * Already reserved by a thread that will not back
 216                          * off for us. We need to back off.
 217                          */
 218                         if (unlikely(sequence - bo->val_seq < (1 << 31)))
 219                                 return -EAGAIN;
 220                 }
 221
 222                 if (no_wait)
 223                         return -EBUSY;
 224
 225                 ret = ttm_bo_wait_unreserved_locked(bo, interruptible);
 226
 227                 if (unlikely(ret))
 228                         return ret;
 229         }
 230
 231         if (use_sequence) {
 232                 bool wake_up = false;
 233                 /**
 234                  * Wake up waiters that may need to recheck for deadlock,
 235                  * if we decreased the sequence number.
 236                  */
 237                 if (unlikely((bo->val_seq - sequence < (1 << 31))
 238                              || !bo->seq_valid))
 239                         wake_up = true;
 240
 241                 /*
 242                  * In the worst case with memory ordering these values can be
 243                  * seen in the wrong order. However since we call wake_up_all
 244                  * in that case, this will hopefully not pose a problem,
 245                  * and the worst case would only cause someone to accidentally
 246                  * hit -EAGAIN in ttm_bo_reserve when they see old value of
 247                  * val_seq. However this would only happen if seq_valid was
 248                  * written before val_seq was, and just means some slightly
 249                  * increased cpu usage
 250                  */
 251                 bo->val_seq = sequence;
 252                 bo->seq_valid = true;
 253                 if (wake_up)
 254                         wakeup(bo);
 255         } else {
 256                 bo->seq_valid = false;
 257         }
 258
 259         return 0;
 260 }
 261
 262 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
 263                          bool never_free)
 264 {
 265         u_int old;
 266
 267         old = atomic_fetchadd_int(&bo->list_kref, -count);
 268         if (old <= count) {
 269                 if (never_free)
 270                         panic("ttm_bo_ref_buf");
 271                 ttm_bo_release_list(bo);
 272         }
 273 }
 274
 275 int ttm_bo_reserve(struct ttm_buffer_object *bo,
 276                    bool interruptible,
 277                    bool no_wait, bool use_sequence, uint32_t sequence)
 278 {
 279         struct ttm_bo_global *glob = bo->glob;
 280         int put_count = 0;
 281         int ret;
 282
 283         lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE);
 284         ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence,
 285                                    sequence);
 286         if (likely(ret == 0)) {
 287                 put_count = ttm_bo_del_from_lru(bo);
 288                 lockmgr(&glob->lru_lock, LK_RELEASE);
 289                 ttm_bo_list_ref_sub(bo, put_count, true);
 290         } else
 291                 lockmgr(&bo->glob->lru_lock, LK_RELEASE);
 292
 293         return ret;
 294 }
 295
 296 int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,
 297                                   bool interruptible, uint32_t sequence)
 298 {
 299         bool wake_up = false;
 300         int ret;
 301
 302         while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {
 303                 if (bo->seq_valid && sequence == bo->val_seq) {
 304                         DRM_ERROR(
 305                             "%s: bo->seq_valid && sequence == bo->val_seq",
 306                             __func__);
 307                 }
 308
 309                 ret = ttm_bo_wait_unreserved_locked(bo, interruptible);
 310
 311                 if (unlikely(ret))
 312                         return ret;
 313         }
 314
 315         if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid)
 316                 wake_up = true;
 317
 318         /**
 319          * Wake up waiters that may need to recheck for deadlock,
 320          * if we decreased the sequence number.
 321          */
 322         bo->val_seq = sequence;
 323         bo->seq_valid = true;
 324         if (wake_up)
 325                 wakeup(bo);
 326
 327         return 0;
 328 }
 329
 330 int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,
 331                             bool interruptible, uint32_t sequence)
 332 {
 333         struct ttm_bo_global *glob = bo->glob;
 334         int put_count, ret;
 335
 336         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 337         ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence);
 338         if (likely(!ret)) {
 339                 put_count = ttm_bo_del_from_lru(bo);
 340                 lockmgr(&glob->lru_lock, LK_RELEASE);
 341                 ttm_bo_list_ref_sub(bo, put_count, true);
 342         } else
 343                 lockmgr(&glob->lru_lock, LK_RELEASE);
 344         return ret;
 345 }
 346
 347 void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
 348 {
 349         ttm_bo_add_to_lru(bo);
 350         atomic_set(&bo->reserved, 0);
 351         wakeup(bo);
 352 }
 353
 354 void ttm_bo_unreserve(struct ttm_buffer_object *bo)
 355 {
 356         struct ttm_bo_global *glob = bo->glob;
 357
 358         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 359         ttm_bo_unreserve_locked(bo);
 360         lockmgr(&glob->lru_lock, LK_RELEASE);
 361 }
 362
 363 /*
 364  * Call bo->mutex locked.
 365  */
 366 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
 367 {
 368         struct ttm_bo_device *bdev = bo->bdev;
 369         struct ttm_bo_global *glob = bo->glob;
 370         int ret = 0;
 371         uint32_t page_flags = 0;
 372
 373         TTM_ASSERT_LOCKED(&bo->mutex);
 374         bo->ttm = NULL;
 375
 376         if (bdev->need_dma32)
 377                 page_flags |= TTM_PAGE_FLAG_DMA32;
 378
 379         switch (bo->type) {
 380         case ttm_bo_type_device:
 381                 if (zero_alloc)
 382                         page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
 383         case ttm_bo_type_kernel:
 384                 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
 385                                                       page_flags, glob->dummy_read_page);
 386                 if (unlikely(bo->ttm == NULL))
 387                         ret = -ENOMEM;
 388                 break;
 389         case ttm_bo_type_sg:
 390                 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
 391                                                       page_flags | TTM_PAGE_FLAG_SG,
 392                                                       glob->dummy_read_page);
 393                 if (unlikely(bo->ttm == NULL)) {
 394                         ret = -ENOMEM;
 395                         break;
 396                 }
 397                 bo->ttm->sg = bo->sg;
 398                 break;
 399         default:
 400                 kprintf("[TTM] Illegal buffer object type\n");
 401                 ret = -EINVAL;
 402                 break;
 403         }
 404
 405         return ret;
 406 }
 407
 408 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 409                                   struct ttm_mem_reg *mem,
 410                                   bool evict, bool interruptible,
 411                                   bool no_wait_gpu)
 412 {
 413         struct ttm_bo_device *bdev = bo->bdev;
 414         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 415         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 416         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 417         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 418         int ret = 0;
 419
 420         if (old_is_pci || new_is_pci ||
 421             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 422                 ret = ttm_mem_io_lock(old_man, true);
 423                 if (unlikely(ret != 0))
 424                         goto out_err;
 425                 ttm_bo_unmap_virtual_locked(bo);
 426                 ttm_mem_io_unlock(old_man);
 427         }
 428
 429         /*
 430          * Create and bind a ttm if required.
 431          */
 432
 433         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 434                 if (bo->ttm == NULL) {
 435                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 436                         ret = ttm_bo_add_ttm(bo, zero);
 437                         if (ret)
 438                                 goto out_err;
 439                 }
 440
 441                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 442                 if (ret)
 443                         goto out_err;
 444
 445                 if (mem->mem_type != TTM_PL_SYSTEM) {
 446                         ret = ttm_tt_bind(bo->ttm, mem);
 447                         if (ret)
 448                                 goto out_err;
 449                 }
 450
 451                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 452                         if (bdev->driver->move_notify)
 453                                 bdev->driver->move_notify(bo, mem);
 454                         bo->mem = *mem;
 455                         mem->mm_node = NULL;
 456                         goto moved;
 457                 }
 458         }
 459
 460         if (bdev->driver->move_notify)
 461                 bdev->driver->move_notify(bo, mem);
 462
 463         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 464             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 465                 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem);
 466         else if (bdev->driver->move)
 467                 ret = bdev->driver->move(bo, evict, interruptible,
 468                                          no_wait_gpu, mem);
 469         else
 470                 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem);
 471
 472         if (ret) {
 473                 if (bdev->driver->move_notify) {
 474                         struct ttm_mem_reg tmp_mem = *mem;
 475                         *mem = bo->mem;
 476                         bo->mem = tmp_mem;
 477                         bdev->driver->move_notify(bo, mem);
 478                         bo->mem = *mem;
 479                         *mem = tmp_mem;
 480                 }
 481
 482                 goto out_err;
 483         }
 484
 485 moved:
 486         if (bo->evicted) {
 487                 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
 488                 if (ret)
 489                         kprintf("[TTM] Can not flush read caches\n");
 490                 bo->evicted = false;
 491         }
 492
 493         if (bo->mem.mm_node) {
 494                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 495                     bdev->man[bo->mem.mem_type].gpu_offset;
 496                 bo->cur_placement = bo->mem.placement;
 497         } else
 498                 bo->offset = 0;
 499
 500         return 0;
 501
 502 out_err:
 503         new_man = &bdev->man[bo->mem.mem_type];
 504         if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
 505                 ttm_tt_unbind(bo->ttm);
 506                 ttm_tt_destroy(bo->ttm);
 507                 bo->ttm = NULL;
 508         }
 509
 510         return ret;
 511 }
 512
 513 /**
 514  * Call bo::reserved.
 515  * Will release GPU memory type usage on destruction.
 516  * This is the place to put in driver specific hooks to release
 517  * driver private resources.
 518  * Will release the bo::reserved lock.
 519  */
 520
 521 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 522 {
 523         if (bo->bdev->driver->move_notify)
 524                 bo->bdev->driver->move_notify(bo, NULL);
 525
 526         if (bo->ttm) {
 527                 ttm_tt_unbind(bo->ttm);
 528                 ttm_tt_destroy(bo->ttm);
 529                 bo->ttm = NULL;
 530         }
 531         ttm_bo_mem_put(bo, &bo->mem);
 532
 533         atomic_set(&bo->reserved, 0);
 534         wakeup(&bo);
 535
 536         /*
 537          * Since the final reference to this bo may not be dropped by
 538          * the current task we have to put a memory barrier here to make
 539          * sure the changes done in this function are always visible.
 540          *
 541          * This function only needs protection against the final kref_put.
 542          */
 543         cpu_mfence();
 544 }
 545
 546 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 547 {
 548         struct ttm_bo_device *bdev = bo->bdev;
 549         struct ttm_bo_global *glob = bo->glob;
 550         struct ttm_bo_driver *driver = bdev->driver;
 551         void *sync_obj = NULL;
 552         int put_count;
 553         int ret;
 554
 555         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 556         ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
 557
 558         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 559         (void) ttm_bo_wait(bo, false, false, true);
 560         if (!ret && !bo->sync_obj) {
 561                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 562                 put_count = ttm_bo_del_from_lru(bo);
 563
 564                 lockmgr(&glob->lru_lock, LK_RELEASE);
 565                 ttm_bo_cleanup_memtype_use(bo);
 566
 567                 ttm_bo_list_ref_sub(bo, put_count, true);
 568
 569                 return;
 570         }
 571         if (bo->sync_obj)
 572                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
 573         lockmgr(&bdev->fence_lock, LK_RELEASE);
 574
 575         if (!ret) {
 576                 atomic_set(&bo->reserved, 0);
 577                 wakeup(bo);
 578         }
 579
 580         refcount_acquire(&bo->list_kref);
 581         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 582         lockmgr(&glob->lru_lock, LK_RELEASE);
 583
 584         if (sync_obj) {
 585                 driver->sync_obj_flush(sync_obj);
 586                 driver->sync_obj_unref(&sync_obj);
 587         }
 588         taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq,
 589             ((hz / 100) < 1) ? 1 : hz / 100);
 590 }
 591
 592 /**
 593  * function ttm_bo_cleanup_refs_and_unlock
 594  * If bo idle, remove from delayed- and lru lists, and unref.
 595  * If not idle, do nothing.
 596  *
 597  * Must be called with lru_lock and reservation held, this function
 598  * will drop both before returning.
 599  *
 600  * @interruptible         Any sleeps should occur interruptibly.
 601  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 602  */
 603
 604 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 605                                           bool interruptible,
 606                                           bool no_wait_gpu)
 607 {
 608         struct ttm_bo_device *bdev = bo->bdev;
 609         struct ttm_bo_driver *driver = bdev->driver;
 610         struct ttm_bo_global *glob = bo->glob;
 611         int put_count;
 612         int ret;
 613
 614         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 615         ret = ttm_bo_wait(bo, false, false, true);
 616
 617         if (ret && !no_wait_gpu) {
 618                 void *sync_obj;
 619
 620                 /*
 621                  * Take a reference to the fence and unreserve,
 622                  * at this point the buffer should be dead, so
 623                  * no new sync objects can be attached.
 624                  */
 625                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
 626                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 627
 628                 atomic_set(&bo->reserved, 0);
 629                 wakeup(bo);
 630                 lockmgr(&glob->lru_lock, LK_RELEASE);
 631
 632                 ret = driver->sync_obj_wait(sync_obj, false, interruptible);
 633                 driver->sync_obj_unref(&sync_obj);
 634                 if (ret)
 635                         return ret;
 636
 637                 /*
 638                  * remove sync_obj with ttm_bo_wait, the wait should be
 639                  * finished, and no new wait object should have been added.
 640                  */
 641                 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 642                 ret = ttm_bo_wait(bo, false, false, true);
 643                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 644                 if (ret)
 645                         return ret;
 646
 647                 lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 648                 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
 649
 650                 /*
 651                  * We raced, and lost, someone else holds the reservation now,
 652                  * and is probably busy in ttm_bo_cleanup_memtype_use.
 653                  *
 654                  * Even if it's not the case, because we finished waiting any
 655                  * delayed destruction would succeed, so just return success
 656                  * here.
 657                  */
 658                 if (ret) {
 659                         lockmgr(&glob->lru_lock, LK_RELEASE);
 660                         return 0;
 661                 }
 662         } else
 663                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 664
 665         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 666                 atomic_set(&bo->reserved, 0);
 667                 wakeup(bo);
 668                 lockmgr(&glob->lru_lock, LK_RELEASE);
 669                 return ret;
 670         }
 671
 672         put_count = ttm_bo_del_from_lru(bo);
 673         list_del_init(&bo->ddestroy);
 674         ++put_count;
 675
 676         lockmgr(&glob->lru_lock, LK_RELEASE);
 677         ttm_bo_cleanup_memtype_use(bo);
 678
 679         ttm_bo_list_ref_sub(bo, put_count, true);
 680
 681         return 0;
 682 }
 683
 684 /**
 685  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 686  * encountered buffers.
 687  */
 688
 689 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 690 {
 691         struct ttm_bo_global *glob = bdev->glob;
 692         struct ttm_buffer_object *entry = NULL;
 693         int ret = 0;
 694
 695         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 696         if (list_empty(&bdev->ddestroy))
 697                 goto out_unlock;
 698
 699         entry = list_first_entry(&bdev->ddestroy,
 700                 struct ttm_buffer_object, ddestroy);
 701         refcount_acquire(&entry->list_kref);
 702
 703         for (;;) {
 704                 struct ttm_buffer_object *nentry = NULL;
 705
 706                 if (entry->ddestroy.next != &bdev->ddestroy) {
 707                         nentry = list_first_entry(&entry->ddestroy,
 708                                 struct ttm_buffer_object, ddestroy);
 709                         refcount_acquire(&nentry->list_kref);
 710                 }
 711
 712                 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);
 713                 if (remove_all && ret) {
 714                         ret = ttm_bo_reserve_nolru(entry, false, false,
 715                                                    false, 0);
 716                 }
 717
 718                 if (!ret)
 719                         ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
 720                                                              !remove_all);
 721                 else
 722                         lockmgr(&glob->lru_lock, LK_RELEASE);
 723
 724                 if (refcount_release(&entry->list_kref))
 725                         ttm_bo_release_list(entry);
 726                 entry = nentry;
 727
 728                 if (ret || !entry)
 729                         goto out;
 730
 731                 lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 732                 if (list_empty(&entry->ddestroy))
 733                         break;
 734         }
 735
 736 out_unlock:
 737         lockmgr(&glob->lru_lock, LK_RELEASE);
 738 out:
 739         if (entry && refcount_release(&entry->list_kref))
 740                 ttm_bo_release_list(entry);
 741         return ret;
 742 }
 743
 744 static void ttm_bo_delayed_workqueue(void *arg, int pending __unused)
 745 {
 746         struct ttm_bo_device *bdev = arg;
 747
 748         if (ttm_bo_delayed_delete(bdev, false)) {
 749                 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq,
 750                     ((hz / 100) < 1) ? 1 : hz / 100);
 751         }
 752 }
 753
 754 static void ttm_bo_release(struct ttm_buffer_object *bo)
 755 {
 756         struct ttm_bo_device *bdev = bo->bdev;
 757         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 758
 759         lockmgr(&bdev->vm_lock, LK_EXCLUSIVE);
 760         if (likely(bo->vm_node != NULL)) {
 761                 RB_REMOVE(ttm_bo_device_buffer_objects,
 762                     &bdev->addr_space_rb, bo);
 763                 drm_mm_put_block(bo->vm_node);
 764                 bo->vm_node = NULL;
 765         }
 766         lockmgr(&bdev->vm_lock, LK_RELEASE);
 767         ttm_mem_io_lock(man, false);
 768         ttm_mem_io_free_vm(bo);
 769         ttm_mem_io_unlock(man);
 770         ttm_bo_cleanup_refs_or_queue(bo);
 771         if (refcount_release(&bo->list_kref))
 772                 ttm_bo_release_list(bo);
 773 }
 774
 775 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 776 {
 777         struct ttm_buffer_object *bo = *p_bo;
 778
 779         *p_bo = NULL;
 780         if (refcount_release(&bo->kref))
 781                 ttm_bo_release(bo);
 782 }
 783
 784 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 785 {
 786         int pending;
 787
 788         taskqueue_cancel_timeout(taskqueue_thread[mycpuid], &bdev->wq, &pending);
 789         if (pending)
 790                 taskqueue_drain_timeout(taskqueue_thread[mycpuid], &bdev->wq);
 791         return (pending);
 792 }
 793
 794 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 795 {
 796         if (resched) {
 797                 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq,
 798                     ((hz / 100) < 1) ? 1 : hz / 100);
 799         }
 800 }
 801
 802 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 803                         bool no_wait_gpu)
 804 {
 805         struct ttm_bo_device *bdev = bo->bdev;
 806         struct ttm_mem_reg evict_mem;
 807         struct ttm_placement placement;
 808         int ret = 0;
 809
 810         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 811         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
 812         lockmgr(&bdev->fence_lock, LK_RELEASE);
 813
 814         if (unlikely(ret != 0)) {
 815                 if (ret != -ERESTART) {
 816                         kprintf("[TTM] Failed to expire sync object before buffer eviction\n");
 817                 }
 818                 goto out;
 819         }
 820
 821         KKASSERT(ttm_bo_is_reserved(bo));
 822
 823         evict_mem = bo->mem;
 824         evict_mem.mm_node = NULL;
 825         evict_mem.bus.io_reserved_vm = false;
 826         evict_mem.bus.io_reserved_count = 0;
 827
 828         placement.fpfn = 0;
 829         placement.lpfn = 0;
 830         placement.num_placement = 0;
 831         placement.num_busy_placement = 0;
 832         bdev->driver->evict_flags(bo, &placement);
 833         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
 834                                 no_wait_gpu);
 835         if (ret) {
 836                 if (ret != -ERESTART) {
 837                         kprintf("[TTM] Failed to find memory space for buffer 0x%p eviction\n",
 838                                bo);
 839                         ttm_bo_mem_space_debug(bo, &placement);
 840                 }
 841                 goto out;
 842         }
 843
 844         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
 845                                      no_wait_gpu);
 846         if (ret) {
 847                 if (ret != -ERESTART)
 848                         kprintf("[TTM] Buffer eviction failed\n");
 849                 ttm_bo_mem_put(bo, &evict_mem);
 850                 goto out;
 851         }
 852         bo->evicted = true;
 853 out:
 854         return ret;
 855 }
 856
 857 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 858                                 uint32_t mem_type,
 859                                 bool interruptible,
 860                                 bool no_wait_gpu)
 861 {
 862         struct ttm_bo_global *glob = bdev->glob;
 863         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 864         struct ttm_buffer_object *bo;
 865         int ret = -EBUSY, put_count;
 866
 867         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
 868         list_for_each_entry(bo, &man->lru, lru) {
 869                 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
 870                 if (!ret)
 871                         break;
 872         }
 873
 874         if (ret) {
 875                 lockmgr(&glob->lru_lock, LK_RELEASE);
 876                 return ret;
 877         }
 878
 879         refcount_acquire(&bo->list_kref);
 880
 881         if (!list_empty(&bo->ddestroy)) {
 882                 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,
 883                                                      no_wait_gpu);
 884                 if (refcount_release(&bo->list_kref))
 885                         ttm_bo_release_list(bo);
 886                 return ret;
 887         }
 888
 889         put_count = ttm_bo_del_from_lru(bo);
 890         lockmgr(&glob->lru_lock, LK_RELEASE);
 891
 892         KKASSERT(ret == 0);
 893
 894         ttm_bo_list_ref_sub(bo, put_count, true);
 895
 896         ret = ttm_bo_evict(bo, interruptible, no_wait_gpu);
 897         ttm_bo_unreserve(bo);
 898
 899         if (refcount_release(&bo->list_kref))
 900                 ttm_bo_release_list(bo);
 901         return ret;
 902 }
 903
 904 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 905 {
 906         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 907
 908         if (mem->mm_node)
 909                 (*man->func->put_node)(man, mem);
 910 }
 911
 912 /**
 913  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 914  * space, or we've evicted everything and there isn't enough space.
 915  */
 916 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 917                                         uint32_t mem_type,
 918                                         struct ttm_placement *placement,
 919                                         struct ttm_mem_reg *mem,
 920                                         bool interruptible,
 921                                         bool no_wait_gpu)
 922 {
 923         struct ttm_bo_device *bdev = bo->bdev;
 924         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 925         int ret;
 926
 927         do {
 928                 ret = (*man->func->get_node)(man, bo, placement, mem);
 929                 if (unlikely(ret != 0))
 930                         return ret;
 931                 if (mem->mm_node)
 932                         break;
 933                 ret = ttm_mem_evict_first(bdev, mem_type,
 934                                           interruptible, no_wait_gpu);
 935                 if (unlikely(ret != 0))
 936                         return ret;
 937         } while (1);
 938         if (mem->mm_node == NULL)
 939                 return -ENOMEM;
 940         mem->mem_type = mem_type;
 941         return 0;
 942 }
 943
 944 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 945                                       uint32_t cur_placement,
 946                                       uint32_t proposed_placement)
 947 {
 948         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 949         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 950
 951         /**
 952          * Keep current caching if possible.
 953          */
 954
 955         if ((cur_placement & caching) != 0)
 956                 result |= (cur_placement & caching);
 957         else if ((man->default_caching & caching) != 0)
 958                 result |= man->default_caching;
 959         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 960                 result |= TTM_PL_FLAG_CACHED;
 961         else if ((TTM_PL_FLAG_WC & caching) != 0)
 962                 result |= TTM_PL_FLAG_WC;
 963         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 964                 result |= TTM_PL_FLAG_UNCACHED;
 965
 966         return result;
 967 }
 968
 969 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 970                                  uint32_t mem_type,
 971                                  uint32_t proposed_placement,
 972                                  uint32_t *masked_placement)
 973 {
 974         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 975
 976         if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
 977                 return false;
 978
 979         if ((proposed_placement & man->available_caching) == 0)
 980                 return false;
 981
 982         cur_flags |= (proposed_placement & man->available_caching);
 983
 984         *masked_placement = cur_flags;
 985         return true;
 986 }
 987
 988 /**
 989  * Creates space for memory region @mem according to its type.
 990  *
 991  * This function first searches for free space in compatible memory types in
 992  * the priority order defined by the driver.  If free space isn't found, then
 993  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 994  * space.
 995  */
 996 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 997                         struct ttm_placement *placement,
 998                         struct ttm_mem_reg *mem,
 999                         bool interruptible,
1000                         bool no_wait_gpu)
1001 {
1002         struct ttm_bo_device *bdev = bo->bdev;
1003         struct ttm_mem_type_manager *man;
1004         uint32_t mem_type = TTM_PL_SYSTEM;
1005         uint32_t cur_flags = 0;
1006         bool type_found = false;
1007         bool type_ok = false;
1008         bool has_erestartsys = false;
1009         int i, ret;
1010
1011         mem->mm_node = NULL;
1012         for (i = 0; i < placement->num_placement; ++i) {
1013                 ret = ttm_mem_type_from_flags(placement->placement[i],
1014                                                 &mem_type);
1015                 if (ret)
1016                         return ret;
1017                 man = &bdev->man[mem_type];
1018
1019                 type_ok = ttm_bo_mt_compatible(man,
1020                                                 mem_type,
1021                                                 placement->placement[i],
1022                                                 &cur_flags);
1023
1024                 if (!type_ok)
1025                         continue;
1026
1027                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1028                                                   cur_flags);
1029                 /*
1030                  * Use the access and other non-mapping-related flag bits from
1031                  * the memory placement flags to the current flags
1032                  */
1033                 ttm_flag_masked(&cur_flags, placement->placement[i],
1034                                 ~TTM_PL_MASK_MEMTYPE);
1035
1036                 if (mem_type == TTM_PL_SYSTEM)
1037                         break;
1038
1039                 if (man->has_type && man->use_type) {
1040                         type_found = true;
1041                         ret = (*man->func->get_node)(man, bo, placement, mem);
1042                         if (unlikely(ret))
1043                                 return ret;
1044                 }
1045                 if (mem->mm_node)
1046                         break;
1047         }
1048
1049         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
1050                 mem->mem_type = mem_type;
1051                 mem->placement = cur_flags;
1052                 return 0;
1053         }
1054
1055         if (!type_found)
1056                 return -EINVAL;
1057
1058         for (i = 0; i < placement->num_busy_placement; ++i) {
1059                 ret = ttm_mem_type_from_flags(placement->busy_placement[i],
1060                                                 &mem_type);
1061                 if (ret)
1062                         return ret;
1063                 man = &bdev->man[mem_type];
1064                 if (!man->has_type)
1065                         continue;
1066                 if (!ttm_bo_mt_compatible(man,
1067                                                 mem_type,
1068                                                 placement->busy_placement[i],
1069                                                 &cur_flags))
1070                         continue;
1071
1072                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
1073                                                   cur_flags);
1074                 /*
1075                  * Use the access and other non-mapping-related flag bits from
1076                  * the memory placement flags to the current flags
1077                  */
1078                 ttm_flag_masked(&cur_flags, placement->busy_placement[i],
1079                                 ~TTM_PL_MASK_MEMTYPE);
1080
1081
1082                 if (mem_type == TTM_PL_SYSTEM) {
1083                         mem->mem_type = mem_type;
1084                         mem->placement = cur_flags;
1085                         mem->mm_node = NULL;
1086                         return 0;
1087                 }
1088
1089                 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
1090                                                 interruptible, no_wait_gpu);
1091                 if (ret == 0 && mem->mm_node) {
1092                         mem->placement = cur_flags;
1093                         return 0;
1094                 }
1095                 if (ret == -ERESTART)
1096                         has_erestartsys = true;
1097         }
1098         ret = (has_erestartsys) ? -ERESTART : -ENOMEM;
1099         return ret;
1100 }
1101
1102 static
1103 int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1104                         struct ttm_placement *placement,
1105                         bool interruptible,
1106                         bool no_wait_gpu)
1107 {
1108         int ret = 0;
1109         struct ttm_mem_reg mem;
1110         struct ttm_bo_device *bdev = bo->bdev;
1111
1112         KKASSERT(ttm_bo_is_reserved(bo));
1113
1114         /*
1115          * FIXME: It's possible to pipeline buffer moves.
1116          * Have the driver move function wait for idle when necessary,
1117          * instead of doing it here.
1118          */
1119         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1120         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
1121         lockmgr(&bdev->fence_lock, LK_RELEASE);
1122         if (ret)
1123                 return ret;
1124         mem.num_pages = bo->num_pages;
1125         mem.size = mem.num_pages << PAGE_SHIFT;
1126         mem.page_alignment = bo->mem.page_alignment;
1127         mem.bus.io_reserved_vm = false;
1128         mem.bus.io_reserved_count = 0;
1129         /*
1130          * Determine where to move the buffer.
1131          */
1132         ret = ttm_bo_mem_space(bo, placement, &mem,
1133                                interruptible, no_wait_gpu);
1134         if (ret)
1135                 goto out_unlock;
1136         ret = ttm_bo_handle_move_mem(bo, &mem, false,
1137                                      interruptible, no_wait_gpu);
1138 out_unlock:
1139         if (ret && mem.mm_node)
1140                 ttm_bo_mem_put(bo, &mem);
1141         return ret;
1142 }
1143
1144 static int ttm_bo_mem_compat(struct ttm_placement *placement,
1145                              struct ttm_mem_reg *mem)
1146 {
1147         int i;
1148
1149         if (mem->mm_node && placement->lpfn != 0 &&
1150             (mem->start < placement->fpfn ||
1151              mem->start + mem->num_pages > placement->lpfn))
1152                 return -1;
1153
1154         for (i = 0; i < placement->num_placement; i++) {
1155                 if ((placement->placement[i] & mem->placement &
1156                         TTM_PL_MASK_CACHING) &&
1157                         (placement->placement[i] & mem->placement &
1158                         TTM_PL_MASK_MEM))
1159                         return i;
1160         }
1161         return -1;
1162 }
1163
1164 int ttm_bo_validate(struct ttm_buffer_object *bo,
1165                         struct ttm_placement *placement,
1166                         bool interruptible,
1167                         bool no_wait_gpu)
1168 {
1169         int ret;
1170
1171         KKASSERT(ttm_bo_is_reserved(bo));
1172         /* Check that range is valid */
1173         if (placement->lpfn || placement->fpfn)
1174                 if (placement->fpfn > placement->lpfn ||
1175                         (placement->lpfn - placement->fpfn) < bo->num_pages)
1176                         return -EINVAL;
1177         /*
1178          * Check whether we need to move buffer.
1179          */
1180         ret = ttm_bo_mem_compat(placement, &bo->mem);
1181         if (ret < 0) {
1182                 ret = ttm_bo_move_buffer(bo, placement, interruptible,
1183                                          no_wait_gpu);
1184                 if (ret)
1185                         return ret;
1186         } else {
1187                 /*
1188                  * Use the access and other non-mapping-related flag bits from
1189                  * the compatible memory placement flags to the active flags
1190                  */
1191                 ttm_flag_masked(&bo->mem.placement, placement->placement[ret],
1192                                 ~TTM_PL_MASK_MEMTYPE);
1193         }
1194         /*
1195          * We might need to add a TTM.
1196          */
1197         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1198                 ret = ttm_bo_add_ttm(bo, true);
1199                 if (ret)
1200                         return ret;
1201         }
1202         return 0;
1203 }
1204
1205 int ttm_bo_check_placement(struct ttm_buffer_object *bo,
1206                                 struct ttm_placement *placement)
1207 {
1208         KKASSERT(!((placement->fpfn || placement->lpfn) &&
1209             (bo->mem.num_pages > (placement->lpfn - placement->fpfn))));
1210
1211         return 0;
1212 }
1213
1214 int ttm_bo_init(struct ttm_bo_device *bdev,
1215                 struct ttm_buffer_object *bo,
1216                 unsigned long size,
1217                 enum ttm_bo_type type,
1218                 struct ttm_placement *placement,
1219                 uint32_t page_alignment,
1220                 bool interruptible,
1221                 struct vm_object *persistent_swap_storage,
1222                 size_t acc_size,
1223                 struct sg_table *sg,
1224                 void (*destroy) (struct ttm_buffer_object *))
1225 {
1226         int ret = 0;
1227         unsigned long num_pages;
1228         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1229
1230         ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
1231         if (ret) {
1232                 kprintf("[TTM] Out of kernel memory\n");
1233                 if (destroy)
1234                         (*destroy)(bo);
1235                 else
1236                         drm_free(bo, M_TTM_BO);
1237                 return -ENOMEM;
1238         }
1239
1240         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1241         if (num_pages == 0) {
1242                 kprintf("[TTM] Illegal buffer object size\n");
1243                 if (destroy)
1244                         (*destroy)(bo);
1245                 else
1246                         drm_free(bo, M_TTM_BO);
1247                 ttm_mem_global_free(mem_glob, acc_size);
1248                 return -EINVAL;
1249         }
1250         bo->destroy = destroy;
1251
1252         refcount_init(&bo->kref, 1);
1253         refcount_init(&bo->list_kref, 1);
1254         atomic_set(&bo->cpu_writers, 0);
1255         atomic_set(&bo->reserved, 1);
1256         INIT_LIST_HEAD(&bo->lru);
1257         INIT_LIST_HEAD(&bo->ddestroy);
1258         INIT_LIST_HEAD(&bo->swap);
1259         INIT_LIST_HEAD(&bo->io_reserve_lru);
1260         bo->bdev = bdev;
1261         bo->glob = bdev->glob;
1262         bo->type = type;
1263         bo->num_pages = num_pages;
1264         bo->mem.size = num_pages << PAGE_SHIFT;
1265         bo->mem.mem_type = TTM_PL_SYSTEM;
1266         bo->mem.num_pages = bo->num_pages;
1267         bo->mem.mm_node = NULL;
1268         bo->mem.page_alignment = page_alignment;
1269         bo->mem.bus.io_reserved_vm = false;
1270         bo->mem.bus.io_reserved_count = 0;
1271         bo->priv_flags = 0;
1272         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1273         bo->seq_valid = false;
1274         bo->persistent_swap_storage = persistent_swap_storage;
1275         bo->acc_size = acc_size;
1276         bo->sg = sg;
1277         atomic_inc(&bo->glob->bo_count);
1278
1279         ret = ttm_bo_check_placement(bo, placement);
1280         if (unlikely(ret != 0))
1281                 goto out_err;
1282
1283         /*
1284          * For ttm_bo_type_device buffers, allocate
1285          * address space from the device.
1286          */
1287         if (bo->type == ttm_bo_type_device ||
1288             bo->type == ttm_bo_type_sg) {
1289                 ret = ttm_bo_setup_vm(bo);
1290                 if (ret)
1291                         goto out_err;
1292         }
1293
1294         ret = ttm_bo_validate(bo, placement, interruptible, false);
1295         if (ret)
1296                 goto out_err;
1297
1298         ttm_bo_unreserve(bo);
1299         return 0;
1300
1301 out_err:
1302         ttm_bo_unreserve(bo);
1303         ttm_bo_unref(&bo);
1304
1305         return ret;
1306 }
1307
1308 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1309                        unsigned long bo_size,
1310                        unsigned struct_size)
1311 {
1312         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1313         size_t size = 0;
1314
1315         size += ttm_round_pot(struct_size);
1316         size += PAGE_ALIGN(npages * sizeof(void *));
1317         size += ttm_round_pot(sizeof(struct ttm_tt));
1318         return size;
1319 }
1320
1321 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1322                            unsigned long bo_size,
1323                            unsigned struct_size)
1324 {
1325         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1326         size_t size = 0;
1327
1328         size += ttm_round_pot(struct_size);
1329         size += PAGE_ALIGN(npages * sizeof(void *));
1330         size += PAGE_ALIGN(npages * sizeof(dma_addr_t));
1331         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1332         return size;
1333 }
1334
1335 int ttm_bo_create(struct ttm_bo_device *bdev,
1336                         unsigned long size,
1337                         enum ttm_bo_type type,
1338                         struct ttm_placement *placement,
1339                         uint32_t page_alignment,
1340                         bool interruptible,
1341                         struct vm_object *persistent_swap_storage,
1342                         struct ttm_buffer_object **p_bo)
1343 {
1344         struct ttm_buffer_object *bo;
1345         size_t acc_size;
1346         int ret;
1347
1348         bo = kmalloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO);
1349         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1350         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1351                           interruptible, persistent_swap_storage, acc_size,
1352                           NULL, NULL);
1353         if (likely(ret == 0))
1354                 *p_bo = bo;
1355
1356         return ret;
1357 }
1358
1359 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1360                                         unsigned mem_type, bool allow_errors)
1361 {
1362         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1363         struct ttm_bo_global *glob = bdev->glob;
1364         int ret;
1365
1366         /*
1367          * Can't use standard list traversal since we're unlocking.
1368          */
1369
1370         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
1371         while (!list_empty(&man->lru)) {
1372                 lockmgr(&glob->lru_lock, LK_RELEASE);
1373                 ret = ttm_mem_evict_first(bdev, mem_type, false, false);
1374                 if (ret) {
1375                         if (allow_errors) {
1376                                 return ret;
1377                         } else {
1378                                 kprintf("[TTM] Cleanup eviction failed\n");
1379                         }
1380                 }
1381                 lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
1382         }
1383         lockmgr(&glob->lru_lock, LK_RELEASE);
1384         return 0;
1385 }
1386
1387 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1388 {
1389         struct ttm_mem_type_manager *man;
1390         int ret = -EINVAL;
1391
1392         if (mem_type >= TTM_NUM_MEM_TYPES) {
1393                 kprintf("[TTM] Illegal memory type %d\n", mem_type);
1394                 return ret;
1395         }
1396         man = &bdev->man[mem_type];
1397
1398         if (!man->has_type) {
1399                 kprintf("[TTM] Trying to take down uninitialized memory manager type %u\n",
1400                        mem_type);
1401                 return ret;
1402         }
1403
1404         man->use_type = false;
1405         man->has_type = false;
1406
1407         ret = 0;
1408         if (mem_type > 0) {
1409                 ttm_bo_force_list_clean(bdev, mem_type, false);
1410
1411                 ret = (*man->func->takedown)(man);
1412         }
1413
1414         return ret;
1415 }
1416
1417 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1418 {
1419         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1420
1421         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1422                 kprintf("[TTM] Illegal memory manager memory type %u\n", mem_type);
1423                 return -EINVAL;
1424         }
1425
1426         if (!man->has_type) {
1427                 kprintf("[TTM] Memory type %u has not been initialized\n", mem_type);
1428                 return 0;
1429         }
1430
1431         return ttm_bo_force_list_clean(bdev, mem_type, true);
1432 }
1433
1434 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1435                         unsigned long p_size)
1436 {
1437         int ret = -EINVAL;
1438         struct ttm_mem_type_manager *man;
1439
1440         KKASSERT(type < TTM_NUM_MEM_TYPES);
1441         man = &bdev->man[type];
1442         KKASSERT(!man->has_type);
1443         man->io_reserve_fastpath = true;
1444         man->use_io_reserve_lru = false;
1445         lockinit(&man->io_reserve_mutex, "ttmman", 0, LK_CANRECURSE);
1446         INIT_LIST_HEAD(&man->io_reserve_lru);
1447
1448         ret = bdev->driver->init_mem_type(bdev, type, man);
1449         if (ret)
1450                 return ret;
1451         man->bdev = bdev;
1452
1453         ret = 0;
1454         if (type != TTM_PL_SYSTEM) {
1455                 ret = (*man->func->init)(man, p_size);
1456                 if (ret)
1457                         return ret;
1458         }
1459         man->has_type = true;
1460         man->use_type = true;
1461         man->size = p_size;
1462
1463         INIT_LIST_HEAD(&man->lru);
1464
1465         return 0;
1466 }
1467
1468 static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob)
1469 {
1470
1471         ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
1472         vm_page_free(glob->dummy_read_page);
1473 }
1474
1475 void ttm_bo_global_release(struct drm_global_reference *ref)
1476 {
1477         struct ttm_bo_global *glob = ref->object;
1478
1479         if (refcount_release(&glob->kobj_ref))
1480                 ttm_bo_global_kobj_release(glob);
1481 }
1482
1483 int ttm_bo_global_init(struct drm_global_reference *ref)
1484 {
1485         struct ttm_bo_global_ref *bo_ref =
1486                 container_of(ref, struct ttm_bo_global_ref, ref);
1487         struct ttm_bo_global *glob = ref->object;
1488         int ret;
1489
1490         lockinit(&glob->device_list_mutex, "ttmdlm", 0, LK_CANRECURSE);
1491         lockinit(&glob->lru_lock, "ttmlru", 0, LK_CANRECURSE);
1492         glob->mem_glob = bo_ref->mem_glob;
1493         glob->dummy_read_page = vm_page_alloc_contig(
1494             0, VM_MAX_ADDRESS, PAGE_SIZE, 0, 1*PAGE_SIZE, VM_MEMATTR_UNCACHEABLE);
1495
1496         if (unlikely(glob->dummy_read_page == NULL)) {
1497                 ret = -ENOMEM;
1498                 goto out_no_drp;
1499         }
1500
1501         INIT_LIST_HEAD(&glob->swap_lru);
1502         INIT_LIST_HEAD(&glob->device_list);
1503
1504         ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
1505         ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
1506         if (unlikely(ret != 0)) {
1507                 kprintf("[TTM] Could not register buffer object swapout\n");
1508                 goto out_no_shrink;
1509         }
1510
1511         atomic_set(&glob->bo_count, 0);
1512
1513         refcount_init(&glob->kobj_ref, 1);
1514         return (0);
1515
1516 out_no_shrink:
1517         vm_page_free(glob->dummy_read_page);
1518 out_no_drp:
1519         drm_free(glob, M_DRM_GLOBAL);
1520         return ret;
1521 }
1522
1523 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1524 {
1525         int ret = 0;
1526         unsigned i = TTM_NUM_MEM_TYPES;
1527         struct ttm_mem_type_manager *man;
1528         struct ttm_bo_global *glob = bdev->glob;
1529
1530         while (i--) {
1531                 man = &bdev->man[i];
1532                 if (man->has_type) {
1533                         man->use_type = false;
1534                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1535                                 ret = -EBUSY;
1536                                 kprintf("[TTM] DRM memory manager type %d is not clean\n",
1537                                        i);
1538                         }
1539                         man->has_type = false;
1540                 }
1541         }
1542
1543         lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE);
1544         list_del(&bdev->device_list);
1545         lockmgr(&glob->device_list_mutex, LK_RELEASE);
1546
1547         if (taskqueue_cancel_timeout(taskqueue_thread[mycpuid], &bdev->wq, NULL))
1548                 taskqueue_drain_timeout(taskqueue_thread[mycpuid], &bdev->wq);
1549
1550         while (ttm_bo_delayed_delete(bdev, true))
1551                 ;
1552
1553         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
1554         if (list_empty(&bdev->ddestroy))
1555                 TTM_DEBUG("Delayed destroy list was clean\n");
1556
1557         if (list_empty(&bdev->man[0].lru))
1558                 TTM_DEBUG("Swap list was clean\n");
1559         lockmgr(&glob->lru_lock, LK_RELEASE);
1560
1561         KKASSERT(drm_mm_clean(&bdev->addr_space_mm));
1562         lockmgr(&bdev->vm_lock, LK_EXCLUSIVE);
1563         drm_mm_takedown(&bdev->addr_space_mm);
1564         lockmgr(&bdev->vm_lock, LK_RELEASE);
1565
1566         return ret;
1567 }
1568
1569 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1570                        struct ttm_bo_global *glob,
1571                        struct ttm_bo_driver *driver,
1572                        uint64_t file_page_offset,
1573                        bool need_dma32)
1574 {
1575         int ret = -EINVAL;
1576
1577         lockinit(&bdev->vm_lock, "ttmvml", 0, LK_CANRECURSE);
1578         bdev->driver = driver;
1579
1580         memset(bdev->man, 0, sizeof(bdev->man));
1581
1582         /*
1583          * Initialize the system memory buffer type.
1584          * Other types need to be driver / IOCTL initialized.
1585          */
1586         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1587         if (unlikely(ret != 0))
1588                 goto out_no_sys;
1589
1590         RB_INIT(&bdev->addr_space_rb);
1591         ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
1592         if (unlikely(ret != 0))
1593                 goto out_no_addr_mm;
1594
1595         TIMEOUT_TASK_INIT(taskqueue_thread[mycpuid], &bdev->wq, 0,
1596             ttm_bo_delayed_workqueue, bdev);
1597         INIT_LIST_HEAD(&bdev->ddestroy);
1598         bdev->dev_mapping = NULL;
1599         bdev->glob = glob;
1600         bdev->need_dma32 = need_dma32;
1601         bdev->val_seq = 0;
1602         lockinit(&bdev->fence_lock, "ttmfence", 0, LK_CANRECURSE);
1603         lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE);
1604         list_add_tail(&bdev->device_list, &glob->device_list);
1605         lockmgr(&glob->device_list_mutex, LK_RELEASE);
1606
1607         return 0;
1608 out_no_addr_mm:
1609         ttm_bo_clean_mm(bdev, 0);
1610 out_no_sys:
1611         return ret;
1612 }
1613
1614 /*
1615  * buffer object vm functions.
1616  */
1617
1618 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1619 {
1620         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1621
1622         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1623                 if (mem->mem_type == TTM_PL_SYSTEM)
1624                         return false;
1625
1626                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1627                         return false;
1628
1629                 if (mem->placement & TTM_PL_FLAG_CACHED)
1630                         return false;
1631         }
1632         return true;
1633 }
1634
1635 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1636 {
1637
1638         ttm_bo_release_mmap(bo);
1639         ttm_mem_io_free_vm(bo);
1640 }
1641
1642 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1643 {
1644         struct ttm_bo_device *bdev = bo->bdev;
1645         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1646
1647         ttm_mem_io_lock(man, false);
1648         ttm_bo_unmap_virtual_locked(bo);
1649         ttm_mem_io_unlock(man);
1650 }
1651
1652 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
1653 {
1654         struct ttm_bo_device *bdev = bo->bdev;
1655
1656         /* The caller acquired bdev->vm_lock. */
1657         RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo);
1658 }
1659
1660 /**
1661  * ttm_bo_setup_vm:
1662  *
1663  * @bo: the buffer to allocate address space for
1664  *
1665  * Allocate address space in the drm device so that applications
1666  * can mmap the buffer and access the contents. This only
1667  * applies to ttm_bo_type_device objects as others are not
1668  * placed in the drm device address space.
1669  */
1670
1671 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo)
1672 {
1673         struct ttm_bo_device *bdev = bo->bdev;
1674         int ret;
1675
1676 retry_pre_get:
1677         ret = drm_mm_pre_get(&bdev->addr_space_mm);
1678         if (unlikely(ret != 0))
1679                 return ret;
1680
1681         lockmgr(&bdev->vm_lock, LK_EXCLUSIVE);
1682         bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
1683                                          bo->mem.num_pages, 0, 0);
1684
1685         if (unlikely(bo->vm_node == NULL)) {
1686                 ret = -ENOMEM;
1687                 goto out_unlock;
1688         }
1689
1690         bo->vm_node = drm_mm_get_block_atomic(bo->vm_node,
1691                                               bo->mem.num_pages, 0);
1692
1693         if (unlikely(bo->vm_node == NULL)) {
1694                 lockmgr(&bdev->vm_lock, LK_RELEASE);
1695                 goto retry_pre_get;
1696         }
1697
1698         ttm_bo_vm_insert_rb(bo);
1699         lockmgr(&bdev->vm_lock, LK_RELEASE);
1700         bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT;
1701
1702         return 0;
1703 out_unlock:
1704         lockmgr(&bdev->vm_lock, LK_RELEASE);
1705         return ret;
1706 }
1707
1708 int ttm_bo_wait(struct ttm_buffer_object *bo,
1709                 bool lazy, bool interruptible, bool no_wait)
1710 {
1711         struct ttm_bo_driver *driver = bo->bdev->driver;
1712         struct ttm_bo_device *bdev = bo->bdev;
1713         void *sync_obj;
1714         int ret = 0;
1715
1716         if (likely(bo->sync_obj == NULL))
1717                 return 0;
1718
1719         while (bo->sync_obj) {
1720
1721                 if (driver->sync_obj_signaled(bo->sync_obj)) {
1722                         void *tmp_obj = bo->sync_obj;
1723                         bo->sync_obj = NULL;
1724                         clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1725                         lockmgr(&bdev->fence_lock, LK_RELEASE);
1726                         driver->sync_obj_unref(&tmp_obj);
1727                         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1728                         continue;
1729                 }
1730
1731                 if (no_wait)
1732                         return -EBUSY;
1733
1734                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
1735                 lockmgr(&bdev->fence_lock, LK_RELEASE);
1736                 ret = driver->sync_obj_wait(sync_obj,
1737                                             lazy, interruptible);
1738                 if (unlikely(ret != 0)) {
1739                         driver->sync_obj_unref(&sync_obj);
1740                         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1741                         return ret;
1742                 }
1743                 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1744                 if (likely(bo->sync_obj == sync_obj)) {
1745                         void *tmp_obj = bo->sync_obj;
1746                         bo->sync_obj = NULL;
1747                         clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1748                                   &bo->priv_flags);
1749                         lockmgr(&bdev->fence_lock, LK_RELEASE);
1750                         driver->sync_obj_unref(&sync_obj);
1751                         driver->sync_obj_unref(&tmp_obj);
1752                         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1753                 } else {
1754                         lockmgr(&bdev->fence_lock, LK_RELEASE);
1755                         driver->sync_obj_unref(&sync_obj);
1756                         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1757                 }
1758         }
1759         return 0;
1760 }
1761
1762 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1763 {
1764         struct ttm_bo_device *bdev = bo->bdev;
1765         int ret = 0;
1766
1767         /*
1768          * Using ttm_bo_reserve makes sure the lru lists are updated.
1769          */
1770
1771         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1772         if (unlikely(ret != 0))
1773                 return ret;
1774         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
1775         ret = ttm_bo_wait(bo, false, true, no_wait);
1776         lockmgr(&bdev->fence_lock, LK_RELEASE);
1777         if (likely(ret == 0))
1778                 atomic_inc(&bo->cpu_writers);
1779         ttm_bo_unreserve(bo);
1780         return ret;
1781 }
1782
1783 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1784 {
1785         atomic_dec(&bo->cpu_writers);
1786 }
1787
1788 /**
1789  * A buffer object shrink method that tries to swap out the first
1790  * buffer object on the bo_global::swap_lru list.
1791  */
1792
1793 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1794 {
1795         struct ttm_bo_global *glob =
1796             container_of(shrink, struct ttm_bo_global, shrink);
1797         struct ttm_buffer_object *bo;
1798         int ret = -EBUSY;
1799         int put_count;
1800         uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM);
1801
1802         lockmgr(&glob->lru_lock, LK_EXCLUSIVE);
1803         list_for_each_entry(bo, &glob->swap_lru, swap) {
1804                 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
1805                 if (!ret)
1806                         break;
1807         }
1808
1809         if (ret) {
1810                 lockmgr(&glob->lru_lock, LK_RELEASE);
1811                 return ret;
1812         }
1813
1814         refcount_acquire(&bo->list_kref);
1815
1816         if (!list_empty(&bo->ddestroy)) {
1817                 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false);
1818                 if (refcount_release(&bo->list_kref))
1819                         ttm_bo_release_list(bo);
1820                 return ret;
1821         }
1822
1823         put_count = ttm_bo_del_from_lru(bo);
1824         lockmgr(&glob->lru_lock, LK_RELEASE);
1825
1826         ttm_bo_list_ref_sub(bo, put_count, true);
1827
1828         /**
1829          * Wait for GPU, then move to system cached.
1830          */
1831
1832         lockmgr(&bo->bdev->fence_lock, LK_EXCLUSIVE);
1833         ret = ttm_bo_wait(bo, false, false, false);
1834         lockmgr(&bo->bdev->fence_lock, LK_RELEASE);
1835
1836         if (unlikely(ret != 0))
1837                 goto out;
1838
1839         if ((bo->mem.placement & swap_placement) != swap_placement) {
1840                 struct ttm_mem_reg evict_mem;
1841
1842                 evict_mem = bo->mem;
1843                 evict_mem.mm_node = NULL;
1844                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1845                 evict_mem.mem_type = TTM_PL_SYSTEM;
1846
1847                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
1848                                              false, false);
1849                 if (unlikely(ret != 0))
1850                         goto out;
1851         }
1852
1853         ttm_bo_unmap_virtual(bo);
1854
1855         /**
1856          * Swap out. Buffer will be swapped in again as soon as
1857          * anyone tries to access a ttm page.
1858          */
1859
1860         if (bo->bdev->driver->swap_notify)
1861                 bo->bdev->driver->swap_notify(bo);
1862
1863         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1864 out:
1865
1866         /**
1867          *
1868          * Unreserve without putting on LRU to avoid swapping out an
1869          * already swapped buffer.
1870          */
1871
1872         atomic_set(&bo->reserved, 0);
1873         wakeup(bo);
1874         if (refcount_release(&bo->list_kref))
1875                 ttm_bo_release_list(bo);
1876         return ret;
1877 }
1878
1879 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1880 {
1881         while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
1882                 ;
1883 }