sys/dev/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include <drm/ttm/ttm_bo_driver.h>
  32 #include <drm/ttm/ttm_placement.h>
  33 #include <sys/sfbuf.h>
  34 #include <linux/export.h>
  35 #include <linux/wait.h>
  36
  37 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  38 {
  39         ttm_bo_mem_put(bo, &bo->mem);
  40 }
  41
  42 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  43                     bool evict,
  44                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  45 {
  46         struct ttm_tt *ttm = bo->ttm;
  47         struct ttm_mem_reg *old_mem = &bo->mem;
  48         int ret;
  49
  50         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  51                 ttm_tt_unbind(ttm);
  52                 ttm_bo_free_old_node(bo);
  53                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  54                                 TTM_PL_MASK_MEM);
  55                 old_mem->mem_type = TTM_PL_SYSTEM;
  56         }
  57
  58         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  59         if (unlikely(ret != 0))
  60                 return ret;
  61
  62         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  63                 ret = ttm_tt_bind(ttm, new_mem);
  64                 if (unlikely(ret != 0))
  65                         return ret;
  66         }
  67
  68         *old_mem = *new_mem;
  69         new_mem->mm_node = NULL;
  70
  71         return 0;
  72 }
  73 EXPORT_SYMBOL(ttm_bo_move_ttm);
  74
  75 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
  76 {
  77         if (likely(man->io_reserve_fastpath))
  78                 return 0;
  79
  80         if (interruptible) {
  81                 if (lockmgr(&man->io_reserve_mutex,
  82                             LK_EXCLUSIVE | LK_SLEEPFAIL))
  83                         return (-EINTR);
  84                 else
  85                         return (0);
  86         }
  87
  88         lockmgr(&man->io_reserve_mutex, LK_EXCLUSIVE);
  89         return 0;
  90 }
  91
  92 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
  93 {
  94         if (likely(man->io_reserve_fastpath))
  95                 return;
  96
  97         lockmgr(&man->io_reserve_mutex, LK_RELEASE);
  98 }
  99
 100 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
 101 {
 102         struct ttm_buffer_object *bo;
 103
 104         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
 105                 return -EAGAIN;
 106
 107         bo = list_first_entry(&man->io_reserve_lru,
 108                               struct ttm_buffer_object,
 109                               io_reserve_lru);
 110         list_del_init(&bo->io_reserve_lru);
 111         ttm_bo_unmap_virtual_locked(bo);
 112
 113         return 0;
 114 }
 115
 116 static int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
 117                               struct ttm_mem_reg *mem)
 118 {
 119         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 120         int ret = 0;
 121
 122         if (!bdev->driver->io_mem_reserve)
 123                 return 0;
 124         if (likely(man->io_reserve_fastpath))
 125                 return bdev->driver->io_mem_reserve(bdev, mem);
 126
 127         if (bdev->driver->io_mem_reserve &&
 128             mem->bus.io_reserved_count++ == 0) {
 129 retry:
 130                 ret = bdev->driver->io_mem_reserve(bdev, mem);
 131                 if (ret == -EAGAIN) {
 132                         ret = ttm_mem_io_evict(man);
 133                         if (ret == 0)
 134                                 goto retry;
 135                 }
 136         }
 137         return ret;
 138 }
 139
 140 static void ttm_mem_io_free(struct ttm_bo_device *bdev,
 141                             struct ttm_mem_reg *mem)
 142 {
 143         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 144
 145         if (likely(man->io_reserve_fastpath))
 146                 return;
 147
 148         if (bdev->driver->io_mem_reserve &&
 149             --mem->bus.io_reserved_count == 0 &&
 150             bdev->driver->io_mem_free)
 151                 bdev->driver->io_mem_free(bdev, mem);
 152
 153 }
 154
 155 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
 156 {
 157         struct ttm_mem_reg *mem = &bo->mem;
 158         int ret;
 159
 160         if (!mem->bus.io_reserved_vm) {
 161                 struct ttm_mem_type_manager *man =
 162                         &bo->bdev->man[mem->mem_type];
 163
 164                 ret = ttm_mem_io_reserve(bo->bdev, mem);
 165                 if (unlikely(ret != 0))
 166                         return ret;
 167                 mem->bus.io_reserved_vm = true;
 168                 if (man->use_io_reserve_lru)
 169                         list_add_tail(&bo->io_reserve_lru,
 170                                       &man->io_reserve_lru);
 171         }
 172         return 0;
 173 }
 174
 175 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 176 {
 177         struct ttm_mem_reg *mem = &bo->mem;
 178
 179         if (mem->bus.io_reserved_vm) {
 180                 mem->bus.io_reserved_vm = false;
 181                 list_del_init(&bo->io_reserve_lru);
 182                 ttm_mem_io_free(bo->bdev, mem);
 183         }
 184 }
 185
 186 static
 187 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 188                         void **virtual)
 189 {
 190         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 191         int ret;
 192         void *addr;
 193
 194         *virtual = NULL;
 195         (void) ttm_mem_io_lock(man, false);
 196         ret = ttm_mem_io_reserve(bdev, mem);
 197         ttm_mem_io_unlock(man);
 198         if (ret || !mem->bus.is_iomem)
 199                 return ret;
 200
 201         if (mem->bus.addr) {
 202                 addr = mem->bus.addr;
 203         } else {
 204                 addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset,
 205                     mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ?
 206                     VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE);
 207                 if (!addr) {
 208                         (void) ttm_mem_io_lock(man, false);
 209                         ttm_mem_io_free(bdev, mem);
 210                         ttm_mem_io_unlock(man);
 211                         return -ENOMEM;
 212                 }
 213         }
 214         *virtual = addr;
 215         return 0;
 216 }
 217
 218 static
 219 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 220                          void *virtual)
 221 {
 222         struct ttm_mem_type_manager *man;
 223
 224         man = &bdev->man[mem->mem_type];
 225
 226         if (virtual && mem->bus.addr == NULL)
 227                 pmap_unmapdev((vm_offset_t)virtual, mem->bus.size);
 228         (void) ttm_mem_io_lock(man, false);
 229         ttm_mem_io_free(bdev, mem);
 230         ttm_mem_io_unlock(man);
 231 }
 232
 233 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 234 {
 235         uint32_t *dstP =
 236             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 237         uint32_t *srcP =
 238             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 239
 240         int i;
 241         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 242                 /* iowrite32(ioread32(srcP++), dstP++); */
 243                 *dstP++ = *srcP++;
 244         return 0;
 245 }
 246
 247 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 248                                 unsigned long page,
 249                                 vm_memattr_t prot)
 250 {
 251         vm_page_t d = ttm->pages[page];
 252         void *dst;
 253
 254         if (!d)
 255                 return -ENOMEM;
 256
 257         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 258
 259         /* XXXKIB can't sleep ? */
 260         dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot);
 261         if (!dst)
 262                 return -ENOMEM;
 263
 264         memcpy_fromio(dst, src, PAGE_SIZE);
 265
 266         pmap_unmapdev((vm_offset_t)dst, PAGE_SIZE);
 267
 268         return 0;
 269 }
 270
 271 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 272                                 unsigned long page,
 273                                 vm_memattr_t prot)
 274 {
 275         vm_page_t s = ttm->pages[page];
 276         void *src;
 277
 278         if (!s)
 279                 return -ENOMEM;
 280
 281         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 282         src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot);
 283         if (!src)
 284                 return -ENOMEM;
 285
 286         memcpy_toio(dst, src, PAGE_SIZE);
 287
 288         pmap_unmapdev((vm_offset_t)src, PAGE_SIZE);
 289
 290         return 0;
 291 }
 292
 293 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 294                        bool evict, bool no_wait_gpu,
 295                        struct ttm_mem_reg *new_mem)
 296 {
 297         struct ttm_bo_device *bdev = bo->bdev;
 298         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 299         struct ttm_tt *ttm = bo->ttm;
 300         struct ttm_mem_reg *old_mem = &bo->mem;
 301         struct ttm_mem_reg old_copy = *old_mem;
 302         void *old_iomap;
 303         void *new_iomap;
 304         int ret;
 305         unsigned long i;
 306         unsigned long page;
 307         unsigned long add = 0;
 308         int dir;
 309
 310         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 311         if (ret)
 312                 return ret;
 313         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 314         if (ret)
 315                 goto out;
 316
 317         if (old_iomap == NULL && new_iomap == NULL)
 318                 goto out2;
 319         if (old_iomap == NULL && ttm == NULL)
 320                 goto out2;
 321
 322         if (ttm->state == tt_unpopulated) {
 323                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
 324                 if (ret) {
 325                         /* if we fail here don't nuke the mm node
 326                          * as the bo still owns it */
 327                         old_copy.mm_node = NULL;
 328                         goto out1;
 329                 }
 330         }
 331
 332         add = 0;
 333         dir = 1;
 334
 335         if ((old_mem->mem_type == new_mem->mem_type) &&
 336             (new_mem->start < old_mem->start + old_mem->size)) {
 337                 dir = -1;
 338                 add = new_mem->num_pages - 1;
 339         }
 340
 341         for (i = 0; i < new_mem->num_pages; ++i) {
 342                 page = i * dir + add;
 343                 if (old_iomap == NULL) {
 344                         vm_memattr_t prot = ttm_io_prot(old_mem->placement);
 345                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 346                                                    prot);
 347                 } else if (new_iomap == NULL) {
 348                         vm_memattr_t prot = ttm_io_prot(new_mem->placement);
 349                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 350                                                    prot);
 351                 } else
 352                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 353                 if (ret) {
 354                         /* failing here, means keep old copy as-is */
 355                         old_copy.mm_node = NULL;
 356                         goto out1;
 357                 }
 358         }
 359         cpu_mfence();
 360 out2:
 361         old_copy = *old_mem;
 362         *old_mem = *new_mem;
 363         new_mem->mm_node = NULL;
 364
 365         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 366                 ttm_tt_unbind(ttm);
 367                 ttm_tt_destroy(ttm);
 368                 bo->ttm = NULL;
 369         }
 370
 371 out1:
 372         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 373 out:
 374         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 375         ttm_bo_mem_put(bo, &old_copy);
 376         return ret;
 377 }
 378 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 379
 380 MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects");
 381
 382 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 383 {
 384         kfree(bo, M_TTM_TRANSF_OBJ);
 385 }
 386
 387 /**
 388  * ttm_buffer_object_transfer
 389  *
 390  * @bo: A pointer to a struct ttm_buffer_object.
 391  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 392  * holding the data of @bo with the old placement.
 393  *
 394  * This is a utility function that may be called after an accelerated move
 395  * has been scheduled. A new buffer object is created as a placeholder for
 396  * the old data while it's being copied. When that buffer object is idle,
 397  * it can be destroyed, releasing the space of the old placement.
 398  * Returns:
 399  * !0: Failure.
 400  */
 401
 402 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 403                                       struct ttm_buffer_object **new_obj)
 404 {
 405         struct ttm_buffer_object *fbo;
 406         struct ttm_bo_device *bdev = bo->bdev;
 407         struct ttm_bo_driver *driver = bdev->driver;
 408
 409         fbo = kmalloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_WAITOK | M_ZERO);
 410         if (!fbo)
 411                 return -ENOMEM;
 412
 413         *fbo = *bo;
 414
 415         /**
 416          * Fix up members that we shouldn't copy directly:
 417          * TODO: Explicit member copy would probably be better here.
 418          */
 419
 420         init_waitqueue_head(&fbo->event_queue);
 421         INIT_LIST_HEAD(&fbo->ddestroy);
 422         INIT_LIST_HEAD(&fbo->lru);
 423         INIT_LIST_HEAD(&fbo->swap);
 424         INIT_LIST_HEAD(&fbo->io_reserve_lru);
 425         fbo->vm_node = NULL;
 426         atomic_set(&fbo->cpu_writers, 0);
 427
 428         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 429         if (bo->sync_obj)
 430                 fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 431         else
 432                 fbo->sync_obj = NULL;
 433         lockmgr(&bdev->fence_lock, LK_RELEASE);
 434         kref_init(&fbo->list_kref);
 435         kref_init(&fbo->kref);
 436         fbo->destroy = &ttm_transfered_destroy;
 437         fbo->acc_size = 0;
 438
 439         /*
 440          * Mirror ref from kref_init() for list_kref.
 441          */
 442         set_bit(TTM_BO_PRIV_FLAG_ACTIVE, &fbo->priv_flags);
 443
 444         *new_obj = fbo;
 445         return 0;
 446 }
 447
 448 vm_memattr_t
 449 ttm_io_prot(uint32_t caching_flags)
 450 {
 451 #if defined(__i386__) || defined(__x86_64__)
 452         if (caching_flags & TTM_PL_FLAG_WC)
 453                 return (VM_MEMATTR_WRITE_COMBINING);
 454         else
 455                 /*
 456                  * We do not support i386, look at the linux source
 457                  * for the reason of the comment.
 458                  */
 459                 return (VM_MEMATTR_UNCACHEABLE);
 460 #else
 461 #error Port me
 462 #endif
 463 }
 464 EXPORT_SYMBOL(ttm_io_prot);
 465
 466 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 467                           unsigned long offset,
 468                           unsigned long size,
 469                           struct ttm_bo_kmap_obj *map)
 470 {
 471         struct ttm_mem_reg *mem = &bo->mem;
 472
 473         if (bo->mem.bus.addr) {
 474                 map->bo_kmap_type = ttm_bo_map_premapped;
 475                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 476         } else {
 477                 map->bo_kmap_type = ttm_bo_map_iomap;
 478                 map->virtual = pmap_mapdev_attr(bo->mem.bus.base +
 479                     bo->mem.bus.offset + offset, size,
 480                     (mem->placement & TTM_PL_FLAG_WC) ?
 481                     VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE);
 482                 map->size = size;
 483         }
 484         return (!map->virtual) ? -ENOMEM : 0;
 485 }
 486
 487 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 488                            unsigned long start_page,
 489                            unsigned long num_pages,
 490                            struct ttm_bo_kmap_obj *map)
 491 {
 492         struct ttm_mem_reg *mem = &bo->mem;
 493         vm_memattr_t prot;
 494         struct ttm_tt *ttm = bo->ttm;
 495         int i, ret;
 496
 497         BUG_ON(!ttm);
 498
 499         if (ttm->state == tt_unpopulated) {
 500                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
 501                 if (ret)
 502                         return ret;
 503         }
 504
 505         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 506                 /*
 507                  * We're mapping a single page, and the desired
 508                  * page protection is consistent with the bo.
 509                  */
 510
 511                 map->bo_kmap_type = ttm_bo_map_kmap;
 512                 map->page = ttm->pages[start_page];
 513                 map->sf = sf_buf_alloc(map->page);
 514                 map->virtual = (void *)sf_buf_kva(map->sf);
 515         } else {
 516                 /*
 517                  * We need to use vmap to get the desired page protection
 518                  * or to make the buffer object look contiguous.
 519                  */
 520                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 521                         VM_MEMATTR_WRITE_COMBINING :
 522                         ttm_io_prot(mem->placement);
 523                 map->bo_kmap_type = ttm_bo_map_vmap;
 524                 map->num_pages = num_pages;
 525                 map->virtual = (void *)kmem_alloc_nofault(&kernel_map,
 526                     num_pages * PAGE_SIZE, PAGE_SIZE);
 527                 if (map->virtual != NULL) {
 528                         for (i = 0; i < num_pages; i++) {
 529                                 /* XXXKIB hack */
 530                                 pmap_page_set_memattr(ttm->pages[start_page +
 531                                     i], prot);
 532                         }
 533                         pmap_qenter((vm_offset_t)map->virtual,
 534                             &ttm->pages[start_page], num_pages);
 535                 }
 536         }
 537         return (!map->virtual) ? -ENOMEM : 0;
 538 }
 539
 540 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 541                 unsigned long start_page, unsigned long num_pages,
 542                 struct ttm_bo_kmap_obj *map)
 543 {
 544         struct ttm_mem_type_manager *man =
 545                 &bo->bdev->man[bo->mem.mem_type];
 546         unsigned long offset, size;
 547         int ret;
 548
 549         BUG_ON(!list_empty(&bo->swap));
 550         map->virtual = NULL;
 551         map->bo = bo;
 552         if (num_pages > bo->num_pages)
 553                 return -EINVAL;
 554         if (start_page > bo->num_pages)
 555                 return -EINVAL;
 556 #if 0
 557         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 558                 return -EPERM;
 559 #endif
 560         (void) ttm_mem_io_lock(man, false);
 561         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 562         ttm_mem_io_unlock(man);
 563         if (ret)
 564                 return ret;
 565         if (!bo->mem.bus.is_iomem) {
 566                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 567         } else {
 568                 offset = start_page << PAGE_SHIFT;
 569                 size = num_pages << PAGE_SHIFT;
 570                 return ttm_bo_ioremap(bo, offset, size, map);
 571         }
 572 }
 573 EXPORT_SYMBOL(ttm_bo_kmap);
 574
 575 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 576 {
 577         struct ttm_buffer_object *bo = map->bo;
 578         struct ttm_mem_type_manager *man =
 579                 &bo->bdev->man[bo->mem.mem_type];
 580
 581         if (!map->virtual)
 582                 return;
 583         switch (map->bo_kmap_type) {
 584         case ttm_bo_map_iomap:
 585                 pmap_unmapdev((vm_offset_t)map->virtual, map->size);
 586                 break;
 587         case ttm_bo_map_vmap:
 588                 pmap_qremove((vm_offset_t)(map->virtual), map->num_pages);
 589                 kmem_free(&kernel_map, (vm_offset_t)map->virtual,
 590                     map->num_pages * PAGE_SIZE);
 591                 break;
 592         case ttm_bo_map_kmap:
 593                 sf_buf_free(map->sf);
 594                 break;
 595         case ttm_bo_map_premapped:
 596                 break;
 597         default:
 598                 BUG();
 599         }
 600         (void) ttm_mem_io_lock(man, false);
 601         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 602         ttm_mem_io_unlock(man);
 603         map->virtual = NULL;
 604         map->page = NULL;
 605         map->sf = NULL;
 606 }
 607 EXPORT_SYMBOL(ttm_bo_kunmap);
 608
 609 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 610                               void *sync_obj,
 611                               bool evict,
 612                               bool no_wait_gpu,
 613                               struct ttm_mem_reg *new_mem)
 614 {
 615         struct ttm_bo_device *bdev = bo->bdev;
 616         struct ttm_bo_driver *driver = bdev->driver;
 617         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 618         struct ttm_mem_reg *old_mem = &bo->mem;
 619         int ret;
 620         struct ttm_buffer_object *ghost_obj;
 621         void *tmp_obj = NULL;
 622
 623         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 624         if (bo->sync_obj) {
 625                 tmp_obj = bo->sync_obj;
 626                 bo->sync_obj = NULL;
 627         }
 628         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 629         if (evict) {
 630                 ret = ttm_bo_wait(bo, false, false, false);
 631                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 632                 if (tmp_obj)
 633                         driver->sync_obj_unref(&tmp_obj);
 634                 if (ret)
 635                         return ret;
 636
 637                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 638                     (bo->ttm != NULL)) {
 639                         ttm_tt_unbind(bo->ttm);
 640                         ttm_tt_destroy(bo->ttm);
 641                         bo->ttm = NULL;
 642                 }
 643                 ttm_bo_free_old_node(bo);
 644         } else {
 645                 /**
 646                  * This should help pipeline ordinary buffer moves.
 647                  *
 648                  * Hang old buffer memory on a new buffer object,
 649                  * and leave it to be released when the GPU
 650                  * operation has completed.
 651                  */
 652
 653                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 654                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 655                 if (tmp_obj)
 656                         driver->sync_obj_unref(&tmp_obj);
 657
 658                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 659                 if (ret)
 660                         return ret;
 661
 662                 /**
 663                  * If we're not moving to fixed memory, the TTM object
 664                  * needs to stay alive. Otherwhise hang it on the ghost
 665                  * bo to be unbound and destroyed.
 666                  */
 667
 668                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 669                         ghost_obj->ttm = NULL;
 670                 else
 671                         bo->ttm = NULL;
 672
 673                 ttm_bo_unreserve(ghost_obj);
 674                 ttm_bo_unref(&ghost_obj);
 675         }
 676
 677         *old_mem = *new_mem;
 678         new_mem->mm_node = NULL;
 679
 680         return 0;
 681 }
 682 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);