sys/dev/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  *
  30  * $FreeBSD: head/sys/dev/drm2/ttm/ttm_bo_util.c 248666 2013-03-23 22:23:15Z kib $
  31  */
  32
  33 #include <drm/drmP.h>
  34 #include <dev/drm/ttm/ttm_bo_driver.h>
  35 #include <dev/drm/ttm/ttm_placement.h>
  36 #include <sys/sfbuf.h>
  37
  38 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  39 {
  40         ttm_bo_mem_put(bo, &bo->mem);
  41 }
  42
  43 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  44                     bool evict,
  45                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  46 {
  47         struct ttm_tt *ttm = bo->ttm;
  48         struct ttm_mem_reg *old_mem = &bo->mem;
  49         int ret;
  50
  51         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  52                 ttm_tt_unbind(ttm);
  53                 ttm_bo_free_old_node(bo);
  54                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  55                                 TTM_PL_MASK_MEM);
  56                 old_mem->mem_type = TTM_PL_SYSTEM;
  57         }
  58
  59         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  60         if (unlikely(ret != 0))
  61                 return ret;
  62
  63         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  64                 ret = ttm_tt_bind(ttm, new_mem);
  65                 if (unlikely(ret != 0))
  66                         return ret;
  67         }
  68
  69         *old_mem = *new_mem;
  70         new_mem->mm_node = NULL;
  71
  72         return 0;
  73 }
  74
  75 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
  76 {
  77         if (likely(man->io_reserve_fastpath))
  78                 return 0;
  79
  80         if (interruptible) {
  81                 if (lockmgr(&man->io_reserve_mutex,
  82                             LK_EXCLUSIVE | LK_SLEEPFAIL))
  83                         return (-EINTR);
  84                 else
  85                         return (0);
  86         }
  87
  88         lockmgr(&man->io_reserve_mutex, LK_EXCLUSIVE);
  89         return 0;
  90 }
  91
  92 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
  93 {
  94         if (likely(man->io_reserve_fastpath))
  95                 return;
  96
  97         lockmgr(&man->io_reserve_mutex, LK_RELEASE);
  98 }
  99
 100 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
 101 {
 102         struct ttm_buffer_object *bo;
 103
 104         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
 105                 return -EAGAIN;
 106
 107         bo = list_first_entry(&man->io_reserve_lru,
 108                               struct ttm_buffer_object,
 109                               io_reserve_lru);
 110         list_del_init(&bo->io_reserve_lru);
 111         ttm_bo_unmap_virtual_locked(bo);
 112
 113         return 0;
 114 }
 115
 116 static int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
 117                               struct ttm_mem_reg *mem)
 118 {
 119         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 120         int ret = 0;
 121
 122         if (!bdev->driver->io_mem_reserve)
 123                 return 0;
 124         if (likely(man->io_reserve_fastpath))
 125                 return bdev->driver->io_mem_reserve(bdev, mem);
 126
 127         if (bdev->driver->io_mem_reserve &&
 128             mem->bus.io_reserved_count++ == 0) {
 129 retry:
 130                 ret = bdev->driver->io_mem_reserve(bdev, mem);
 131                 if (ret == -EAGAIN) {
 132                         ret = ttm_mem_io_evict(man);
 133                         if (ret == 0)
 134                                 goto retry;
 135                 }
 136         }
 137         return ret;
 138 }
 139
 140 static void ttm_mem_io_free(struct ttm_bo_device *bdev,
 141                             struct ttm_mem_reg *mem)
 142 {
 143         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 144
 145         if (likely(man->io_reserve_fastpath))
 146                 return;
 147
 148         if (bdev->driver->io_mem_reserve &&
 149             --mem->bus.io_reserved_count == 0 &&
 150             bdev->driver->io_mem_free)
 151                 bdev->driver->io_mem_free(bdev, mem);
 152
 153 }
 154
 155 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
 156 {
 157         struct ttm_mem_reg *mem = &bo->mem;
 158         int ret;
 159
 160         if (!mem->bus.io_reserved_vm) {
 161                 struct ttm_mem_type_manager *man =
 162                         &bo->bdev->man[mem->mem_type];
 163
 164                 ret = ttm_mem_io_reserve(bo->bdev, mem);
 165                 if (unlikely(ret != 0))
 166                         return ret;
 167                 mem->bus.io_reserved_vm = true;
 168                 if (man->use_io_reserve_lru)
 169                         list_add_tail(&bo->io_reserve_lru,
 170                                       &man->io_reserve_lru);
 171         }
 172         return 0;
 173 }
 174
 175 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 176 {
 177         struct ttm_mem_reg *mem = &bo->mem;
 178
 179         if (mem->bus.io_reserved_vm) {
 180                 mem->bus.io_reserved_vm = false;
 181                 list_del_init(&bo->io_reserve_lru);
 182                 ttm_mem_io_free(bo->bdev, mem);
 183         }
 184 }
 185
 186 static
 187 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 188                         void **virtual)
 189 {
 190         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 191         int ret;
 192         void *addr;
 193
 194         *virtual = NULL;
 195         (void) ttm_mem_io_lock(man, false);
 196         ret = ttm_mem_io_reserve(bdev, mem);
 197         ttm_mem_io_unlock(man);
 198         if (ret || !mem->bus.is_iomem)
 199                 return ret;
 200
 201         if (mem->bus.addr) {
 202                 addr = mem->bus.addr;
 203         } else {
 204                 addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset,
 205                     mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ?
 206                     VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE);
 207                 if (!addr) {
 208                         (void) ttm_mem_io_lock(man, false);
 209                         ttm_mem_io_free(bdev, mem);
 210                         ttm_mem_io_unlock(man);
 211                         return -ENOMEM;
 212                 }
 213         }
 214         *virtual = addr;
 215         return 0;
 216 }
 217
 218 static
 219 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 220                          void *virtual)
 221 {
 222         struct ttm_mem_type_manager *man;
 223
 224         man = &bdev->man[mem->mem_type];
 225
 226         if (virtual && mem->bus.addr == NULL)
 227                 pmap_unmapdev((vm_offset_t)virtual, mem->bus.size);
 228         (void) ttm_mem_io_lock(man, false);
 229         ttm_mem_io_free(bdev, mem);
 230         ttm_mem_io_unlock(man);
 231 }
 232
 233 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 234 {
 235         uint32_t *dstP =
 236             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 237         uint32_t *srcP =
 238             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 239
 240         int i;
 241         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 242                 /* iowrite32(ioread32(srcP++), dstP++); */
 243                 *dstP++ = *srcP++;
 244         return 0;
 245 }
 246
 247 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 248                                 unsigned long page,
 249                                 vm_memattr_t prot)
 250 {
 251         vm_page_t d = ttm->pages[page];
 252         void *dst;
 253
 254         if (!d)
 255                 return -ENOMEM;
 256
 257         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 258
 259         /* XXXKIB can't sleep ? */
 260         dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot);
 261         if (!dst)
 262                 return -ENOMEM;
 263
 264         memcpy(dst, src, PAGE_SIZE);
 265
 266         pmap_unmapdev((vm_offset_t)dst, PAGE_SIZE);
 267
 268         return 0;
 269 }
 270
 271 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 272                                 unsigned long page,
 273                                 vm_memattr_t prot)
 274 {
 275         vm_page_t s = ttm->pages[page];
 276         void *src;
 277
 278         if (!s)
 279                 return -ENOMEM;
 280
 281         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 282         src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot);
 283         if (!src)
 284                 return -ENOMEM;
 285
 286         memcpy(dst, src, PAGE_SIZE);
 287
 288         pmap_unmapdev((vm_offset_t)src, PAGE_SIZE);
 289
 290         return 0;
 291 }
 292
 293 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 294                        bool evict, bool no_wait_gpu,
 295                        struct ttm_mem_reg *new_mem)
 296 {
 297         struct ttm_bo_device *bdev = bo->bdev;
 298         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 299         struct ttm_tt *ttm = bo->ttm;
 300         struct ttm_mem_reg *old_mem = &bo->mem;
 301         struct ttm_mem_reg old_copy = *old_mem;
 302         void *old_iomap;
 303         void *new_iomap;
 304         int ret;
 305         unsigned long i;
 306         unsigned long page;
 307         unsigned long add = 0;
 308         int dir;
 309
 310         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 311         if (ret)
 312                 return ret;
 313         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 314         if (ret)
 315                 goto out;
 316
 317         if (old_iomap == NULL && new_iomap == NULL)
 318                 goto out2;
 319         if (old_iomap == NULL && ttm == NULL)
 320                 goto out2;
 321
 322         if (ttm->state == tt_unpopulated) {
 323                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
 324                 if (ret) {
 325                         /* if we fail here don't nuke the mm node
 326                          * as the bo still owns it */
 327                         old_copy.mm_node = NULL;
 328                         goto out1;
 329                 }
 330         }
 331
 332         add = 0;
 333         dir = 1;
 334
 335         if ((old_mem->mem_type == new_mem->mem_type) &&
 336             (new_mem->start < old_mem->start + old_mem->size)) {
 337                 dir = -1;
 338                 add = new_mem->num_pages - 1;
 339         }
 340
 341         for (i = 0; i < new_mem->num_pages; ++i) {
 342                 page = i * dir + add;
 343                 if (old_iomap == NULL) {
 344                         vm_memattr_t prot = ttm_io_prot(old_mem->placement);
 345                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 346                                                    prot);
 347                 } else if (new_iomap == NULL) {
 348                         vm_memattr_t prot = ttm_io_prot(new_mem->placement);
 349                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 350                                                    prot);
 351                 } else
 352                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 353                 if (ret) {
 354                         /* failing here, means keep old copy as-is */
 355                         old_copy.mm_node = NULL;
 356                         goto out1;
 357                 }
 358         }
 359         cpu_mfence();
 360 out2:
 361         old_copy = *old_mem;
 362         *old_mem = *new_mem;
 363         new_mem->mm_node = NULL;
 364
 365         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 366                 ttm_tt_unbind(ttm);
 367                 ttm_tt_destroy(ttm);
 368                 bo->ttm = NULL;
 369         }
 370
 371 out1:
 372         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 373 out:
 374         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 375         ttm_bo_mem_put(bo, &old_copy);
 376         return ret;
 377 }
 378
 379 MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects");
 380
 381 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 382 {
 383         drm_free(bo, M_TTM_TRANSF_OBJ);
 384 }
 385
 386 /**
 387  * ttm_buffer_object_transfer
 388  *
 389  * @bo: A pointer to a struct ttm_buffer_object.
 390  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 391  * holding the data of @bo with the old placement.
 392  *
 393  * This is a utility function that may be called after an accelerated move
 394  * has been scheduled. A new buffer object is created as a placeholder for
 395  * the old data while it's being copied. When that buffer object is idle,
 396  * it can be destroyed, releasing the space of the old placement.
 397  * Returns:
 398  * !0: Failure.
 399  */
 400
 401 static int
 402 ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 403     struct ttm_buffer_object **new_obj)
 404 {
 405         struct ttm_buffer_object *fbo;
 406         struct ttm_bo_device *bdev = bo->bdev;
 407         struct ttm_bo_driver *driver = bdev->driver;
 408
 409         fbo = kmalloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_WAITOK);
 410         *fbo = *bo;
 411
 412         /**
 413          * Fix up members that we shouldn't copy directly:
 414          * TODO: Explicit member copy would probably be better here.
 415          */
 416
 417         INIT_LIST_HEAD(&fbo->ddestroy);
 418         INIT_LIST_HEAD(&fbo->lru);
 419         INIT_LIST_HEAD(&fbo->swap);
 420         INIT_LIST_HEAD(&fbo->io_reserve_lru);
 421         fbo->vm_node = NULL;
 422         atomic_set(&fbo->cpu_writers, 0);
 423
 424         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 425         if (bo->sync_obj)
 426                 fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 427         else
 428                 fbo->sync_obj = NULL;
 429         lockmgr(&bdev->fence_lock, LK_RELEASE);
 430         refcount_init(&fbo->list_kref, 1);
 431         refcount_init(&fbo->kref, 1);
 432         fbo->destroy = &ttm_transfered_destroy;
 433         fbo->acc_size = 0;
 434
 435         *new_obj = fbo;
 436         return 0;
 437 }
 438
 439 vm_memattr_t
 440 ttm_io_prot(uint32_t caching_flags)
 441 {
 442 #if defined(__i386__) || defined(__amd64__)
 443         if (caching_flags & TTM_PL_FLAG_WC)
 444                 return (VM_MEMATTR_WRITE_COMBINING);
 445         else
 446                 /*
 447                  * We do not support i386, look at the linux source
 448                  * for the reason of the comment.
 449                  */
 450                 return (VM_MEMATTR_UNCACHEABLE);
 451 #else
 452 #error Port me
 453 #endif
 454 }
 455
 456 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 457                           unsigned long offset,
 458                           unsigned long size,
 459                           struct ttm_bo_kmap_obj *map)
 460 {
 461         struct ttm_mem_reg *mem = &bo->mem;
 462
 463         if (bo->mem.bus.addr) {
 464                 map->bo_kmap_type = ttm_bo_map_premapped;
 465                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 466         } else {
 467                 map->bo_kmap_type = ttm_bo_map_iomap;
 468                 map->virtual = pmap_mapdev_attr(bo->mem.bus.base +
 469                     bo->mem.bus.offset + offset, size,
 470                     (mem->placement & TTM_PL_FLAG_WC) ?
 471                     VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE);
 472                 map->size = size;
 473         }
 474         return (!map->virtual) ? -ENOMEM : 0;
 475 }
 476
 477 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 478                            unsigned long start_page,
 479                            unsigned long num_pages,
 480                            struct ttm_bo_kmap_obj *map)
 481 {
 482         struct ttm_mem_reg *mem = &bo->mem;
 483         vm_memattr_t prot;
 484         struct ttm_tt *ttm = bo->ttm;
 485         int i, ret;
 486
 487         KKASSERT(ttm != NULL);
 488
 489         if (ttm->state == tt_unpopulated) {
 490                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
 491                 if (ret)
 492                         return ret;
 493         }
 494
 495         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 496                 /*
 497                  * We're mapping a single page, and the desired
 498                  * page protection is consistent with the bo.
 499                  */
 500
 501                 map->bo_kmap_type = ttm_bo_map_kmap;
 502                 map->page = ttm->pages[start_page];
 503                 map->sf = sf_buf_alloc(map->page);
 504                 map->virtual = (void *)sf_buf_kva(map->sf);
 505         } else {
 506                 /*
 507                  * We need to use vmap to get the desired page protection
 508                  * or to make the buffer object look contiguous.
 509                  */
 510                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 511                         VM_MEMATTR_WRITE_COMBINING :
 512                         ttm_io_prot(mem->placement);
 513                 map->bo_kmap_type = ttm_bo_map_vmap;
 514                 map->num_pages = num_pages;
 515                 map->virtual = (void *)kmem_alloc_nofault(&kernel_map,
 516                     num_pages * PAGE_SIZE, PAGE_SIZE);
 517                 if (map->virtual != NULL) {
 518                         for (i = 0; i < num_pages; i++) {
 519                                 /* XXXKIB hack */
 520                                 pmap_page_set_memattr(ttm->pages[start_page +
 521                                     i], prot);
 522                         }
 523                         pmap_qenter((vm_offset_t)map->virtual,
 524                             &ttm->pages[start_page], num_pages);
 525                 }
 526         }
 527         return (!map->virtual) ? -ENOMEM : 0;
 528 }
 529
 530 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 531                 unsigned long start_page, unsigned long num_pages,
 532                 struct ttm_bo_kmap_obj *map)
 533 {
 534         struct ttm_mem_type_manager *man =
 535                 &bo->bdev->man[bo->mem.mem_type];
 536         unsigned long offset, size;
 537         int ret;
 538
 539         KKASSERT(list_empty(&bo->swap));
 540         map->virtual = NULL;
 541         map->bo = bo;
 542         if (num_pages > bo->num_pages)
 543                 return -EINVAL;
 544         if (start_page > bo->num_pages)
 545                 return -EINVAL;
 546 #if 0
 547         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 548                 return -EPERM;
 549 #endif
 550         (void) ttm_mem_io_lock(man, false);
 551         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 552         ttm_mem_io_unlock(man);
 553         if (ret)
 554                 return ret;
 555         if (!bo->mem.bus.is_iomem) {
 556                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 557         } else {
 558                 offset = start_page << PAGE_SHIFT;
 559                 size = num_pages << PAGE_SHIFT;
 560                 return ttm_bo_ioremap(bo, offset, size, map);
 561         }
 562 }
 563
 564 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 565 {
 566         struct ttm_buffer_object *bo = map->bo;
 567         struct ttm_mem_type_manager *man =
 568                 &bo->bdev->man[bo->mem.mem_type];
 569
 570         if (!map->virtual)
 571                 return;
 572         switch (map->bo_kmap_type) {
 573         case ttm_bo_map_iomap:
 574                 pmap_unmapdev((vm_offset_t)map->virtual, map->size);
 575                 break;
 576         case ttm_bo_map_vmap:
 577                 pmap_qremove((vm_offset_t)(map->virtual), map->num_pages);
 578                 kmem_free(&kernel_map, (vm_offset_t)map->virtual,
 579                     map->num_pages * PAGE_SIZE);
 580                 break;
 581         case ttm_bo_map_kmap:
 582                 sf_buf_free(map->sf);
 583                 break;
 584         case ttm_bo_map_premapped:
 585                 break;
 586         default:
 587                 KKASSERT(0);
 588         }
 589         (void) ttm_mem_io_lock(man, false);
 590         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 591         ttm_mem_io_unlock(man);
 592         map->virtual = NULL;
 593         map->page = NULL;
 594         map->sf = NULL;
 595 }
 596
 597 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 598                               void *sync_obj,
 599                               bool evict,
 600                               bool no_wait_gpu,
 601                               struct ttm_mem_reg *new_mem)
 602 {
 603         struct ttm_bo_device *bdev = bo->bdev;
 604         struct ttm_bo_driver *driver = bdev->driver;
 605         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 606         struct ttm_mem_reg *old_mem = &bo->mem;
 607         int ret;
 608         struct ttm_buffer_object *ghost_obj;
 609         void *tmp_obj = NULL;
 610
 611         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 612         if (bo->sync_obj) {
 613                 tmp_obj = bo->sync_obj;
 614                 bo->sync_obj = NULL;
 615         }
 616         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 617         if (evict) {
 618                 ret = ttm_bo_wait(bo, false, false, false);
 619                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 620                 if (tmp_obj)
 621                         driver->sync_obj_unref(&tmp_obj);
 622                 if (ret)
 623                         return ret;
 624
 625                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 626                     (bo->ttm != NULL)) {
 627                         ttm_tt_unbind(bo->ttm);
 628                         ttm_tt_destroy(bo->ttm);
 629                         bo->ttm = NULL;
 630                 }
 631                 ttm_bo_free_old_node(bo);
 632         } else {
 633                 /**
 634                  * This should help pipeline ordinary buffer moves.
 635                  *
 636                  * Hang old buffer memory on a new buffer object,
 637                  * and leave it to be released when the GPU
 638                  * operation has completed.
 639                  */
 640
 641                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 642                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 643                 if (tmp_obj)
 644                         driver->sync_obj_unref(&tmp_obj);
 645
 646                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 647                 if (ret)
 648                         return ret;
 649
 650                 /**
 651                  * If we're not moving to fixed memory, the TTM object
 652                  * needs to stay alive. Otherwhise hang it on the ghost
 653                  * bo to be unbound and destroyed.
 654                  */
 655
 656                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 657                         ghost_obj->ttm = NULL;
 658                 else
 659                         bo->ttm = NULL;
 660
 661                 ttm_bo_unreserve(ghost_obj);
 662                 ttm_bo_unref(&ghost_obj);
 663         }
 664
 665         *old_mem = *new_mem;
 666         new_mem->mm_node = NULL;
 667
 668         return 0;
 669 }