sys/dev/drm/ttm/ttm_bo_vm.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #define pr_fmt(fmt) "[TTM] " fmt
  32
  33 #include <ttm/ttm_module.h>
  34 #include <ttm/ttm_bo_driver.h>
  35 #include <ttm/ttm_bo_api.h>
  36 #include <ttm/ttm_placement.h>
  37 #include <drm/drm_vma_manager.h>
  38 #include <linux/mm.h>
  39 #include <linux/rbtree.h>
  40 #include <linux/module.h>
  41 #include <linux/uaccess.h>
  42
  43 #include <sys/sysctl.h>
  44 #include <vm/vm.h>
  45 #include <vm/vm_page.h>
  46 #include <vm/vm_page2.h>
  47
  48 #define TTM_BO_VM_NUM_PREFAULT 16
  49
  50 /*
  51  * Always unstall on unexpected vm_page alias, fatal bus fault.
  52  * Set to 0 to stall, set to positive count to unstall N times,
  53  * then stall again.
  54  */
  55 static int drm_unstall = -1;
  56 SYSCTL_INT(_debug, OID_AUTO, unstall, CTLFLAG_RW, &drm_unstall, 0, "");
  57
  58 static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
  59 {
  60         /* see ttm_bo_mmap_single() at end of this file */
  61         /* ttm_bo_vm_ops not currently used, no entry should occur */
  62         panic("ttm_bo_vm_fault");
  63 #if 0
  64         struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
  65             vma->vm_private_data;
  66         struct ttm_bo_device *bdev = bo->bdev;
  67         unsigned long page_offset;
  68         unsigned long page_last;
  69         unsigned long pfn;
  70         struct ttm_tt *ttm = NULL;
  71         struct page *page;
  72         int ret;
  73         int i;
  74         unsigned long address = (unsigned long)vmf->virtual_address;
  75         int retval = VM_FAULT_NOPAGE;
  76         struct ttm_mem_type_manager *man =
  77                 &bdev->man[bo->mem.mem_type];
  78
  79         /*
  80          * Work around locking order reversal in fault / nopfn
  81          * between mmap_sem and bo_reserve: Perform a trylock operation
  82          * for reserve, and if it fails, retry the fault after scheduling.
  83          */
  84
  85         ret = ttm_bo_reserve(bo, true, true, false, 0);
  86         if (unlikely(ret != 0)) {
  87                 if (ret == -EBUSY)
  88                         set_need_resched();
  89                 return VM_FAULT_NOPAGE;
  90         }
  91
  92         if (bdev->driver->fault_reserve_notify) {
  93                 ret = bdev->driver->fault_reserve_notify(bo);
  94                 switch (ret) {
  95                 case 0:
  96                         break;
  97                 case -EBUSY:
  98                         set_need_resched();
  99                 case -ERESTARTSYS:
 100                         retval = VM_FAULT_NOPAGE;
 101                         goto out_unlock;
 102                 default:
 103                         retval = VM_FAULT_SIGBUS;
 104                         goto out_unlock;
 105                 }
 106         }
 107
 108         /*
 109          * Wait for buffer data in transit, due to a pipelined
 110          * move.
 111          */
 112
 113         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 114         if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
 115                 ret = ttm_bo_wait(bo, false, true, false);
 116                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 117                 if (unlikely(ret != 0)) {
 118                         retval = (ret != -ERESTARTSYS) ?
 119                             VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
 120                         goto out_unlock;
 121                 }
 122         } else
 123                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 124
 125         ret = ttm_mem_io_lock(man, true);
 126         if (unlikely(ret != 0)) {
 127                 retval = VM_FAULT_NOPAGE;
 128                 goto out_unlock;
 129         }
 130         ret = ttm_mem_io_reserve_vm(bo);
 131         if (unlikely(ret != 0)) {
 132                 retval = VM_FAULT_SIGBUS;
 133                 goto out_io_unlock;
 134         }
 135
 136         page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
 137             drm_vma_node_start(&bo->vma_node) - vma->vm_pgoff;
 138         page_last = vma_pages(vma) +
 139             drm_vma_node_start(&bo->vma_node) - vma->vm_pgoff;
 140
 141         if (unlikely(page_offset >= bo->num_pages)) {
 142                 retval = VM_FAULT_SIGBUS;
 143                 goto out_io_unlock;
 144         }
 145
 146         /*
 147          * Strictly, we're not allowed to modify vma->vm_page_prot here,
 148          * since the mmap_sem is only held in read mode. However, we
 149          * modify only the caching bits of vma->vm_page_prot and
 150          * consider those bits protected by
 151          * the bo->mutex, as we should be the only writers.
 152          * There shouldn't really be any readers of these bits except
 153          * within vm_insert_mixed()? fork?
 154          *
 155          * TODO: Add a list of vmas to the bo, and change the
 156          * vma->vm_page_prot when the object changes caching policy, with
 157          * the correct locks held.
 158          */
 159         if (bo->mem.bus.is_iomem) {
 160                 vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
 161                                                 vma->vm_page_prot);
 162         } else {
 163                 ttm = bo->ttm;
 164                 vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
 165                     vm_get_page_prot(vma->vm_flags) :
 166                     ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
 167
 168                 /* Allocate all page at once, most common usage */
 169                 if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
 170                         retval = VM_FAULT_OOM;
 171                         goto out_io_unlock;
 172                 }
 173         }
 174
 175         /*
 176          * Speculatively prefault a number of pages. Only error on
 177          * first page.
 178          */
 179         for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
 180                 if (bo->mem.bus.is_iomem)
 181                         pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset;
 182                 else {
 183                         page = ttm->pages[page_offset];
 184                         if (unlikely(!page && i == 0)) {
 185                                 retval = VM_FAULT_OOM;
 186                                 goto out_io_unlock;
 187                         } else if (unlikely(!page)) {
 188                                 break;
 189                         }
 190                         pfn = page_to_pfn(page);
 191                 }
 192
 193                 ret = vm_insert_mixed(vma, address, pfn);
 194                 /*
 195                  * Somebody beat us to this PTE or prefaulting to
 196                  * an already populated PTE, or prefaulting error.
 197                  */
 198
 199                 if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
 200                         break;
 201                 else if (unlikely(ret != 0)) {
 202                         retval =
 203                             (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
 204                         goto out_io_unlock;
 205                 }
 206
 207                 address += PAGE_SIZE;
 208                 if (unlikely(++page_offset >= page_last))
 209                         break;
 210         }
 211 out_io_unlock:
 212         ttm_mem_io_unlock(man);
 213 out_unlock:
 214         ttm_bo_unreserve(bo);
 215         return retval;
 216 #endif
 217 }
 218
 219 /* ttm_bo_vm_ops not currently used, no entry should occur */
 220 static void ttm_bo_vm_open(struct vm_area_struct *vma)
 221 {
 222         struct ttm_buffer_object *bo =
 223             (struct ttm_buffer_object *)vma->vm_private_data;
 224
 225         (void)ttm_bo_reference(bo);
 226 }
 227
 228 /* ttm_bo_vm_ops not currently used, no entry should occur */
 229 static void ttm_bo_vm_close(struct vm_area_struct *vma)
 230 {
 231         struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data;
 232
 233         ttm_bo_unref(&bo);
 234         vma->vm_private_data = NULL;
 235 }
 236
 237 static const struct vm_operations_struct ttm_bo_vm_ops = {
 238         .fault = ttm_bo_vm_fault,
 239         .open = ttm_bo_vm_open,
 240         .close = ttm_bo_vm_close
 241 };
 242
 243 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
 244                                                   unsigned long offset,
 245                                                   unsigned long pages)
 246 {
 247         struct drm_vma_offset_node *node;
 248         struct ttm_buffer_object *bo = NULL;
 249
 250         drm_vma_offset_lock_lookup(&bdev->vma_manager);
 251
 252         node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages);
 253         if (likely(node)) {
 254                 bo = container_of(node, struct ttm_buffer_object, vma_node);
 255                 if (!kref_get_unless_zero(&bo->kref))
 256                         bo = NULL;
 257         }
 258
 259         drm_vma_offset_unlock_lookup(&bdev->vma_manager);
 260
 261         if (!bo)
 262                 pr_err("Could not find buffer object to map\n");
 263
 264         return bo;
 265 }
 266
 267 int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
 268                 struct ttm_bo_device *bdev)
 269 {
 270         struct ttm_bo_driver *driver;
 271         struct ttm_buffer_object *bo;
 272         int ret;
 273
 274         bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma));
 275         if (unlikely(!bo))
 276                 return -EINVAL;
 277
 278         driver = bo->bdev->driver;
 279         if (unlikely(!driver->verify_access)) {
 280                 ret = -EPERM;
 281                 goto out_unref;
 282         }
 283         ret = driver->verify_access(bo, filp);
 284         if (unlikely(ret != 0))
 285                 goto out_unref;
 286
 287         vma->vm_ops = &ttm_bo_vm_ops;
 288
 289         /*
 290          * Note: We're transferring the bo reference to
 291          * vma->vm_private_data here.
 292          */
 293
 294         vma->vm_private_data = bo;
 295         vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP;
 296         return 0;
 297 out_unref:
 298         ttm_bo_unref(&bo);
 299         return ret;
 300 }
 301 EXPORT_SYMBOL(ttm_bo_mmap);
 302
 303 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
 304 {
 305         if (vma->vm_pgoff != 0)
 306                 return -EACCES;
 307
 308         vma->vm_ops = &ttm_bo_vm_ops;
 309         vma->vm_private_data = ttm_bo_reference(bo);
 310         vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
 311         return 0;
 312 }
 313 EXPORT_SYMBOL(ttm_fbdev_mmap);
 314
 315
 316 ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
 317                   const char __user *wbuf, char __user *rbuf, size_t count,
 318                   loff_t *f_pos, bool write)
 319 {
 320         struct ttm_buffer_object *bo;
 321         struct ttm_bo_driver *driver;
 322         struct ttm_bo_kmap_obj map;
 323         unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
 324         unsigned long kmap_offset;
 325         unsigned long kmap_end;
 326         unsigned long kmap_num;
 327         size_t io_size;
 328         unsigned int page_offset;
 329         char *virtual;
 330         int ret;
 331         bool no_wait = false;
 332         bool dummy;
 333
 334         bo = ttm_bo_vm_lookup(bdev, dev_offset, 1);
 335         if (unlikely(bo == NULL))
 336                 return -EFAULT;
 337
 338         driver = bo->bdev->driver;
 339         if (unlikely(!driver->verify_access)) {
 340                 ret = -EPERM;
 341                 goto out_unref;
 342         }
 343
 344         ret = driver->verify_access(bo, filp);
 345         if (unlikely(ret != 0))
 346                 goto out_unref;
 347
 348         kmap_offset = dev_offset - drm_vma_node_start(&bo->vma_node);
 349         if (unlikely(kmap_offset >= bo->num_pages)) {
 350                 ret = -EFBIG;
 351                 goto out_unref;
 352         }
 353
 354         page_offset = *f_pos & ~PAGE_MASK;
 355         io_size = bo->num_pages - kmap_offset;
 356         io_size = (io_size << PAGE_SHIFT) - page_offset;
 357         if (count < io_size)
 358                 io_size = count;
 359
 360         kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
 361         kmap_num = kmap_end - kmap_offset + 1;
 362
 363         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
 364
 365         switch (ret) {
 366         case 0:
 367                 break;
 368         case -EBUSY:
 369                 ret = -EAGAIN;
 370                 goto out_unref;
 371         default:
 372                 goto out_unref;
 373         }
 374
 375         ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
 376         if (unlikely(ret != 0)) {
 377                 ttm_bo_unreserve(bo);
 378                 goto out_unref;
 379         }
 380
 381         virtual = ttm_kmap_obj_virtual(&map, &dummy);
 382         virtual += page_offset;
 383
 384         if (write)
 385                 ret = copy_from_user(virtual, wbuf, io_size);
 386         else
 387                 ret = copy_to_user(rbuf, virtual, io_size);
 388
 389         ttm_bo_kunmap(&map);
 390         ttm_bo_unreserve(bo);
 391         ttm_bo_unref(&bo);
 392
 393         if (unlikely(ret != 0))
 394                 return -EFBIG;
 395
 396         *f_pos += io_size;
 397
 398         return io_size;
 399 out_unref:
 400         ttm_bo_unref(&bo);
 401         return ret;
 402 }
 403
 404 ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
 405                         char __user *rbuf, size_t count, loff_t *f_pos,
 406                         bool write)
 407 {
 408         struct ttm_bo_kmap_obj map;
 409         unsigned long kmap_offset;
 410         unsigned long kmap_end;
 411         unsigned long kmap_num;
 412         size_t io_size;
 413         unsigned int page_offset;
 414         char *virtual;
 415         int ret;
 416         bool no_wait = false;
 417         bool dummy;
 418
 419         kmap_offset = (*f_pos >> PAGE_SHIFT);
 420         if (unlikely(kmap_offset >= bo->num_pages))
 421                 return -EFBIG;
 422
 423         page_offset = *f_pos & ~PAGE_MASK;
 424         io_size = bo->num_pages - kmap_offset;
 425         io_size = (io_size << PAGE_SHIFT) - page_offset;
 426         if (count < io_size)
 427                 io_size = count;
 428
 429         kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
 430         kmap_num = kmap_end - kmap_offset + 1;
 431
 432         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
 433
 434         switch (ret) {
 435         case 0:
 436                 break;
 437         case -EBUSY:
 438                 return -EAGAIN;
 439         default:
 440                 return ret;
 441         }
 442
 443         ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
 444         if (unlikely(ret != 0)) {
 445                 ttm_bo_unreserve(bo);
 446                 return ret;
 447         }
 448
 449         virtual = ttm_kmap_obj_virtual(&map, &dummy);
 450         virtual += page_offset;
 451
 452         if (write)
 453                 ret = copy_from_user(virtual, wbuf, io_size);
 454         else
 455                 ret = copy_to_user(rbuf, virtual, io_size);
 456
 457         ttm_bo_kunmap(&map);
 458         ttm_bo_unreserve(bo);
 459         ttm_bo_unref(&bo);
 460
 461         if (unlikely(ret != 0))
 462                 return ret;
 463
 464         *f_pos += io_size;
 465
 466         return io_size;
 467 }
 468
 469 /*
 470  * DragonFlyBSD Interface
 471  */
 472
 473 #include "opt_vm.h"
 474
 475 #include <vm/vm.h>
 476 #include <vm/vm_page.h>
 477 #include <linux/errno.h>
 478 #include <linux/export.h>
 479
 480 #include <vm/vm_page2.h>
 481
 482 static int
 483 ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset,
 484                      int prot, vm_page_t *mres)
 485 {
 486         struct ttm_buffer_object *bo = vm_obj->handle;
 487         struct ttm_bo_device *bdev = bo->bdev;
 488         struct ttm_tt *ttm = NULL;
 489         vm_page_t m, oldm;
 490         int ret;
 491         int retval = VM_PAGER_OK;
 492         struct ttm_mem_type_manager *man;
 493
 494         man = &bdev->man[bo->mem.mem_type];
 495
 496         /*kprintf("FAULT %p %p/%ld\n", vm_obj, bo, offset);*/
 497
 498         vm_object_pip_add(vm_obj, 1);
 499         oldm = *mres;
 500         *mres = NULL;
 501
 502 retry:
 503         VM_OBJECT_UNLOCK(vm_obj);
 504         m = NULL;
 505
 506         /*
 507          * NOTE: set nowait to false, we don't have ttm_bo_wait_unreserved()
 508          *       for the -BUSY case yet.
 509          */
 510         ret = ttm_bo_reserve(bo, true, false, false, 0);
 511         if (unlikely(ret != 0)) {
 512                 retval = VM_PAGER_ERROR;
 513                 VM_OBJECT_LOCK(vm_obj);
 514                 goto out_unlock2;
 515         }
 516
 517         if (bdev->driver->fault_reserve_notify) {
 518                 ret = bdev->driver->fault_reserve_notify(bo);
 519                 switch (ret) {
 520                 case 0:
 521                         break;
 522                 case -EBUSY:
 523                         lwkt_yield();
 524                         /* fall through */
 525                 case -ERESTARTSYS:
 526                 case -EINTR:
 527                         retval = VM_PAGER_ERROR;
 528                         goto out_unlock;
 529                 default:
 530                         retval = VM_PAGER_ERROR;
 531                         goto out_unlock;
 532                 }
 533         }
 534
 535         /*
 536          * Wait for buffer data in transit, due to a pipelined
 537          * move.
 538          */
 539         lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
 540         if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
 541                 /*
 542                  * Here, the behavior differs between Linux and FreeBSD.
 543                  *
 544                  * On Linux, the wait is interruptible (3rd argument to
 545                  * ttm_bo_wait). There must be some mechanism to resume
 546                  * page fault handling, once the signal is processed.
 547                  *
 548                  * On FreeBSD, the wait is uninteruptible. This is not a
 549                  * problem as we can't end up with an unkillable process
 550                  * here, because the wait will eventually time out.
 551                  *
 552                  * An example of this situation is the Xorg process
 553                  * which uses SIGALRM internally. The signal could
 554                  * interrupt the wait, causing the page fault to fail
 555                  * and the process to receive SIGSEGV.
 556                  */
 557                 ret = ttm_bo_wait(bo, false, false, false);
 558                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 559                 if (unlikely(ret != 0)) {
 560                         retval = VM_PAGER_ERROR;
 561                         goto out_unlock;
 562                 }
 563         } else {
 564                 lockmgr(&bdev->fence_lock, LK_RELEASE);
 565         }
 566
 567         ret = ttm_mem_io_lock(man, true);
 568         if (unlikely(ret != 0)) {
 569                 retval = VM_PAGER_ERROR;
 570                 goto out_unlock;
 571         }
 572         ret = ttm_mem_io_reserve_vm(bo);
 573         if (unlikely(ret != 0)) {
 574                 retval = VM_PAGER_ERROR;
 575                 goto out_io_unlock;
 576         }
 577         if (unlikely(OFF_TO_IDX(offset) >= bo->num_pages)) {
 578                 retval = VM_PAGER_ERROR;
 579                 goto out_io_unlock;
 580         }
 581
 582         /*
 583          * Strictly, we're not allowed to modify vma->vm_page_prot here,
 584          * since the mmap_sem is only held in read mode. However, we
 585          * modify only the caching bits of vma->vm_page_prot and
 586          * consider those bits protected by
 587          * the bo->mutex, as we should be the only writers.
 588          * There shouldn't really be any readers of these bits except
 589          * within vm_insert_mixed()? fork?
 590          *
 591          * TODO: Add a list of vmas to the bo, and change the
 592          * vma->vm_page_prot when the object changes caching policy, with
 593          * the correct locks held.
 594          */
 595
 596         if (bo->mem.bus.is_iomem) {
 597                 m = vm_phys_fictitious_to_vm_page(bo->mem.bus.base +
 598                                                   bo->mem.bus.offset + offset);
 599                 pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement, 0));
 600         } else {
 601                 /* Allocate all page at once, most common usage */
 602                 ttm = bo->ttm;
 603                 if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
 604                         retval = VM_PAGER_ERROR;
 605                         goto out_io_unlock;
 606                 }
 607                 ttm = bo->ttm;
 608
 609                 m = (struct vm_page *)ttm->pages[OFF_TO_IDX(offset)];
 610                 if (unlikely(!m)) {
 611                         retval = VM_PAGER_ERROR;
 612                         goto out_io_unlock;
 613                 }
 614                 pmap_page_set_memattr(m,
 615                     (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
 616                     VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement, 0));
 617         }
 618
 619         VM_OBJECT_LOCK(vm_obj);
 620
 621         if (vm_page_busy_try(m, FALSE)) {
 622                 kprintf("r");
 623                 vm_page_sleep_busy(m, FALSE, "ttmvmf");
 624                 ttm_mem_io_unlock(man);
 625                 ttm_bo_unreserve(bo);
 626                 goto retry;
 627         }
 628
 629         /*
 630          * We want our fake page in the VM object, not the page the OS
 631          * allocatedd for us as a placeholder.
 632          */
 633         m->valid = VM_PAGE_BITS_ALL;
 634         *mres = m;
 635         if (oldm != NULL) {
 636                 vm_page_remove(oldm);
 637                 if (m->object) {
 638                         retval = VM_PAGER_ERROR;
 639                         kprintf("ttm_bo_vm_fault_dfly: m(%p) already inserted "
 640                                 "in obj %p, attempt obj %p\n",
 641                                 m, m->object, vm_obj);
 642                         while (drm_unstall == 0) {
 643                                 tsleep(&retval, 0, "DEBUG", hz/10);
 644                         }
 645                         if (drm_unstall > 0)
 646                                 --drm_unstall;
 647                 } else {
 648                         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
 649                 }
 650                 vm_page_free(oldm);
 651                 oldm = NULL;
 652         } else {
 653                 vm_page_t mtmp;
 654
 655                 kprintf("oldm NULL\n");
 656
 657                 mtmp = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
 658                 KASSERT(mtmp == NULL || mtmp == m,
 659                     ("inconsistent insert bo %p m %p mtmp %p offset %jx",
 660                     bo, m, mtmp, (uintmax_t)offset));
 661                 if (mtmp == NULL)
 662                         vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
 663         }
 664         /*vm_page_busy_try(m, FALSE);*/
 665
 666 out_io_unlock1:
 667         ttm_mem_io_unlock(man);
 668 out_unlock1:
 669         ttm_bo_unreserve(bo);
 670 out_unlock2:
 671         if (oldm) {
 672                 vm_page_remove(oldm);
 673                 vm_page_free(oldm);
 674         }
 675         vm_object_pip_wakeup(vm_obj);
 676         return (retval);
 677
 678 out_io_unlock:
 679         VM_OBJECT_LOCK(vm_obj);
 680         goto out_io_unlock1;
 681
 682 out_unlock:
 683         VM_OBJECT_LOCK(vm_obj);
 684         goto out_unlock1;
 685 }
 686
 687 static int
 688 ttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
 689                vm_ooffset_t foff, struct ucred *cred, u_short *color)
 690 {
 691
 692         /*
 693          * On Linux, a reference to the buffer object is acquired here.
 694          * The reason is that this function is not called when the
 695          * mmap() is initialized, but only when a process forks for
 696          * instance. Therefore on Linux, the reference on the bo is
 697          * acquired either in ttm_bo_mmap() or ttm_bo_vm_open(). It's
 698          * then released in ttm_bo_vm_close().
 699          *
 700          * Here, this function is called during mmap() intialization.
 701          * Thus, the reference acquired in ttm_bo_mmap_single() is
 702          * sufficient.
 703          */
 704         *color = 0;
 705         return (0);
 706 }
 707
 708 static void
 709 ttm_bo_vm_dtor(void *handle)
 710 {
 711         struct ttm_buffer_object *bo = handle;
 712
 713         ttm_bo_unref(&bo);
 714 }
 715
 716 static struct cdev_pager_ops ttm_pager_ops = {
 717         .cdev_pg_fault = ttm_bo_vm_fault_dfly,
 718         .cdev_pg_ctor = ttm_bo_vm_ctor,
 719         .cdev_pg_dtor = ttm_bo_vm_dtor
 720 };
 721
 722 /*
 723  * Called from drm_drv.c
 724  *
 725  * *offset - object offset in bytes
 726  * size    - map size in bytes
 727  *
 728  * We setup a dummy vma (for now) and call ttm_bo_mmap().  Then we setup
 729  * our own VM object and dfly ops.  Note that the ops supplied by
 730  * ttm_bo_mmap() are not currently used.
 731  */
 732 int
 733 ttm_bo_mmap_single(struct drm_device *dev, vm_ooffset_t *offset,
 734                    vm_size_t size, struct vm_object **obj_res, int nprot)
 735 {
 736         struct ttm_bo_device *bdev = dev->drm_ttm_bdev;
 737         struct ttm_buffer_object *bo;
 738         struct vm_object *vm_obj;
 739         struct vm_area_struct vma;
 740         int ret;
 741
 742         *obj_res = NULL;
 743
 744         bzero(&vma, sizeof(vma));
 745         vma.vm_start = *offset;         /* bdev-relative offset */
 746         vma.vm_end = vma.vm_start + size;
 747         vma.vm_pgoff = vma.vm_start >> PAGE_SHIFT;
 748         /* vma.vm_page_prot */
 749         /* vma.vm_flags */
 750
 751         /*
 752          * Call the linux-ported code to do the work, and on success just
 753          * setup our own VM object and ignore what the linux code did other
 754          * then supplying us the 'bo'.
 755          */
 756         ret = ttm_bo_mmap(NULL, &vma, bdev);
 757
 758         if (ret == 0) {
 759                 bo = vma.vm_private_data;
 760                 vm_obj = cdev_pager_allocate(bo, OBJT_MGTDEVICE,
 761                                              &ttm_pager_ops,
 762                                              size, nprot, 0,
 763                                              curthread->td_ucred);
 764                 if (vm_obj) {
 765                         *obj_res = vm_obj;
 766                         *offset = 0;            /* object-relative offset */
 767                 } else {
 768                         ttm_bo_unref(&bo);
 769                         ret = EINVAL;
 770                 }
 771         }
 772         return ret;
 773 }
 774 EXPORT_SYMBOL(ttm_bo_mmap_single);
 775
 776 void
 777 ttm_bo_release_mmap(struct ttm_buffer_object *bo)
 778 {
 779         vm_object_t vm_obj;
 780         vm_page_t m;
 781         int i;
 782
 783         vm_obj = cdev_pager_lookup(bo);
 784         if (vm_obj == NULL)
 785                 return;
 786
 787         VM_OBJECT_LOCK(vm_obj);
 788         for (i = 0; i < bo->num_pages; i++) {
 789                 m = vm_page_lookup_busy_wait(vm_obj, i, TRUE, "ttm_unm");
 790                 if (m == NULL)
 791                         continue;
 792                 cdev_pager_free_page(vm_obj, m);
 793         }
 794         VM_OBJECT_UNLOCK(vm_obj);
 795
 796         vm_object_deallocate(vm_obj);
 797 }
 798
 799 #if 0
 800 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
 801 {
 802         if (vma->vm_pgoff != 0)
 803                 return -EACCES;
 804
 805         vma->vm_ops = &ttm_bo_vm_ops;
 806         vma->vm_private_data = ttm_bo_reference(bo);
 807         vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
 808         return 0;
 809 }
 810 EXPORT_SYMBOL(ttm_fbdev_mmap);
 811 #endif