drm/ttm: convert to unified vma offset manager
[dragonfly.git] / sys / dev / drm / ttm / ttm_bo_vm.c
CommitLineData
5c9d63cf
FT
1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
5c9d63cf 30
c66857eb
FT
31#define pr_fmt(fmt) "[TTM] " fmt
32
a34b4168
MD
33#include <ttm/ttm_module.h>
34#include <ttm/ttm_bo_driver.h>
35#include <ttm/ttm_bo_api.h>
36#include <ttm/ttm_placement.h>
37#include <drm/drm_vma_manager.h>
38#include <linux/mm.h>
39#include <linux/rbtree.h>
40#include <linux/module.h>
41#include <linux/uaccess.h>
42
43#include <sys/sysctl.h>
5c9d63cf
FT
44#include <vm/vm.h>
45#include <vm/vm_page.h>
62bba8f6
FT
46#include <vm/vm_page2.h>
47
a34b4168 48#define TTM_BO_VM_NUM_PREFAULT 16
ddea3d8d 49
a34b4168
MD
50/*
51 * Always unstall on unexpected vm_page alias, fatal bus fault.
52 * Set to 0 to stall, set to positive count to unstall N times,
53 * then stall again.
54 */
55static int drm_unstall = -1;
56SYSCTL_INT(_debug, OID_AUTO, unstall, CTLFLAG_RW, &drm_unstall, 0, "");
5c9d63cf 57
a34b4168
MD
58static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
59{
60 /* see ttm_bo_mmap_single() at end of this file */
61 /* ttm_bo_vm_ops not currently used, no entry should occur */
62 panic("ttm_bo_vm_fault");
63#if 0
64 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
65 vma->vm_private_data;
66 struct ttm_bo_device *bdev = bo->bdev;
67 unsigned long page_offset;
68 unsigned long page_last;
69 unsigned long pfn;
70 struct ttm_tt *ttm = NULL;
71 struct page *page;
72 int ret;
73 int i;
74 unsigned long address = (unsigned long)vmf->virtual_address;
75 int retval = VM_FAULT_NOPAGE;
76 struct ttm_mem_type_manager *man =
77 &bdev->man[bo->mem.mem_type];
5c9d63cf 78
a34b4168
MD
79 /*
80 * Work around locking order reversal in fault / nopfn
81 * between mmap_sem and bo_reserve: Perform a trylock operation
82 * for reserve, and if it fails, retry the fault after scheduling.
83 */
84
85 ret = ttm_bo_reserve(bo, true, true, false, 0);
86 if (unlikely(ret != 0)) {
87 if (ret == -EBUSY)
88 set_need_resched();
89 return VM_FAULT_NOPAGE;
90 }
91
92 if (bdev->driver->fault_reserve_notify) {
93 ret = bdev->driver->fault_reserve_notify(bo);
94 switch (ret) {
95 case 0:
96 break;
97 case -EBUSY:
98 set_need_resched();
99 case -ERESTARTSYS:
100 retval = VM_FAULT_NOPAGE;
101 goto out_unlock;
102 default:
103 retval = VM_FAULT_SIGBUS;
104 goto out_unlock;
105 }
106 }
107
108 /*
109 * Wait for buffer data in transit, due to a pipelined
110 * move.
111 */
112
113 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
114 if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
115 ret = ttm_bo_wait(bo, false, true, false);
116 lockmgr(&bdev->fence_lock, LK_RELEASE);
117 if (unlikely(ret != 0)) {
118 retval = (ret != -ERESTARTSYS) ?
119 VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
120 goto out_unlock;
121 }
122 } else
123 lockmgr(&bdev->fence_lock, LK_RELEASE);
124
125 ret = ttm_mem_io_lock(man, true);
126 if (unlikely(ret != 0)) {
127 retval = VM_FAULT_NOPAGE;
128 goto out_unlock;
129 }
130 ret = ttm_mem_io_reserve_vm(bo);
131 if (unlikely(ret != 0)) {
132 retval = VM_FAULT_SIGBUS;
133 goto out_io_unlock;
134 }
135
136 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
137 drm_vma_node_start(&bo->vma_node) - vma->vm_pgoff;
138 page_last = vma_pages(vma) +
139 drm_vma_node_start(&bo->vma_node) - vma->vm_pgoff;
140
141 if (unlikely(page_offset >= bo->num_pages)) {
142 retval = VM_FAULT_SIGBUS;
143 goto out_io_unlock;
144 }
145
146 /*
147 * Strictly, we're not allowed to modify vma->vm_page_prot here,
148 * since the mmap_sem is only held in read mode. However, we
149 * modify only the caching bits of vma->vm_page_prot and
150 * consider those bits protected by
151 * the bo->mutex, as we should be the only writers.
152 * There shouldn't really be any readers of these bits except
153 * within vm_insert_mixed()? fork?
154 *
155 * TODO: Add a list of vmas to the bo, and change the
156 * vma->vm_page_prot when the object changes caching policy, with
157 * the correct locks held.
158 */
159 if (bo->mem.bus.is_iomem) {
160 vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
161 vma->vm_page_prot);
162 } else {
163 ttm = bo->ttm;
164 vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
165 vm_get_page_prot(vma->vm_flags) :
166 ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
167
168 /* Allocate all page at once, most common usage */
169 if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
170 retval = VM_FAULT_OOM;
171 goto out_io_unlock;
172 }
173 }
174
175 /*
176 * Speculatively prefault a number of pages. Only error on
177 * first page.
178 */
179 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
180 if (bo->mem.bus.is_iomem)
181 pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset;
182 else {
183 page = ttm->pages[page_offset];
184 if (unlikely(!page && i == 0)) {
185 retval = VM_FAULT_OOM;
186 goto out_io_unlock;
187 } else if (unlikely(!page)) {
188 break;
189 }
190 pfn = page_to_pfn(page);
191 }
192
193 ret = vm_insert_mixed(vma, address, pfn);
194 /*
195 * Somebody beat us to this PTE or prefaulting to
196 * an already populated PTE, or prefaulting error.
197 */
198
199 if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0)))
200 break;
201 else if (unlikely(ret != 0)) {
202 retval =
203 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
204 goto out_io_unlock;
205 }
206
207 address += PAGE_SIZE;
208 if (unlikely(++page_offset >= page_last))
209 break;
210 }
211out_io_unlock:
212 ttm_mem_io_unlock(man);
213out_unlock:
214 ttm_bo_unreserve(bo);
215 return retval;
216#endif
217}
218
219/* ttm_bo_vm_ops not currently used, no entry should occur */
220static void ttm_bo_vm_open(struct vm_area_struct *vma)
ddea3d8d 221{
a34b4168
MD
222 struct ttm_buffer_object *bo =
223 (struct ttm_buffer_object *)vma->vm_private_data;
224
225 (void)ttm_bo_reference(bo);
226}
227
228/* ttm_bo_vm_ops not currently used, no entry should occur */
229static void ttm_bo_vm_close(struct vm_area_struct *vma)
230{
231 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data;
232
233 ttm_bo_unref(&bo);
234 vma->vm_private_data = NULL;
ddea3d8d 235}
236
a34b4168
MD
237static const struct vm_operations_struct ttm_bo_vm_ops = {
238 .fault = ttm_bo_vm_fault,
239 .open = ttm_bo_vm_open,
240 .close = ttm_bo_vm_close
241};
242
243static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
244 unsigned long offset,
245 unsigned long pages)
246{
247 struct drm_vma_offset_node *node;
248 struct ttm_buffer_object *bo = NULL;
249
250 drm_vma_offset_lock_lookup(&bdev->vma_manager);
251
252 node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages);
253 if (likely(node)) {
254 bo = container_of(node, struct ttm_buffer_object, vma_node);
255 if (!kref_get_unless_zero(&bo->kref))
256 bo = NULL;
257 }
258
259 drm_vma_offset_unlock_lookup(&bdev->vma_manager);
260
261 if (!bo)
262 pr_err("Could not find buffer object to map\n");
263
264 return bo;
265}
ddea3d8d 266
a34b4168
MD
267int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
268 struct ttm_bo_device *bdev)
5c9d63cf 269{
a34b4168 270 struct ttm_bo_driver *driver;
5c9d63cf 271 struct ttm_buffer_object *bo;
a34b4168
MD
272 int ret;
273
274 bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma));
275 if (unlikely(!bo))
276 return -EINVAL;
277
278 driver = bo->bdev->driver;
279 if (unlikely(!driver->verify_access)) {
280 ret = -EPERM;
281 goto out_unref;
282 }
283 ret = driver->verify_access(bo, filp);
284 if (unlikely(ret != 0))
285 goto out_unref;
286
287 vma->vm_ops = &ttm_bo_vm_ops;
288
289 /*
290 * Note: We're transferring the bo reference to
291 * vma->vm_private_data here.
292 */
293
294 vma->vm_private_data = bo;
295 vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP;
296 return 0;
297out_unref:
298 ttm_bo_unref(&bo);
299 return ret;
300}
301EXPORT_SYMBOL(ttm_bo_mmap);
302
303int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
304{
305 if (vma->vm_pgoff != 0)
306 return -EACCES;
307
308 vma->vm_ops = &ttm_bo_vm_ops;
309 vma->vm_private_data = ttm_bo_reference(bo);
310 vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
311 return 0;
312}
313EXPORT_SYMBOL(ttm_fbdev_mmap);
314
315
316ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
317 const char __user *wbuf, char __user *rbuf, size_t count,
318 loff_t *f_pos, bool write)
319{
320 struct ttm_buffer_object *bo;
321 struct ttm_bo_driver *driver;
322 struct ttm_bo_kmap_obj map;
323 unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
324 unsigned long kmap_offset;
325 unsigned long kmap_end;
326 unsigned long kmap_num;
327 size_t io_size;
328 unsigned int page_offset;
329 char *virtual;
330 int ret;
331 bool no_wait = false;
332 bool dummy;
333
334 bo = ttm_bo_vm_lookup(bdev, dev_offset, 1);
335 if (unlikely(bo == NULL))
336 return -EFAULT;
337
338 driver = bo->bdev->driver;
339 if (unlikely(!driver->verify_access)) {
340 ret = -EPERM;
341 goto out_unref;
342 }
343
344 ret = driver->verify_access(bo, filp);
345 if (unlikely(ret != 0))
346 goto out_unref;
347
348 kmap_offset = dev_offset - drm_vma_node_start(&bo->vma_node);
349 if (unlikely(kmap_offset >= bo->num_pages)) {
350 ret = -EFBIG;
351 goto out_unref;
352 }
353
354 page_offset = *f_pos & ~PAGE_MASK;
355 io_size = bo->num_pages - kmap_offset;
356 io_size = (io_size << PAGE_SHIFT) - page_offset;
357 if (count < io_size)
358 io_size = count;
359
360 kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
361 kmap_num = kmap_end - kmap_offset + 1;
362
363 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
364
365 switch (ret) {
366 case 0:
367 break;
368 case -EBUSY:
369 ret = -EAGAIN;
370 goto out_unref;
371 default:
372 goto out_unref;
373 }
374
375 ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
376 if (unlikely(ret != 0)) {
377 ttm_bo_unreserve(bo);
378 goto out_unref;
379 }
380
381 virtual = ttm_kmap_obj_virtual(&map, &dummy);
382 virtual += page_offset;
383
384 if (write)
385 ret = copy_from_user(virtual, wbuf, io_size);
386 else
387 ret = copy_to_user(rbuf, virtual, io_size);
388
389 ttm_bo_kunmap(&map);
390 ttm_bo_unreserve(bo);
391 ttm_bo_unref(&bo);
392
393 if (unlikely(ret != 0))
394 return -EFBIG;
395
396 *f_pos += io_size;
397
398 return io_size;
399out_unref:
400 ttm_bo_unref(&bo);
401 return ret;
402}
403
404ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
405 char __user *rbuf, size_t count, loff_t *f_pos,
406 bool write)
407{
408 struct ttm_bo_kmap_obj map;
409 unsigned long kmap_offset;
410 unsigned long kmap_end;
411 unsigned long kmap_num;
412 size_t io_size;
413 unsigned int page_offset;
414 char *virtual;
415 int ret;
416 bool no_wait = false;
417 bool dummy;
418
419 kmap_offset = (*f_pos >> PAGE_SHIFT);
420 if (unlikely(kmap_offset >= bo->num_pages))
421 return -EFBIG;
422
423 page_offset = *f_pos & ~PAGE_MASK;
424 io_size = bo->num_pages - kmap_offset;
425 io_size = (io_size << PAGE_SHIFT) - page_offset;
426 if (count < io_size)
427 io_size = count;
428
429 kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
430 kmap_num = kmap_end - kmap_offset + 1;
431
432 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
433
434 switch (ret) {
435 case 0:
436 break;
437 case -EBUSY:
438 return -EAGAIN;
439 default:
440 return ret;
5c9d63cf
FT
441 }
442
a34b4168
MD
443 ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
444 if (unlikely(ret != 0)) {
445 ttm_bo_unreserve(bo);
446 return ret;
447 }
5c9d63cf 448
a34b4168
MD
449 virtual = ttm_kmap_obj_virtual(&map, &dummy);
450 virtual += page_offset;
5c9d63cf 451
a34b4168
MD
452 if (write)
453 ret = copy_from_user(virtual, wbuf, io_size);
454 else
455 ret = copy_to_user(rbuf, virtual, io_size);
456
457 ttm_bo_kunmap(&map);
458 ttm_bo_unreserve(bo);
459 ttm_bo_unref(&bo);
460
461 if (unlikely(ret != 0))
462 return ret;
463
464 *f_pos += io_size;
465
466 return io_size;
5c9d63cf
FT
467}
468
a34b4168
MD
469/*
470 * DragonFlyBSD Interface
471 */
472
473#include "opt_vm.h"
474
475#include <vm/vm.h>
476#include <vm/vm_page.h>
477#include <linux/errno.h>
478#include <linux/export.h>
479
480#include <vm/vm_page2.h>
481
5c9d63cf 482static int
a34b4168
MD
483ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset,
484 int prot, vm_page_t *mres)
5c9d63cf 485{
5c9d63cf
FT
486 struct ttm_buffer_object *bo = vm_obj->handle;
487 struct ttm_bo_device *bdev = bo->bdev;
488 struct ttm_tt *ttm = NULL;
a34b4168 489 vm_page_t m, oldm;
5c9d63cf
FT
490 int ret;
491 int retval = VM_PAGER_OK;
a34b4168
MD
492 struct ttm_mem_type_manager *man;
493
494 man = &bdev->man[bo->mem.mem_type];
495
496 /*kprintf("FAULT %p %p/%ld\n", vm_obj, bo, offset);*/
5c9d63cf
FT
497
498 vm_object_pip_add(vm_obj, 1);
499 oldm = *mres;
a34b4168
MD
500 *mres = NULL;
501
5c9d63cf 502retry:
04b45e6f 503 VM_OBJECT_UNLOCK(vm_obj);
5c9d63cf
FT
504 m = NULL;
505
a34b4168
MD
506 /*
507 * NOTE: set nowait to false, we don't have ttm_bo_wait_unreserved()
508 * for the -BUSY case yet.
509 */
510 ret = ttm_bo_reserve(bo, true, false, false, 0);
5c9d63cf 511 if (unlikely(ret != 0)) {
a34b4168
MD
512 retval = VM_PAGER_ERROR;
513 VM_OBJECT_LOCK(vm_obj);
514 goto out_unlock2;
5c9d63cf
FT
515 }
516
517 if (bdev->driver->fault_reserve_notify) {
518 ret = bdev->driver->fault_reserve_notify(bo);
519 switch (ret) {
520 case 0:
521 break;
522 case -EBUSY:
a34b4168
MD
523 lwkt_yield();
524 /* fall through */
797013cf 525 case -ERESTARTSYS:
5c9d63cf 526 case -EINTR:
a34b4168
MD
527 retval = VM_PAGER_ERROR;
528 goto out_unlock;
5c9d63cf
FT
529 default:
530 retval = VM_PAGER_ERROR;
531 goto out_unlock;
532 }
533 }
534
535 /*
536 * Wait for buffer data in transit, due to a pipelined
537 * move.
538 */
1716017e 539 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE);
5c9d63cf 540 if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
6f486c69
FT
541 /*
542 * Here, the behavior differs between Linux and FreeBSD.
543 *
544 * On Linux, the wait is interruptible (3rd argument to
545 * ttm_bo_wait). There must be some mechanism to resume
546 * page fault handling, once the signal is processed.
547 *
548 * On FreeBSD, the wait is uninteruptible. This is not a
549 * problem as we can't end up with an unkillable process
550 * here, because the wait will eventually time out.
551 *
552 * An example of this situation is the Xorg process
553 * which uses SIGALRM internally. The signal could
554 * interrupt the wait, causing the page fault to fail
555 * and the process to receive SIGSEGV.
556 */
557 ret = ttm_bo_wait(bo, false, false, false);
1716017e 558 lockmgr(&bdev->fence_lock, LK_RELEASE);
5c9d63cf
FT
559 if (unlikely(ret != 0)) {
560 retval = VM_PAGER_ERROR;
561 goto out_unlock;
562 }
a34b4168 563 } else {
1716017e 564 lockmgr(&bdev->fence_lock, LK_RELEASE);
a34b4168 565 }
5c9d63cf
FT
566
567 ret = ttm_mem_io_lock(man, true);
568 if (unlikely(ret != 0)) {
569 retval = VM_PAGER_ERROR;
570 goto out_unlock;
571 }
572 ret = ttm_mem_io_reserve_vm(bo);
573 if (unlikely(ret != 0)) {
574 retval = VM_PAGER_ERROR;
575 goto out_io_unlock;
576 }
a34b4168
MD
577 if (unlikely(OFF_TO_IDX(offset) >= bo->num_pages)) {
578 retval = VM_PAGER_ERROR;
579 goto out_io_unlock;
580 }
5c9d63cf
FT
581
582 /*
583 * Strictly, we're not allowed to modify vma->vm_page_prot here,
584 * since the mmap_sem is only held in read mode. However, we
585 * modify only the caching bits of vma->vm_page_prot and
586 * consider those bits protected by
587 * the bo->mutex, as we should be the only writers.
588 * There shouldn't really be any readers of these bits except
589 * within vm_insert_mixed()? fork?
590 *
591 * TODO: Add a list of vmas to the bo, and change the
592 * vma->vm_page_prot when the object changes caching policy, with
593 * the correct locks held.
594 */
a34b4168
MD
595
596 if (bo->mem.bus.is_iomem) {
597 m = vm_phys_fictitious_to_vm_page(bo->mem.bus.base +
598 bo->mem.bus.offset + offset);
599 pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement, 0));
600 } else {
5c9d63cf
FT
601 /* Allocate all page at once, most common usage */
602 ttm = bo->ttm;
603 if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
604 retval = VM_PAGER_ERROR;
605 goto out_io_unlock;
606 }
5c9d63cf 607 ttm = bo->ttm;
a34b4168 608
f0bba3d1 609 m = (struct vm_page *)ttm->pages[OFF_TO_IDX(offset)];
5c9d63cf
FT
610 if (unlikely(!m)) {
611 retval = VM_PAGER_ERROR;
612 goto out_io_unlock;
613 }
614 pmap_page_set_memattr(m,
615 (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
4f125aea 616 VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement, 0));
5c9d63cf
FT
617 }
618
04b45e6f 619 VM_OBJECT_LOCK(vm_obj);
a34b4168
MD
620
621 if (vm_page_busy_try(m, FALSE)) {
622 kprintf("r");
623 vm_page_sleep_busy(m, FALSE, "ttmvmf");
5c9d63cf
FT
624 ttm_mem_io_unlock(man);
625 ttm_bo_unreserve(bo);
626 goto retry;
627 }
a34b4168
MD
628
629 /*
630 * We want our fake page in the VM object, not the page the OS
631 * allocatedd for us as a placeholder.
632 */
5c9d63cf
FT
633 m->valid = VM_PAGE_BITS_ALL;
634 *mres = m;
5c9d63cf 635 if (oldm != NULL) {
a34b4168
MD
636 vm_page_remove(oldm);
637 if (m->object) {
638 retval = VM_PAGER_ERROR;
639 kprintf("ttm_bo_vm_fault_dfly: m(%p) already inserted "
640 "in obj %p, attempt obj %p\n",
641 m, m->object, vm_obj);
642 while (drm_unstall == 0) {
643 tsleep(&retval, 0, "DEBUG", hz/10);
644 }
645 if (drm_unstall > 0)
646 --drm_unstall;
647 } else {
648 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
649 }
5c9d63cf 650 vm_page_free(oldm);
a34b4168
MD
651 oldm = NULL;
652 } else {
653 vm_page_t mtmp;
654
655 kprintf("oldm NULL\n");
656
657 mtmp = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
658 KASSERT(mtmp == NULL || mtmp == m,
659 ("inconsistent insert bo %p m %p mtmp %p offset %jx",
660 bo, m, mtmp, (uintmax_t)offset));
661 if (mtmp == NULL)
662 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
5c9d63cf 663 }
a34b4168 664 /*vm_page_busy_try(m, FALSE);*/
5c9d63cf
FT
665
666out_io_unlock1:
667 ttm_mem_io_unlock(man);
668out_unlock1:
669 ttm_bo_unreserve(bo);
a34b4168
MD
670out_unlock2:
671 if (oldm) {
672 vm_page_remove(oldm);
673 vm_page_free(oldm);
674 }
5c9d63cf
FT
675 vm_object_pip_wakeup(vm_obj);
676 return (retval);
677
678out_io_unlock:
04b45e6f 679 VM_OBJECT_LOCK(vm_obj);
5c9d63cf
FT
680 goto out_io_unlock1;
681
682out_unlock:
04b45e6f 683 VM_OBJECT_LOCK(vm_obj);
5c9d63cf
FT
684 goto out_unlock1;
685}
686
687static int
688ttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
a34b4168 689 vm_ooffset_t foff, struct ucred *cred, u_short *color)
5c9d63cf
FT
690{
691
692 /*
6f486c69
FT
693 * On Linux, a reference to the buffer object is acquired here.
694 * The reason is that this function is not called when the
695 * mmap() is initialized, but only when a process forks for
696 * instance. Therefore on Linux, the reference on the bo is
697 * acquired either in ttm_bo_mmap() or ttm_bo_vm_open(). It's
698 * then released in ttm_bo_vm_close().
699 *
700 * Here, this function is called during mmap() intialization.
701 * Thus, the reference acquired in ttm_bo_mmap_single() is
702 * sufficient.
5c9d63cf 703 */
5c9d63cf
FT
704 *color = 0;
705 return (0);
706}
707
708static void
709ttm_bo_vm_dtor(void *handle)
710{
711 struct ttm_buffer_object *bo = handle;
712
713 ttm_bo_unref(&bo);
714}
715
716static struct cdev_pager_ops ttm_pager_ops = {
a34b4168 717 .cdev_pg_fault = ttm_bo_vm_fault_dfly,
5c9d63cf
FT
718 .cdev_pg_ctor = ttm_bo_vm_ctor,
719 .cdev_pg_dtor = ttm_bo_vm_dtor
720};
721
a34b4168
MD
722/*
723 * Called from drm_drv.c
724 *
725 * *offset - object offset in bytes
726 * size - map size in bytes
727 *
728 * We setup a dummy vma (for now) and call ttm_bo_mmap(). Then we setup
729 * our own VM object and dfly ops. Note that the ops supplied by
730 * ttm_bo_mmap() are not currently used.
731 */
5c9d63cf 732int
a34b4168
MD
733ttm_bo_mmap_single(struct drm_device *dev, vm_ooffset_t *offset,
734 vm_size_t size, struct vm_object **obj_res, int nprot)
5c9d63cf 735{
a34b4168 736 struct ttm_bo_device *bdev = dev->drm_ttm_bdev;
5c9d63cf
FT
737 struct ttm_buffer_object *bo;
738 struct vm_object *vm_obj;
a34b4168 739 struct vm_area_struct vma;
5c9d63cf
FT
740 int ret;
741
f6201ebf
MD
742 *obj_res = NULL;
743
a34b4168
MD
744 bzero(&vma, sizeof(vma));
745 vma.vm_start = *offset; /* bdev-relative offset */
746 vma.vm_end = vma.vm_start + size;
747 vma.vm_pgoff = vma.vm_start >> PAGE_SHIFT;
748 /* vma.vm_page_prot */
749 /* vma.vm_flags */
5c9d63cf 750
5c9d63cf 751 /*
a34b4168
MD
752 * Call the linux-ported code to do the work, and on success just
753 * setup our own VM object and ignore what the linux code did other
754 * then supplying us the 'bo'.
5c9d63cf 755 */
a34b4168
MD
756 ret = ttm_bo_mmap(NULL, &vma, bdev);
757
758 if (ret == 0) {
759 bo = vma.vm_private_data;
760 vm_obj = cdev_pager_allocate(bo, OBJT_MGTDEVICE,
761 &ttm_pager_ops,
762 size, nprot, 0,
763 curthread->td_ucred);
764 if (vm_obj) {
765 *obj_res = vm_obj;
766 *offset = 0; /* object-relative offset */
767 } else {
768 ttm_bo_unref(&bo);
769 ret = EINVAL;
770 }
771 }
5c9d63cf
FT
772 return ret;
773}
a34b4168 774EXPORT_SYMBOL(ttm_bo_mmap_single);
5c9d63cf 775
6f486c69
FT
776void
777ttm_bo_release_mmap(struct ttm_buffer_object *bo)
778{
779 vm_object_t vm_obj;
780 vm_page_t m;
781 int i;
782
783 vm_obj = cdev_pager_lookup(bo);
784 if (vm_obj == NULL)
785 return;
786
04b45e6f 787 VM_OBJECT_LOCK(vm_obj);
6f486c69
FT
788 for (i = 0; i < bo->num_pages; i++) {
789 m = vm_page_lookup_busy_wait(vm_obj, i, TRUE, "ttm_unm");
790 if (m == NULL)
791 continue;
792 cdev_pager_free_page(vm_obj, m);
793 }
04b45e6f 794 VM_OBJECT_UNLOCK(vm_obj);
6f486c69
FT
795
796 vm_object_deallocate(vm_obj);
797}
798
5c9d63cf
FT
799#if 0
800int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
801{
802 if (vma->vm_pgoff != 0)
803 return -EACCES;
804
805 vma->vm_ops = &ttm_bo_vm_ops;
806 vma->vm_private_data = ttm_bo_reference(bo);
807 vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
808 return 0;
809}
c66857eb 810EXPORT_SYMBOL(ttm_fbdev_mmap);
5c9d63cf 811#endif