1 /**************************************************************************
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
31 * Copyright (c) 2013 The FreeBSD Foundation
32 * All rights reserved.
34 * Portions of this software were developed by Konstantin Belousov
35 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
37 * $FreeBSD: head/sys/dev/drm2/ttm/ttm_tt.c 251452 2013-06-06 06:17:20Z alc $
40 #include <dev/drm2/drmP.h>
41 #include <dev/drm2/ttm/ttm_module.h>
42 #include <dev/drm2/ttm/ttm_bo_driver.h>
43 #include <dev/drm2/ttm/ttm_placement.h>
44 #include <dev/drm2/ttm/ttm_page_alloc.h>
46 MALLOC_DEFINE(M_TTM_PD, "ttm_pd", "TTM Page Directories");
49 * Allocates storage for pointers to the pages that back the ttm.
51 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
53 ttm->pages = kmalloc(ttm->num_pages * sizeof(void *),
54 M_TTM_PD, M_WAITOK | M_ZERO);
57 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
59 ttm->ttm.pages = kmalloc(ttm->ttm.num_pages * sizeof(void *),
60 M_TTM_PD, M_WAITOK | M_ZERO);
61 ttm->dma_address = kmalloc(ttm->ttm.num_pages *
62 sizeof(*ttm->dma_address), M_TTM_PD, M_WAITOK);
65 #if defined(__i386__) || defined(__amd64__)
66 static inline int ttm_tt_set_page_caching(vm_page_t p,
67 enum ttm_caching_state c_old,
68 enum ttm_caching_state c_new)
71 /* XXXKIB our VM does not need this. */
73 if (c_old != tt_cached) {
74 /* p isn't in the default caching state, set it to
75 * writeback first to free its current memtype. */
76 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK);
81 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING);
82 else if (c_new == tt_uncached)
83 pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE);
88 static inline int ttm_tt_set_page_caching(vm_page_t p,
89 enum ttm_caching_state c_old,
90 enum ttm_caching_state c_new)
97 * Change caching policy for the linear kernel map
98 * for range of pages in a ttm.
101 static int ttm_tt_set_caching(struct ttm_tt *ttm,
102 enum ttm_caching_state c_state)
108 if (ttm->caching_state == c_state)
111 if (ttm->state == tt_unpopulated) {
112 /* Change caching but don't populate */
113 ttm->caching_state = c_state;
117 if (ttm->caching_state == tt_cached)
118 drm_clflush_pages(ttm->pages, ttm->num_pages);
120 for (i = 0; i < ttm->num_pages; ++i) {
121 cur_page = ttm->pages[i];
122 if (likely(cur_page != NULL)) {
123 ret = ttm_tt_set_page_caching(cur_page,
126 if (unlikely(ret != 0))
131 ttm->caching_state = c_state;
136 for (j = 0; j < i; ++j) {
137 cur_page = ttm->pages[j];
138 if (cur_page != NULL) {
139 (void)ttm_tt_set_page_caching(cur_page, c_state,
147 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
149 enum ttm_caching_state state;
151 if (placement & TTM_PL_FLAG_WC)
153 else if (placement & TTM_PL_FLAG_UNCACHED)
158 return ttm_tt_set_caching(ttm, state);
161 void ttm_tt_destroy(struct ttm_tt *ttm)
163 if (unlikely(ttm == NULL))
166 if (ttm->state == tt_bound) {
170 if (likely(ttm->pages != NULL)) {
171 ttm->bdev->driver->ttm_tt_unpopulate(ttm);
174 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
176 vm_object_deallocate(ttm->swap_storage);
178 ttm->swap_storage = NULL;
179 ttm->func->destroy(ttm);
182 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev,
183 unsigned long size, uint32_t page_flags,
184 vm_page_t dummy_read_page)
187 ttm->glob = bdev->glob;
188 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
189 ttm->caching_state = tt_cached;
190 ttm->page_flags = page_flags;
191 ttm->dummy_read_page = dummy_read_page;
192 ttm->state = tt_unpopulated;
193 ttm->swap_storage = NULL;
195 ttm_tt_alloc_page_directory(ttm);
198 kprintf("Failed allocating page table\n");
204 void ttm_tt_fini(struct ttm_tt *ttm)
206 drm_free(ttm->pages, M_TTM_PD);
210 int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev,
211 unsigned long size, uint32_t page_flags,
212 vm_page_t dummy_read_page)
214 struct ttm_tt *ttm = &ttm_dma->ttm;
217 ttm->glob = bdev->glob;
218 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
219 ttm->caching_state = tt_cached;
220 ttm->page_flags = page_flags;
221 ttm->dummy_read_page = dummy_read_page;
222 ttm->state = tt_unpopulated;
223 ttm->swap_storage = NULL;
225 INIT_LIST_HEAD(&ttm_dma->pages_list);
226 ttm_dma_tt_alloc_page_directory(ttm_dma);
227 if (!ttm->pages || !ttm_dma->dma_address) {
229 kprintf("Failed allocating page table\n");
235 void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma)
237 struct ttm_tt *ttm = &ttm_dma->ttm;
239 drm_free(ttm->pages, M_TTM_PD);
241 drm_free(ttm_dma->dma_address, M_TTM_PD);
242 ttm_dma->dma_address = NULL;
245 void ttm_tt_unbind(struct ttm_tt *ttm)
249 if (ttm->state == tt_bound) {
250 ret = ttm->func->unbind(ttm);
252 ttm->state = tt_unbound;
256 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
263 if (ttm->state == tt_bound)
266 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
270 ret = ttm->func->bind(ttm, bo_mem);
271 if (unlikely(ret != 0))
274 ttm->state = tt_bound;
279 int ttm_tt_swapin(struct ttm_tt *ttm)
282 vm_page_t from_page, to_page;
285 obj = ttm->swap_storage;
287 VM_OBJECT_WLOCK(obj);
288 vm_object_pip_add(obj, 1);
289 for (i = 0; i < ttm->num_pages; ++i) {
290 from_page = vm_page_grab(obj, i, VM_ALLOC_NOBUSY |
292 if (from_page->valid != VM_PAGE_BITS_ALL) {
293 vm_page_busy_try(from_page, FALSE);
294 if (vm_pager_has_page(obj, i)) {
295 rv = vm_pager_get_page(obj, &from_page, 1);
296 if (rv != VM_PAGER_OK) {
297 vm_page_free(from_page);
302 vm_page_zero_invalid(from_page, TRUE);
303 vm_page_wakeup(from_page);
305 to_page = ttm->pages[i];
306 if (unlikely(to_page == NULL)) {
310 pmap_copy_page(from_page, to_page);
312 vm_object_pip_wakeup(obj);
313 VM_OBJECT_WUNLOCK(obj);
315 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
316 vm_object_deallocate(obj);
317 ttm->swap_storage = NULL;
318 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
322 vm_object_pip_wakeup(obj);
323 VM_OBJECT_WUNLOCK(obj);
327 int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage)
330 vm_page_t from_page, to_page;
333 KKASSERT(ttm->state == tt_unbound || ttm->state == tt_unpopulated);
334 KKASSERT(ttm->caching_state == tt_cached);
336 if (persistent_swap_storage == NULL) {
338 obj = vm_pager_allocate(OBJT_SWAP, NULL,
339 IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0,
340 curthread->td_ucred);
342 obj = swap_pager_alloc(NULL,
343 IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0);
346 kprintf("[TTM] Failed allocating swap storage\n");
350 obj = persistent_swap_storage;
352 VM_OBJECT_WLOCK(obj);
353 vm_object_pip_add(obj, 1);
354 for (i = 0; i < ttm->num_pages; ++i) {
355 from_page = ttm->pages[i];
356 if (unlikely(from_page == NULL))
358 to_page = vm_page_grab(obj, i, VM_ALLOC_RETRY);
359 pmap_copy_page(from_page, to_page);
360 vm_page_dirty(to_page);
361 to_page->valid = VM_PAGE_BITS_ALL;
362 vm_page_wakeup(to_page);
364 vm_object_pip_wakeup(obj);
365 VM_OBJECT_WUNLOCK(obj);
367 ttm->bdev->driver->ttm_tt_unpopulate(ttm);
368 ttm->swap_storage = obj;
369 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
370 if (persistent_swap_storage != NULL)
371 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;