Commit | Line | Data |
---|---|---|
984263bc | 1 | /* |
99ad9bc4 | 2 | * (MPSAFE) |
8e7c4729 | 3 | * |
984263bc MD |
4 | * Copyright (c) 1990 University of Utah. |
5 | * Copyright (c) 1991, 1993 | |
6 | * The Regents of the University of California. All rights reserved. | |
7 | * | |
8 | * This code is derived from software contributed to Berkeley by | |
9 | * the Systems Programming Group of the University of Utah Computer | |
10 | * Science Department. | |
11 | * | |
12 | * Redistribution and use in source and binary forms, with or without | |
13 | * modification, are permitted provided that the following conditions | |
14 | * are met: | |
15 | * 1. Redistributions of source code must retain the above copyright | |
16 | * notice, this list of conditions and the following disclaimer. | |
17 | * 2. Redistributions in binary form must reproduce the above copyright | |
18 | * notice, this list of conditions and the following disclaimer in the | |
19 | * documentation and/or other materials provided with the distribution. | |
dc71b7ab | 20 | * 3. Neither the name of the University nor the names of its contributors |
984263bc MD |
21 | * may be used to endorse or promote products derived from this software |
22 | * without specific prior written permission. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |
25 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
26 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
27 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |
28 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
29 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
30 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
31 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
32 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
33 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
34 | * SUCH DAMAGE. | |
35 | * | |
36 | * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 | |
37 | * $FreeBSD: src/sys/vm/device_pager.c,v 1.46.2.1 2000/08/02 21:54:37 peter Exp $ | |
38 | */ | |
39 | ||
40 | #include <sys/param.h> | |
41 | #include <sys/systm.h> | |
27db6797 | 42 | #include <sys/kernel.h> |
984263bc MD |
43 | #include <sys/conf.h> |
44 | #include <sys/mman.h> | |
335dda38 | 45 | #include <sys/device.h> |
27db6797 MD |
46 | #include <sys/queue.h> |
47 | #include <sys/malloc.h> | |
8e7c4729 | 48 | #include <sys/mutex2.h> |
984263bc MD |
49 | |
50 | #include <vm/vm.h> | |
51 | #include <vm/vm_object.h> | |
52 | #include <vm/vm_page.h> | |
53 | #include <vm/vm_pager.h> | |
54 | #include <vm/vm_zone.h> | |
a86ce0cd | 55 | #include <vm/vm_page2.h> |
984263bc | 56 | |
2d7bef58 SW |
57 | static pgo_dealloc_t dev_pager_dealloc; |
58 | static pgo_getpage_t dev_pager_getpage; | |
59 | static pgo_putpages_t dev_pager_putpages; | |
60 | static pgo_haspage_t dev_pager_haspage; | |
61 | ||
62 | struct pagerops devicepagerops = { | |
63 | .pgo_dealloc = dev_pager_dealloc, | |
64 | .pgo_getpage = dev_pager_getpage, | |
65 | .pgo_putpages = dev_pager_putpages, | |
66 | .pgo_haspage = dev_pager_haspage | |
67 | }; | |
984263bc MD |
68 | |
69 | /* list of device pager objects */ | |
107e9bcc MD |
70 | static TAILQ_HEAD(, vm_page) dev_freepages_list = |
71 | TAILQ_HEAD_INITIALIZER(dev_freepages_list); | |
8e7c4729 MD |
72 | static MALLOC_DEFINE(M_FICTITIOUS_PAGES, "device-mapped pages", |
73 | "Device mapped pages"); | |
984263bc | 74 | |
7db38254 | 75 | static vm_page_t dev_pager_getfake (vm_paddr_t, int); |
1388df65 | 76 | static void dev_pager_putfake (vm_page_t); |
984263bc | 77 | |
f2c2051e JH |
78 | /* list of device pager objects */ |
79 | static struct pagerlst dev_pager_object_list = | |
80 | TAILQ_HEAD_INITIALIZER(dev_pager_object_list); | |
81 | /* protect list manipulation */ | |
cabfc9f6 | 82 | static struct mtx dev_pager_mtx = MTX_INITIALIZER("devpgr"); |
8e7c4729 | 83 | |
f2c2051e JH |
84 | static int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, |
85 | vm_ooffset_t foff, struct ucred *cred, u_short *pg_color); | |
86 | static void old_dev_pager_dtor(void *handle); | |
87 | static int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, | |
88 | int prot, vm_page_t *mres); | |
89 | ||
90 | static struct cdev_pager_ops old_dev_pager_ops = { | |
91 | .cdev_pg_ctor = old_dev_pager_ctor, | |
92 | .cdev_pg_dtor = old_dev_pager_dtor, | |
93 | .cdev_pg_fault = old_dev_pager_fault | |
94 | }; | |
95 | ||
5a648714 | 96 | vm_object_t |
f2c2051e | 97 | cdev_pager_lookup(void *handle) |
984263bc | 98 | { |
984263bc | 99 | vm_object_t object; |
8ee19f91 | 100 | |
f2c2051e JH |
101 | mtx_lock(&dev_pager_mtx); |
102 | object = vm_pager_object_lookup(&dev_pager_object_list, handle); | |
103 | mtx_unlock(&dev_pager_mtx); | |
8ee19f91 | 104 | |
f2c2051e JH |
105 | return (object); |
106 | } | |
984263bc | 107 | |
f2c2051e JH |
108 | vm_object_t |
109 | cdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops, | |
110 | vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) | |
111 | { | |
112 | cdev_t dev; | |
113 | vm_object_t object; | |
114 | u_short color; | |
984263bc MD |
115 | |
116 | /* | |
117 | * Offset should be page aligned. | |
118 | */ | |
119 | if (foff & PAGE_MASK) | |
120 | return (NULL); | |
121 | ||
aecf2182 | 122 | size = round_page64(size); |
984263bc | 123 | |
f2c2051e JH |
124 | if (ops->cdev_pg_ctor(handle, size, prot, foff, cred, &color) != 0) |
125 | return (NULL); | |
984263bc | 126 | |
984263bc MD |
127 | /* |
128 | * Look up pager, creating as necessary. | |
129 | */ | |
8e7c4729 | 130 | mtx_lock(&dev_pager_mtx); |
f2c2051e | 131 | object = vm_pager_object_lookup(&dev_pager_object_list, handle); |
984263bc MD |
132 | if (object == NULL) { |
133 | /* | |
134 | * Allocate object and associate it with the pager. | |
135 | */ | |
f2c2051e | 136 | object = vm_object_allocate_hold(tp, |
a2ee730d | 137 | OFF_TO_IDX(foff + size)); |
984263bc | 138 | object->handle = handle; |
f2c2051e JH |
139 | object->un_pager.devp.ops = ops; |
140 | object->un_pager.devp.dev = handle; | |
984263bc | 141 | TAILQ_INIT(&object->un_pager.devp.devp_pglist); |
3b9c5039 MD |
142 | |
143 | /* | |
144 | * handle is only a device for old_dev_pager_ctor. | |
145 | */ | |
146 | if (ops->cdev_pg_ctor == old_dev_pager_ctor) { | |
147 | dev = handle; | |
148 | dev->si_object = object; | |
149 | } | |
f2c2051e JH |
150 | |
151 | TAILQ_INSERT_TAIL(&dev_pager_object_list, object, | |
5b329e62 | 152 | pager_object_entry); |
f2c2051e | 153 | |
a2ee730d | 154 | vm_object_drop(object); |
984263bc MD |
155 | } else { |
156 | /* | |
157 | * Gain a reference to the object. | |
158 | */ | |
b12defdc MD |
159 | vm_object_hold(object); |
160 | vm_object_reference_locked(object); | |
984263bc MD |
161 | if (OFF_TO_IDX(foff + size) > object->size) |
162 | object->size = OFF_TO_IDX(foff + size); | |
b12defdc | 163 | vm_object_drop(object); |
984263bc | 164 | } |
8e7c4729 | 165 | mtx_unlock(&dev_pager_mtx); |
984263bc MD |
166 | |
167 | return (object); | |
168 | } | |
169 | ||
f2c2051e JH |
170 | /* |
171 | * No requirements. | |
172 | */ | |
173 | vm_object_t | |
174 | dev_pager_alloc(void *handle, off_t size, vm_prot_t prot, off_t foff) | |
175 | { | |
176 | return (cdev_pager_allocate(handle, OBJT_DEVICE, &old_dev_pager_ops, | |
f16f9121 | 177 | size, prot, foff, NULL)); |
f2c2051e JH |
178 | } |
179 | ||
8ee19f91 MD |
180 | /* |
181 | * Caller must hold object lock. | |
182 | */ | |
f2c2051e JH |
183 | void |
184 | cdev_pager_free_page(vm_object_t object, vm_page_t m) | |
185 | { | |
186 | if (object->type == OBJT_MGTDEVICE) { | |
f2c2051e JH |
187 | KKASSERT((m->flags & PG_FICTITIOUS) != 0); |
188 | pmap_page_protect(m, VM_PROT_NONE); | |
189 | vm_page_remove(m); | |
190 | vm_page_wakeup(m); | |
191 | } else if (object->type == OBJT_DEVICE) { | |
f2c2051e JH |
192 | TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq); |
193 | dev_pager_putfake(m); | |
194 | } | |
195 | } | |
196 | ||
8e7c4729 MD |
197 | /* |
198 | * No requirements. | |
199 | */ | |
984263bc | 200 | static void |
57e43348 | 201 | dev_pager_dealloc(vm_object_t object) |
984263bc MD |
202 | { |
203 | vm_page_t m; | |
d28e1355 | 204 | |
8ee19f91 MD |
205 | /* |
206 | * NOTE: Callback may recurse into the device pager so do not | |
207 | * obtain dev_pager_mtx until after it returns. | |
208 | * | |
209 | * The mutex should only be needed when manipulating the list. | |
210 | */ | |
f2c2051e | 211 | object->un_pager.devp.ops->cdev_pg_dtor(object->un_pager.devp.dev); |
8e7c4729 | 212 | |
8ee19f91 | 213 | mtx_lock(&dev_pager_mtx); |
5b329e62 | 214 | TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_entry); |
8ee19f91 | 215 | mtx_unlock(&dev_pager_mtx); |
984263bc | 216 | |
f2c2051e JH |
217 | if (object->type == OBJT_DEVICE) { |
218 | /* | |
219 | * Free up our fake pages. | |
220 | */ | |
221 | while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) != | |
222 | NULL) { | |
223 | TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, | |
224 | m, pageq); | |
225 | dev_pager_putfake(m); | |
226 | } | |
984263bc MD |
227 | } |
228 | } | |
229 | ||
8e7c4729 MD |
230 | /* |
231 | * No requirements. | |
f57b0af8 MD |
232 | * |
233 | * WARNING! Do not obtain dev_pager_mtx here, doing so will cause a | |
234 | * deadlock in DRMs VM paging code. | |
8e7c4729 | 235 | */ |
984263bc | 236 | static int |
5ebb17ad MD |
237 | dev_pager_getpage(vm_object_t object, vm_pindex_t pindex, |
238 | vm_page_t *mpp, int seqaccess) | |
984263bc | 239 | { |
f2c2051e | 240 | int error; |
984263bc | 241 | |
f57b0af8 | 242 | error = object->un_pager.devp.ops->cdev_pg_fault( |
5ebb17ad | 243 | object, IDX_TO_OFF(pindex), |
f57b0af8 | 244 | PROT_READ, mpp); |
f2c2051e JH |
245 | |
246 | return (error); | |
984263bc MD |
247 | } |
248 | ||
8e7c4729 MD |
249 | /* |
250 | * No requirements. | |
251 | */ | |
984263bc | 252 | static void |
1b9d3514 | 253 | dev_pager_putpages(vm_object_t object, vm_page_t *m, |
df0b0ead | 254 | int count, int flags, int *rtvals) |
984263bc MD |
255 | { |
256 | panic("dev_pager_putpage called"); | |
257 | } | |
258 | ||
8e7c4729 MD |
259 | /* |
260 | * No requirements. | |
261 | */ | |
984263bc | 262 | static boolean_t |
1b9d3514 | 263 | dev_pager_haspage(vm_object_t object, vm_pindex_t pindex) |
984263bc | 264 | { |
984263bc MD |
265 | return (TRUE); |
266 | } | |
267 | ||
8e7c4729 | 268 | /* |
8ee19f91 MD |
269 | * The caller does not need to hold dev_pager_mtx() but caller must ensure |
270 | * no page-use collision. | |
8e7c4729 | 271 | */ |
984263bc | 272 | static vm_page_t |
7db38254 | 273 | dev_pager_getfake(vm_paddr_t paddr, int pat_mode) |
984263bc MD |
274 | { |
275 | vm_page_t m; | |
276 | ||
f2c2051e | 277 | m = kmalloc(sizeof(*m), M_FICTITIOUS_PAGES, M_WAITOK|M_ZERO); |
984263bc | 278 | |
9e5e1578 MD |
279 | pmap_page_init(m); |
280 | ||
831a8507 | 281 | m->flags = PG_FICTITIOUS | PG_UNQUEUED; |
984263bc MD |
282 | m->valid = VM_PAGE_BITS_ALL; |
283 | m->dirty = 0; | |
984263bc MD |
284 | m->queue = PQ_NONE; |
285 | m->object = NULL; | |
286 | ||
bc0aa189 | 287 | m->busy_count = PBUSY_LOCKED; |
984263bc MD |
288 | m->wire_count = 1; |
289 | m->hold_count = 0; | |
290 | m->phys_addr = paddr; | |
7db38254 | 291 | m->pat_mode = pat_mode; |
984263bc | 292 | |
6ba5daf8 MD |
293 | spin_init(&m->spin, "dev_page"); |
294 | ||
984263bc MD |
295 | return (m); |
296 | } | |
297 | ||
27db6797 | 298 | /* |
8ee19f91 MD |
299 | * The caller does not need to hold dev_pager_mtx() but caller must ensure |
300 | * no page-use collision within the object. | |
27db6797 | 301 | */ |
984263bc | 302 | static void |
57e43348 | 303 | dev_pager_putfake(vm_page_t m) |
984263bc MD |
304 | { |
305 | if (!(m->flags & PG_FICTITIOUS)) | |
306 | panic("dev_pager_putfake: bad page"); | |
fad57d0e | 307 | KKASSERT(m->object == NULL); |
f2c2051e JH |
308 | KKASSERT(m->hold_count == 0); |
309 | kfree(m, M_FICTITIOUS_PAGES); | |
310 | } | |
311 | ||
312 | static int | |
313 | old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, | |
314 | vm_ooffset_t foff, struct ucred *cred, u_short *color) | |
315 | { | |
316 | unsigned int npages; | |
317 | vm_offset_t off; | |
318 | cdev_t dev; | |
319 | ||
320 | dev = handle; | |
321 | ||
322 | /* | |
323 | * Check that the specified range of the device allows the desired | |
324 | * protection. | |
325 | * | |
326 | * XXX assumes VM_PROT_* == PROT_* | |
327 | */ | |
328 | npages = OFF_TO_IDX(size); | |
329 | for (off = foff; npages--; off += PAGE_SIZE) { | |
8c530b23 | 330 | if (dev_dmmap(dev, off, (int)prot, NULL) == -1) |
f2c2051e JH |
331 | return (EINVAL); |
332 | } | |
333 | ||
334 | return (0); | |
335 | } | |
336 | ||
337 | static void old_dev_pager_dtor(void *handle) | |
338 | { | |
339 | cdev_t dev; | |
340 | ||
341 | dev = handle; | |
342 | if (dev != NULL) { | |
343 | KKASSERT(dev->si_object); | |
344 | dev->si_object = NULL; | |
345 | } | |
346 | } | |
347 | ||
831a8507 MD |
348 | static int |
349 | old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, | |
350 | int prot, vm_page_t *mres) | |
f2c2051e JH |
351 | { |
352 | vm_paddr_t paddr; | |
353 | vm_page_t page; | |
123f4114 | 354 | vm_offset_t pidx = OFF_TO_IDX(offset); |
f2c2051e JH |
355 | cdev_t dev; |
356 | ||
357 | page = *mres; | |
358 | dev = object->handle; | |
f2c2051e | 359 | |
76f1911e | 360 | paddr = pmap_phys_address(dev_dmmap(dev, offset, prot, NULL)); |
f2c2051e JH |
361 | KASSERT(paddr != -1,("dev_pager_getpage: map function returns error")); |
362 | KKASSERT(object->type == OBJT_DEVICE); | |
363 | ||
364 | if (page->flags & PG_FICTITIOUS) { | |
365 | /* | |
366 | * If the passed in reqpage page is already a fake page, | |
367 | * update it with the new physical address. | |
368 | */ | |
369 | page->phys_addr = paddr; | |
370 | page->valid = VM_PAGE_BITS_ALL; | |
371 | } else { | |
372 | /* | |
373 | * Replace the passed in reqpage page with our own fake page | |
8ee19f91 MD |
374 | * and free up all the original pages. Object lock must be |
375 | * held when manipulating devp_pglist and inserting the | |
376 | * page. | |
f2c2051e | 377 | */ |
9c89a01f | 378 | page = dev_pager_getfake(paddr, object->memattr); |
8ee19f91 | 379 | vm_object_hold(object); |
f2c2051e JH |
380 | TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, |
381 | page, pageq); | |
f2c2051e | 382 | vm_page_free(*mres); |
123f4114 | 383 | if (vm_page_insert(page, object, pidx) == FALSE) { |
f2c2051e | 384 | panic("dev_pager_getpage: page (%p,%016jx) exists", |
123f4114 | 385 | object, (uintmax_t)pidx); |
f2c2051e JH |
386 | } |
387 | vm_object_drop(object); | |
388 | } | |
f2c2051e | 389 | return (VM_PAGER_OK); |
984263bc | 390 | } |
fad57d0e | 391 |