Commit | Line | Data |
---|---|---|
984263bc | 1 | /* |
0ac0a48a MD |
2 | * (MPSAFE) |
3 | * | |
984263bc MD |
4 | * Copyright (c) 1990 University of Utah. |
5 | * Copyright (c) 1991 The Regents of the University of California. | |
6 | * All rights reserved. | |
7 | * Copyright (c) 1993, 1994 John S. Dyson | |
8 | * Copyright (c) 1995, David Greenman | |
9 | * | |
10 | * This code is derived from software contributed to Berkeley by | |
11 | * the Systems Programming Group of the University of Utah Computer | |
12 | * Science Department. | |
13 | * | |
14 | * Redistribution and use in source and binary forms, with or without | |
15 | * modification, are permitted provided that the following conditions | |
16 | * are met: | |
17 | * 1. Redistributions of source code must retain the above copyright | |
18 | * notice, this list of conditions and the following disclaimer. | |
19 | * 2. Redistributions in binary form must reproduce the above copyright | |
20 | * notice, this list of conditions and the following disclaimer in the | |
21 | * documentation and/or other materials provided with the distribution. | |
dc71b7ab | 22 | * 3. Neither the name of the University nor the names of its contributors |
984263bc MD |
23 | * may be used to endorse or promote products derived from this software |
24 | * without specific prior written permission. | |
25 | * | |
26 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |
27 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
28 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |
29 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |
30 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
31 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |
32 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |
33 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |
34 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |
35 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
36 | * SUCH DAMAGE. | |
37 | * | |
38 | * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 | |
39 | * $FreeBSD: src/sys/vm/vnode_pager.c,v 1.116.2.7 2002/12/31 09:34:51 dillon Exp $ | |
40 | */ | |
41 | ||
42 | /* | |
43 | * Page to/from files (vnodes). | |
44 | */ | |
45 | ||
46 | /* | |
47 | * TODO: | |
48 | * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will | |
49 | * greatly re-simplify the vnode_pager. | |
50 | */ | |
51 | ||
52 | #include <sys/param.h> | |
53 | #include <sys/systm.h> | |
13dd34d8 | 54 | #include <sys/uio.h> |
5fd012e0 | 55 | #include <sys/kernel.h> |
984263bc MD |
56 | #include <sys/proc.h> |
57 | #include <sys/vnode.h> | |
58 | #include <sys/mount.h> | |
59 | #include <sys/buf.h> | |
60 | #include <sys/vmmeter.h> | |
61 | #include <sys/conf.h> | |
5c5185ae SG |
62 | |
63 | #include <cpu/lwbuf.h> | |
984263bc MD |
64 | |
65 | #include <vm/vm.h> | |
66 | #include <vm/vm_object.h> | |
67 | #include <vm/vm_page.h> | |
68 | #include <vm/vm_pager.h> | |
69 | #include <vm/vm_map.h> | |
70 | #include <vm/vnode_pager.h> | |
8d292090 | 71 | #include <vm/swap_pager.h> |
984263bc MD |
72 | #include <vm/vm_extern.h> |
73 | ||
1a54183b MD |
74 | #include <vm/vm_page2.h> |
75 | ||
2d7bef58 SW |
76 | static pgo_dealloc_t vnode_pager_dealloc; |
77 | static pgo_getpage_t vnode_pager_getpage; | |
78 | static pgo_putpages_t vnode_pager_putpages; | |
79 | static pgo_haspage_t vnode_pager_haspage; | |
984263bc MD |
80 | |
81 | struct pagerops vnodepagerops = { | |
2d7bef58 SW |
82 | .pgo_dealloc = vnode_pager_dealloc, |
83 | .pgo_getpage = vnode_pager_getpage, | |
84 | .pgo_putpages = vnode_pager_putpages, | |
85 | .pgo_haspage = vnode_pager_haspage | |
984263bc MD |
86 | }; |
87 | ||
35f59bfa MD |
88 | static struct krate vbadrate = { 1 }; |
89 | static struct krate vresrate = { 1 }; | |
90 | ||
74d62460 | 91 | long vnode_pbuf_freecnt = -1; /* start out unlimited */ |
984263bc MD |
92 | |
93 | /* | |
b0d18f7d MD |
94 | * Allocate a VM object for a vnode, typically a regular file vnode. |
95 | * | |
96 | * Some additional information is required to generate a properly sized | |
97 | * object which covers the entire buffer cache buffer straddling the file | |
98 | * EOF. Userland does not see the extra pages as the VM fault code tests | |
99 | * against v_filesize. | |
984263bc MD |
100 | */ |
101 | vm_object_t | |
b0d18f7d MD |
102 | vnode_pager_alloc(void *handle, off_t length, vm_prot_t prot, off_t offset, |
103 | int blksize, int boff) | |
984263bc MD |
104 | { |
105 | vm_object_t object; | |
106 | struct vnode *vp; | |
b0d18f7d MD |
107 | off_t loffset; |
108 | vm_pindex_t lsize; | |
984263bc MD |
109 | |
110 | /* | |
111 | * Pageout to vnode, no can do yet. | |
112 | */ | |
113 | if (handle == NULL) | |
114 | return (NULL); | |
115 | ||
116 | /* | |
117 | * XXX hack - This initialization should be put somewhere else. | |
118 | */ | |
119 | if (vnode_pbuf_freecnt < 0) { | |
2f0acc22 | 120 | vnode_pbuf_freecnt = nswbuf_kva / 2 + 1; |
984263bc MD |
121 | } |
122 | ||
2de4f77e MD |
123 | /* |
124 | * Serialize potential vnode/object teardowns and interlocks | |
125 | */ | |
126 | vp = (struct vnode *)handle; | |
b12defdc | 127 | lwkt_gettoken(&vp->v_token); |
984263bc | 128 | |
984263bc MD |
129 | /* |
130 | * If the object is being terminated, wait for it to | |
131 | * go away. | |
132 | */ | |
2734d278 MD |
133 | object = vp->v_object; |
134 | if (object) { | |
b12defdc | 135 | vm_object_hold(object); |
2734d278 | 136 | KKASSERT((object->flags & OBJ_DEAD) == 0); |
984263bc MD |
137 | } |
138 | ||
ee173d09 | 139 | if (VREFCNT(vp) <= 0) |
984263bc MD |
140 | panic("vnode_pager_alloc: no vnode reference"); |
141 | ||
b0d18f7d MD |
142 | /* |
143 | * Round up to the *next* block, then destroy the buffers in question. | |
144 | * Since we are only removing some of the buffers we must rely on the | |
145 | * scan count to determine whether a loop is necessary. | |
146 | * | |
147 | * Destroy any pages beyond the last buffer. | |
148 | */ | |
149 | if (boff < 0) | |
150 | boff = (int)(length % blksize); | |
151 | if (boff) | |
152 | loffset = length + (blksize - boff); | |
153 | else | |
154 | loffset = length; | |
155 | lsize = OFF_TO_IDX(round_page64(loffset)); | |
156 | ||
984263bc MD |
157 | if (object == NULL) { |
158 | /* | |
159 | * And an object of the appropriate size | |
160 | */ | |
2b9050f5 | 161 | object = vm_object_allocate_hold(OBJT_VNODE, lsize); |
984263bc MD |
162 | object->handle = handle; |
163 | vp->v_object = object; | |
b0d18f7d | 164 | vp->v_filesize = length; |
2bc7505b | 165 | if (vp->v_mount && (vp->v_mount->mnt_kern_flag & MNTK_NOMSYNC)) |
2a418930 | 166 | vm_object_set_flag(object, OBJ_NOMSYNC); |
2734d278 | 167 | vref(vp); |
984263bc | 168 | } else { |
2734d278 | 169 | vm_object_reference_quick(object); /* also vref's */ |
b0d18f7d MD |
170 | if (object->size != lsize) { |
171 | kprintf("vnode_pager_alloc: Warning, objsize " | |
172 | "mismatch %jd/%jd vp=%p obj=%p\n", | |
173 | (intmax_t)object->size, | |
174 | (intmax_t)lsize, | |
175 | vp, object); | |
176 | } | |
177 | if (vp->v_filesize != length) { | |
973c11b9 | 178 | kprintf("vnode_pager_alloc: Warning, filesize " |
b0d18f7d MD |
179 | "mismatch %jd/%jd vp=%p obj=%p\n", |
180 | (intmax_t)vp->v_filesize, | |
181 | (intmax_t)length, | |
182 | vp, object); | |
973c11b9 | 183 | } |
984263bc | 184 | } |
b12defdc | 185 | vm_object_drop(object); |
2734d278 | 186 | lwkt_reltoken(&vp->v_token); |
2de4f77e | 187 | |
5a648714 MD |
188 | return (object); |
189 | } | |
190 | ||
191 | /* | |
192 | * Add a ref to a vnode's existing VM object, return the object or | |
193 | * NULL if the vnode did not have one. This does not create the | |
194 | * object (we can't since we don't know what the proper blocksize/boff | |
195 | * is to match the VFS's use of the buffer cache). | |
2734d278 MD |
196 | * |
197 | * The vnode must be referenced and is typically open. The object should | |
198 | * be stable in this situation. | |
199 | * | |
200 | * Returns the object with an additional reference but not locked. | |
5a648714 MD |
201 | */ |
202 | vm_object_t | |
203 | vnode_pager_reference(struct vnode *vp) | |
204 | { | |
205 | vm_object_t object; | |
206 | ||
2734d278 MD |
207 | if ((object = vp->v_object) != NULL) |
208 | vm_object_reference_quick(object); /* also vref's vnode */ | |
984263bc MD |
209 | return (object); |
210 | } | |
211 | ||
212 | static void | |
57e43348 | 213 | vnode_pager_dealloc(vm_object_t object) |
984263bc | 214 | { |
5f910b2f | 215 | struct vnode *vp = object->handle; |
984263bc MD |
216 | |
217 | if (vp == NULL) | |
218 | panic("vnode_pager_dealloc: pager already dealloced"); | |
219 | ||
220 | vm_object_pip_wait(object, "vnpdea"); | |
221 | ||
222 | object->handle = NULL; | |
223 | object->type = OBJT_DEAD; | |
224 | vp->v_object = NULL; | |
57f7b636 | 225 | vp->v_filesize = NOOFFSET; |
2247fe02 | 226 | vclrflags(vp, VTEXT | VOBJBUF); |
8d292090 | 227 | swap_pager_freespace_all(object); |
984263bc MD |
228 | } |
229 | ||
54078292 | 230 | /* |
94ff63e3 | 231 | * Return whether the vnode pager has the requested page. |
54078292 | 232 | */ |
984263bc | 233 | static boolean_t |
1b9d3514 | 234 | vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex) |
984263bc MD |
235 | { |
236 | struct vnode *vp = object->handle; | |
54078292 MD |
237 | off_t loffset; |
238 | off_t doffset; | |
239 | int voff; | |
984263bc | 240 | int bsize; |
54078292 | 241 | int error; |
984263bc MD |
242 | |
243 | /* | |
244 | * If no vp or vp is doomed or marked transparent to VM, we do not | |
245 | * have the page. | |
246 | */ | |
5fd012e0 | 247 | if ((vp == NULL) || (vp->v_flag & VRECLAIMED)) |
984263bc MD |
248 | return FALSE; |
249 | ||
250 | /* | |
251 | * If filesystem no longer mounted or offset beyond end of file we do | |
252 | * not have the page. | |
253 | */ | |
54078292 MD |
254 | loffset = IDX_TO_OFF(pindex); |
255 | ||
57f7b636 | 256 | if (vp->v_mount == NULL || loffset >= vp->v_filesize) |
984263bc MD |
257 | return FALSE; |
258 | ||
259 | bsize = vp->v_mount->mnt_stat.f_iosize; | |
54078292 MD |
260 | voff = loffset % bsize; |
261 | ||
bc823b32 | 262 | /* |
94ff63e3 | 263 | * XXX (obsolete - before and after pointers are now NULL) |
1b9d3514 | 264 | * |
bc823b32 MD |
265 | * BMAP returns byte counts before and after, where after |
266 | * is inclusive of the base page. haspage must return page | |
267 | * counts before and after where after does not include the | |
268 | * base page. | |
269 | * | |
270 | * BMAP is allowed to return a *after of 0 for backwards | |
271 | * compatibility. The base page is still considered valid if | |
272 | * no error is returned. | |
273 | */ | |
1b9d3514 MD |
274 | error = VOP_BMAP(vp, loffset - voff, &doffset, NULL, NULL, 0); |
275 | if (error) | |
984263bc | 276 | return TRUE; |
54078292 | 277 | if (doffset == NOOFFSET) |
984263bc | 278 | return FALSE; |
984263bc MD |
279 | return TRUE; |
280 | } | |
281 | ||
282 | /* | |
283 | * Lets the VM system know about a change in size for a file. | |
284 | * We adjust our own internal size and flush any cached pages in | |
285 | * the associated object that are affected by the size change. | |
286 | * | |
57f7b636 | 287 | * NOTE: This routine may be invoked as a result of a pager put |
984263bc | 288 | * operation (possibly at object termination time), so we must be careful. |
57f7b636 MD |
289 | * |
290 | * NOTE: vp->v_filesize is initialized to NOOFFSET (-1), be sure that | |
291 | * we do not blow up on the case. nsize will always be >= 0, however. | |
984263bc MD |
292 | */ |
293 | void | |
57e43348 | 294 | vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize) |
984263bc MD |
295 | { |
296 | vm_pindex_t nobjsize; | |
c4b46cb4 | 297 | vm_pindex_t oobjsize; |
b12defdc | 298 | vm_object_t object; |
984263bc | 299 | |
2734d278 | 300 | object = vp->v_object; |
984263bc MD |
301 | if (object == NULL) |
302 | return; | |
2734d278 MD |
303 | vm_object_hold(object); |
304 | KKASSERT(vp->v_object == object); | |
984263bc MD |
305 | |
306 | /* | |
307 | * Hasn't changed size | |
308 | */ | |
b12defdc MD |
309 | if (nsize == vp->v_filesize) { |
310 | vm_object_drop(object); | |
984263bc | 311 | return; |
b12defdc | 312 | } |
573fb415 | 313 | |
c4b46cb4 MD |
314 | /* |
315 | * Has changed size. Adjust the VM object's size and v_filesize | |
316 | * before we start scanning pages to prevent new pages from being | |
317 | * allocated during the scan. | |
318 | */ | |
984263bc | 319 | nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); |
c4b46cb4 MD |
320 | oobjsize = object->size; |
321 | object->size = nobjsize; | |
984263bc MD |
322 | |
323 | /* | |
324 | * File has shrunk. Toss any cached pages beyond the new EOF. | |
325 | */ | |
57f7b636 | 326 | if (nsize < vp->v_filesize) { |
135bd6a8 | 327 | vp->v_filesize = nsize; |
c4b46cb4 MD |
328 | if (nobjsize < oobjsize) { |
329 | vm_object_page_remove(object, nobjsize, oobjsize, | |
330 | FALSE); | |
984263bc MD |
331 | } |
332 | /* | |
8d429613 MD |
333 | * This gets rid of garbage at the end of a page that is now |
334 | * only partially backed by the vnode. Since we are setting | |
335 | * the entire page valid & clean after we are done we have | |
336 | * to be sure that the portion of the page within the file | |
337 | * bounds is already valid. If it isn't then making it | |
338 | * valid would create a corrupt block. | |
984263bc MD |
339 | */ |
340 | if (nsize & PAGE_MASK) { | |
341 | vm_offset_t kva; | |
342 | vm_page_t m; | |
343 | ||
b12defdc MD |
344 | m = vm_page_lookup_busy_wait(object, OFF_TO_IDX(nsize), |
345 | TRUE, "vsetsz"); | |
17cde63e | 346 | |
984263bc MD |
347 | if (m && m->valid) { |
348 | int base = (int)nsize & PAGE_MASK; | |
349 | int size = PAGE_SIZE - base; | |
5c5185ae | 350 | struct lwbuf *lwb; |
7a683a24 | 351 | struct lwbuf lwb_cache; |
984263bc MD |
352 | |
353 | /* | |
354 | * Clear out partial-page garbage in case | |
355 | * the page has been mapped. | |
c7841cbe MD |
356 | * |
357 | * This is byte aligned. | |
984263bc | 358 | */ |
7a683a24 | 359 | lwb = lwbuf_alloc(m, &lwb_cache); |
5c5185ae | 360 | kva = lwbuf_kva(lwb); |
984263bc | 361 | bzero((caddr_t)kva + base, size); |
5c5185ae | 362 | lwbuf_free(lwb); |
984263bc MD |
363 | |
364 | /* | |
365 | * XXX work around SMP data integrity race | |
366 | * by unmapping the page from user processes. | |
367 | * The garbage we just cleared may be mapped | |
368 | * to a user process running on another cpu | |
369 | * and this code is not running through normal | |
370 | * I/O channels which handle SMP issues for | |
371 | * us, so unmap page to synchronize all cpus. | |
372 | * | |
373 | * XXX should vm_pager_unmap_page() have | |
374 | * dealt with this? | |
375 | */ | |
376 | vm_page_protect(m, VM_PROT_NONE); | |
377 | ||
378 | /* | |
379 | * Clear out partial-page dirty bits. This | |
380 | * has the side effect of setting the valid | |
381 | * bits, but that is ok. There are a bunch | |
382 | * of places in the VM system where we expected | |
383 | * m->dirty == VM_PAGE_BITS_ALL. The file EOF | |
384 | * case is one of them. If the page is still | |
385 | * partially dirty, make it fully dirty. | |
386 | * | |
c7841cbe | 387 | * NOTE: We do not clear out the valid |
984263bc MD |
388 | * bits. This would prevent bogus_page |
389 | * replacement from working properly. | |
c7841cbe MD |
390 | * |
391 | * NOTE: We do not want to clear the dirty | |
392 | * bit for a partial DEV_BSIZE'd truncation! | |
393 | * This is DEV_BSIZE aligned! | |
984263bc | 394 | */ |
1a54183b | 395 | vm_page_clear_dirty_beg_nonincl(m, base, size); |
984263bc MD |
396 | if (m->dirty != 0) |
397 | m->dirty = VM_PAGE_BITS_ALL; | |
17cde63e | 398 | vm_page_wakeup(m); |
b12defdc MD |
399 | } else if (m) { |
400 | vm_page_wakeup(m); | |
984263bc MD |
401 | } |
402 | } | |
135bd6a8 MD |
403 | } else { |
404 | vp->v_filesize = nsize; | |
984263bc | 405 | } |
b12defdc | 406 | vm_object_drop(object); |
984263bc MD |
407 | } |
408 | ||
a55afca2 MD |
409 | /* |
410 | * Release a page busied for a getpages operation. The page may have become | |
411 | * wired (typically due to being used by the buffer cache) or otherwise been | |
412 | * soft-busied and cannot be freed in that case. A held page can still be | |
413 | * freed. | |
414 | */ | |
984263bc | 415 | void |
57e43348 | 416 | vnode_pager_freepage(vm_page_t m) |
984263bc | 417 | { |
bc0aa189 MD |
418 | if ((m->busy_count & PBUSY_MASK) || |
419 | m->wire_count || | |
420 | (m->flags & PG_NEED_COMMIT)) { | |
a55afca2 MD |
421 | vm_page_activate(m); |
422 | vm_page_wakeup(m); | |
423 | } else { | |
424 | vm_page_free(m); | |
425 | } | |
984263bc MD |
426 | } |
427 | ||
984263bc MD |
428 | /* |
429 | * EOPNOTSUPP is no longer legal. For local media VFS's that do not | |
430 | * implement their own VOP_GETPAGES, their VOP_GETPAGES should call to | |
431 | * vnode_pager_generic_getpages() to implement the previous behaviour. | |
432 | * | |
433 | * All other FS's should use the bypass to get to the local media | |
434 | * backing vp's VOP_GETPAGES. | |
435 | */ | |
436 | static int | |
2d396014 MD |
437 | vnode_pager_getpage(vm_object_t object, vm_pindex_t pindex, |
438 | vm_page_t *mpp, int seqaccess) | |
984263bc MD |
439 | { |
440 | int rtval; | |
441 | struct vnode *vp; | |
984263bc MD |
442 | |
443 | vp = object->handle; | |
1c48c952 | 444 | rtval = VOP_GETPAGES(vp, mpp, PAGE_SIZE, 0, 0, seqaccess); |
a05863a7 | 445 | if (rtval == EOPNOTSUPP) |
ed20d0e3 | 446 | panic("vnode_pager: vfs's must implement vop_getpages"); |
984263bc MD |
447 | return rtval; |
448 | } | |
449 | ||
984263bc MD |
450 | /* |
451 | * This is now called from local media FS's to operate against their | |
452 | * own vnodes if they fail to implement VOP_GETPAGES. | |
a05863a7 MD |
453 | * |
454 | * With all the caching local media devices do these days there is really | |
455 | * very little point to attempting to restrict the I/O size to contiguous | |
456 | * blocks on-disk, especially if our caller thinks we need all the specified | |
457 | * pages. Just construct and issue a READ. | |
984263bc MD |
458 | */ |
459 | int | |
1b9d3514 | 460 | vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *mpp, int bytecount, |
1c48c952 | 461 | int reqpage, int seqaccess) |
984263bc | 462 | { |
a05863a7 MD |
463 | struct iovec aiov; |
464 | struct uio auio; | |
465 | off_t foff; | |
466 | int error; | |
984263bc | 467 | int count; |
a05863a7 MD |
468 | int i; |
469 | int ioflags; | |
52e79afa | 470 | int obytecount; |
984263bc MD |
471 | |
472 | /* | |
a05863a7 | 473 | * Do not do anything if the vnode is bad. |
984263bc | 474 | */ |
a05863a7 MD |
475 | if (vp->v_mount == NULL) |
476 | return VM_PAGER_BAD; | |
984263bc MD |
477 | |
478 | /* | |
a05863a7 MD |
479 | * Calculate the number of pages. Since we are paging in whole |
480 | * pages, adjust bytecount to be an integral multiple of the page | |
481 | * size. It will be clipped to the file EOF later on. | |
984263bc | 482 | */ |
a05863a7 MD |
483 | bytecount = round_page(bytecount); |
484 | count = bytecount / PAGE_SIZE; | |
984263bc MD |
485 | |
486 | /* | |
cf1bb2a8 MD |
487 | * We could check m[reqpage]->valid here and shortcut the operation, |
488 | * but doing so breaks read-ahead. Instead assume that the VM | |
489 | * system has already done at least the check, don't worry about | |
490 | * any races, and issue the VOP_READ to allow read-ahead to function. | |
8d429613 | 491 | * |
cf1bb2a8 MD |
492 | * This keeps the pipeline full for I/O bound sequentially scanned |
493 | * mmap()'s | |
984263bc | 494 | */ |
cf1bb2a8 | 495 | /* don't shortcut */ |
984263bc MD |
496 | |
497 | /* | |
a05863a7 MD |
498 | * Discard pages past the file EOF. If the requested page is past |
499 | * the file EOF we just leave its valid bits set to 0, the caller | |
500 | * expects to maintain ownership of the requested page. If the | |
501 | * entire range is past file EOF discard everything and generate | |
502 | * a pagein error. | |
984263bc | 503 | */ |
1b9d3514 | 504 | foff = IDX_TO_OFF(mpp[0]->pindex); |
a05863a7 MD |
505 | if (foff >= vp->v_filesize) { |
506 | for (i = 0; i < count; i++) { | |
507 | if (i != reqpage) | |
1b9d3514 | 508 | vnode_pager_freepage(mpp[i]); |
984263bc | 509 | } |
a05863a7 | 510 | return VM_PAGER_ERROR; |
984263bc MD |
511 | } |
512 | ||
a05863a7 MD |
513 | if (foff + bytecount > vp->v_filesize) { |
514 | bytecount = vp->v_filesize - foff; | |
515 | i = round_page(bytecount) / PAGE_SIZE; | |
516 | while (count > i) { | |
517 | --count; | |
518 | if (count != reqpage) | |
1b9d3514 | 519 | vnode_pager_freepage(mpp[count]); |
984263bc | 520 | } |
984263bc MD |
521 | } |
522 | ||
523 | /* | |
a05863a7 MD |
524 | * The size of the transfer is bytecount. bytecount will be an |
525 | * integral multiple of the page size unless it has been clipped | |
526 | * to the file EOF. The transfer cannot exceed the file EOF. | |
527 | * | |
528 | * When dealing with real devices we must round-up to the device | |
529 | * sector size. | |
984263bc | 530 | */ |
a05863a7 MD |
531 | if (vp->v_type == VBLK || vp->v_type == VCHR) { |
532 | int secmask = vp->v_rdev->si_bsize_phys - 1; | |
ed20d0e3 | 533 | KASSERT(secmask < PAGE_SIZE, ("vnode_pager_generic_getpages: sector size %d too large", secmask + 1)); |
a05863a7 | 534 | bytecount = (bytecount + secmask) & ~secmask; |
984263bc | 535 | } |
52e79afa | 536 | obytecount = bytecount; |
984263bc | 537 | |
ca28958c MD |
538 | /* |
539 | * Severe hack to avoid deadlocks with the buffer cache | |
540 | */ | |
541 | for (i = 0; i < count; ++i) { | |
1b9d3514 | 542 | vm_page_t mt = mpp[i]; |
ca28958c MD |
543 | |
544 | vm_page_io_start(mt); | |
545 | vm_page_wakeup(mt); | |
546 | } | |
547 | ||
984263bc | 548 | /* |
cf1bb2a8 | 549 | * Issue the I/O with some read-ahead if bytecount > PAGE_SIZE |
984263bc | 550 | */ |
a05863a7 | 551 | ioflags = IO_VMIO; |
1c48c952 | 552 | if (seqaccess) |
cf1bb2a8 | 553 | ioflags |= IO_SEQMAX << IO_SEQSHIFT; |
a05863a7 | 554 | |
1b9d3514 | 555 | aiov.iov_base = NULL; |
a05863a7 MD |
556 | aiov.iov_len = bytecount; |
557 | auio.uio_iov = &aiov; | |
558 | auio.uio_iovcnt = 1; | |
559 | auio.uio_offset = foff; | |
560 | auio.uio_segflg = UIO_NOCOPY; | |
561 | auio.uio_rw = UIO_READ; | |
562 | auio.uio_resid = bytecount; | |
563 | auio.uio_td = NULL; | |
12e4aaff MD |
564 | mycpu->gd_cnt.v_vnodein++; |
565 | mycpu->gd_cnt.v_vnodepgsin += count; | |
984263bc | 566 | |
a05863a7 | 567 | error = VOP_READ(vp, &auio, ioflags, proc0.p_ucred); |
984263bc | 568 | |
ca28958c MD |
569 | /* |
570 | * Severe hack to avoid deadlocks with the buffer cache | |
571 | */ | |
572 | for (i = 0; i < count; ++i) { | |
b12defdc MD |
573 | vm_page_busy_wait(mpp[i], FALSE, "getpgs"); |
574 | vm_page_io_finish(mpp[i]); | |
ca28958c MD |
575 | } |
576 | ||
984263bc | 577 | /* |
a05863a7 | 578 | * Calculate the actual number of bytes read and clean up the |
13dd34d8 | 579 | * page list. |
984263bc | 580 | */ |
a05863a7 | 581 | bytecount -= auio.uio_resid; |
984263bc | 582 | |
a05863a7 | 583 | for (i = 0; i < count; ++i) { |
1b9d3514 | 584 | vm_page_t mt = mpp[i]; |
a55afca2 | 585 | |
a05863a7 | 586 | if (i != reqpage) { |
a55afca2 | 587 | if (error == 0 && mt->valid) { |
b12defdc | 588 | if (mt->flags & PG_REFERENCED) |
984263bc MD |
589 | vm_page_activate(mt); |
590 | else | |
591 | vm_page_deactivate(mt); | |
592 | vm_page_wakeup(mt); | |
593 | } else { | |
594 | vnode_pager_freepage(mt); | |
595 | } | |
a55afca2 MD |
596 | } else if (mt->valid == 0) { |
597 | if (error == 0) { | |
fc9ed34d MD |
598 | kprintf("page failed but no I/O error page " |
599 | "%p object %p pindex %d\n", | |
600 | mt, mt->object, (int) mt->pindex); | |
35163fef | 601 | kprintf("i=%d foff=%016lx bytecount=%d/%d " |
52e79afa MD |
602 | "uioresid=%zd\n", |
603 | i, foff, obytecount, bytecount, | |
604 | auio.uio_resid); | |
a55afca2 MD |
605 | /* whoops, something happened */ |
606 | error = EINVAL; | |
607 | } | |
608 | } else if (mt->valid != VM_PAGE_BITS_ALL) { | |
609 | /* | |
610 | * Zero-extend the requested page if necessary (if | |
611 | * the filesystem is using a small block size). | |
612 | */ | |
613 | vm_page_zero_invalid(mt, TRUE); | |
984263bc MD |
614 | } |
615 | } | |
616 | if (error) { | |
1b9d3514 | 617 | kprintf("vnode_pager_getpage: I/O read error\n"); |
984263bc MD |
618 | } |
619 | return (error ? VM_PAGER_ERROR : VM_PAGER_OK); | |
620 | } | |
621 | ||
622 | /* | |
623 | * EOPNOTSUPP is no longer legal. For local media VFS's that do not | |
624 | * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to | |
625 | * vnode_pager_generic_putpages() to implement the previous behaviour. | |
626 | * | |
cb1cf930 MD |
627 | * Caller has already cleared the pmap modified bits, if any. |
628 | * | |
984263bc MD |
629 | * All other FS's should use the bypass to get to the local media |
630 | * backing vp's VOP_PUTPAGES. | |
631 | */ | |
632 | static void | |
57e43348 | 633 | vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count, |
df0b0ead | 634 | int flags, int *rtvals) |
984263bc MD |
635 | { |
636 | int rtval; | |
637 | struct vnode *vp; | |
638 | int bytes = count * PAGE_SIZE; | |
639 | ||
640 | /* | |
641 | * Force synchronous operation if we are extremely low on memory | |
642 | * to prevent a low-memory deadlock. VOP operations often need to | |
13dd34d8 | 643 | * allocate more memory to initiate the I/O ( i.e. do a BMAP |
984263bc MD |
644 | * operation ). The swapper handles the case by limiting the amount |
645 | * of asynchronous I/O, but that sort of solution doesn't scale well | |
646 | * for the vnode pager without a lot of work. | |
647 | * | |
648 | * Also, the backing vnode's iodone routine may not wake the pageout | |
649 | * daemon up. This should be probably be addressed XXX. | |
650 | */ | |
651 | ||
0a4d4828 MD |
652 | if ((vmstats.v_free_count + vmstats.v_cache_count) < |
653 | vmstats.v_pageout_free_min) { | |
df0b0ead | 654 | flags |= OBJPC_SYNC; |
0a4d4828 | 655 | } |
984263bc MD |
656 | |
657 | /* | |
658 | * Call device-specific putpages function | |
659 | */ | |
984263bc | 660 | vp = object->handle; |
df0b0ead | 661 | rtval = VOP_PUTPAGES(vp, m, bytes, flags, rtvals, 0); |
984263bc | 662 | if (rtval == EOPNOTSUPP) { |
086c1d7e | 663 | kprintf("vnode_pager: *** WARNING *** stale FS putpages\n"); |
df0b0ead | 664 | rtval = vnode_pager_generic_putpages( vp, m, bytes, flags, rtvals); |
984263bc MD |
665 | } |
666 | } | |
667 | ||
668 | ||
669 | /* | |
670 | * This is now called from local media FS's to operate against their | |
671 | * own vnodes if they fail to implement VOP_PUTPAGES. | |
672 | * | |
673 | * This is typically called indirectly via the pageout daemon and | |
674 | * clustering has already typically occured, so in general we ask the | |
675 | * underlying filesystem to write the data out asynchronously rather | |
676 | * then delayed. | |
677 | */ | |
678 | int | |
57e43348 | 679 | vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int bytecount, |
a05863a7 | 680 | int flags, int *rtvals) |
984263bc MD |
681 | { |
682 | int i; | |
cb1cf930 | 683 | int maxsize, ncount, count; |
984263bc MD |
684 | vm_ooffset_t poffset; |
685 | struct uio auio; | |
686 | struct iovec aiov; | |
687 | int error; | |
688 | int ioflags; | |
689 | ||
984263bc MD |
690 | count = bytecount / PAGE_SIZE; |
691 | ||
692 | for (i = 0; i < count; i++) | |
693 | rtvals[i] = VM_PAGER_AGAIN; | |
694 | ||
695 | if ((int) m[0]->pindex < 0) { | |
086c1d7e | 696 | kprintf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%x)\n", |
984263bc MD |
697 | (long)m[0]->pindex, m[0]->dirty); |
698 | rtvals[0] = VM_PAGER_BAD; | |
699 | return VM_PAGER_BAD; | |
700 | } | |
701 | ||
702 | maxsize = count * PAGE_SIZE; | |
703 | ncount = count; | |
704 | ||
705 | poffset = IDX_TO_OFF(m[0]->pindex); | |
706 | ||
707 | /* | |
708 | * If the page-aligned write is larger then the actual file we | |
cb1cf930 MD |
709 | * have to invalidate pages occuring beyond the file EOF. |
710 | * | |
711 | * If the file EOF resides in the middle of a page we still clear | |
712 | * all of that page's dirty bits later on. If we didn't it would | |
713 | * endlessly re-write. | |
984263bc MD |
714 | * |
715 | * We do not under any circumstances truncate the valid bits, as | |
716 | * this will screw up bogus page replacement. | |
93afe6be MD |
717 | * |
718 | * The caller has already read-protected the pages. The VFS must | |
719 | * use the buffer cache to wrap the pages. The pages might not | |
720 | * be immediately flushed by the buffer cache but once under its | |
721 | * control the pages themselves can wind up being marked clean | |
722 | * and their covering buffer cache buffer can be marked dirty. | |
984263bc | 723 | */ |
fb69d1c8 MD |
724 | if (poffset + maxsize > vp->v_filesize) { |
725 | if (poffset < vp->v_filesize) { | |
57f7b636 | 726 | maxsize = vp->v_filesize - poffset; |
984263bc | 727 | ncount = btoc(maxsize); |
984263bc MD |
728 | } else { |
729 | maxsize = 0; | |
730 | ncount = 0; | |
731 | } | |
732 | if (ncount < count) { | |
733 | for (i = ncount; i < count; i++) { | |
734 | rtvals[i] = VM_PAGER_BAD; | |
735 | } | |
736 | } | |
737 | } | |
738 | ||
739 | /* | |
740 | * pageouts are already clustered, use IO_ASYNC to force a bawrite() | |
741 | * rather then a bdwrite() to prevent paging I/O from saturating | |
742 | * the buffer cache. Dummy-up the sequential heuristic to cause | |
743 | * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, | |
744 | * the system decides how to cluster. | |
745 | */ | |
746 | ioflags = IO_VMIO; | |
cdf89dcf | 747 | if (flags & (OBJPC_SYNC | OBJPC_INVAL)) |
984263bc | 748 | ioflags |= IO_SYNC; |
cdf89dcf | 749 | else if ((flags & OBJPC_CLUSTER_OK) == 0) |
984263bc | 750 | ioflags |= IO_ASYNC; |
cdf89dcf | 751 | ioflags |= (flags & OBJPC_INVAL) ? IO_INVAL: 0; |
984263bc MD |
752 | ioflags |= IO_SEQMAX << IO_SEQSHIFT; |
753 | ||
754 | aiov.iov_base = (caddr_t) 0; | |
755 | aiov.iov_len = maxsize; | |
756 | auio.uio_iov = &aiov; | |
757 | auio.uio_iovcnt = 1; | |
758 | auio.uio_offset = poffset; | |
759 | auio.uio_segflg = UIO_NOCOPY; | |
760 | auio.uio_rw = UIO_WRITE; | |
761 | auio.uio_resid = maxsize; | |
dadab5e9 | 762 | auio.uio_td = NULL; |
8a8d5d85 | 763 | error = VOP_WRITE(vp, &auio, ioflags, proc0.p_ucred); |
12e4aaff MD |
764 | mycpu->gd_cnt.v_vnodeout++; |
765 | mycpu->gd_cnt.v_vnodepgsout += ncount; | |
984263bc MD |
766 | |
767 | if (error) { | |
35f59bfa MD |
768 | krateprintf(&vbadrate, |
769 | "vnode_pager_putpages: I/O error %d\n", error); | |
984263bc MD |
770 | } |
771 | if (auio.uio_resid) { | |
35f59bfa | 772 | krateprintf(&vresrate, |
bfc09ba0 | 773 | "vnode_pager_putpages: residual I/O %zd at %lu\n", |
35f59bfa | 774 | auio.uio_resid, (u_long)m[0]->pindex); |
984263bc | 775 | } |
cb1cf930 MD |
776 | if (error == 0) { |
777 | for (i = 0; i < ncount; i++) { | |
778 | rtvals[i] = VM_PAGER_OK; | |
779 | vm_page_undirty(m[i]); | |
780 | } | |
781 | } | |
984263bc MD |
782 | return rtvals[0]; |
783 | } | |
784 | ||
b12defdc MD |
785 | /* |
786 | * Run the chain and if the bottom-most object is a vnode-type lock the | |
787 | * underlying vnode. A locked vnode or NULL is returned. | |
9de48ead MD |
788 | * |
789 | * Caller must hold the first object. | |
b12defdc | 790 | */ |
984263bc | 791 | struct vnode * |
44293a80 | 792 | vnode_pager_lock(vm_map_backing_t ba) |
984263bc | 793 | { |
44293a80 | 794 | vm_map_backing_t lba; |
2d396014 | 795 | struct vnode *vp = NULL; |
b12defdc | 796 | vm_object_t lobject; |
5fd012e0 | 797 | int error; |
984263bc | 798 | |
9de48ead MD |
799 | if (ba == NULL) |
800 | return NULL; | |
801 | lba = ba; | |
802 | while (lba->backing_ba) | |
803 | lba = lba->backing_ba; | |
804 | if ((lobject = lba->object) == NULL) | |
805 | return NULL; | |
806 | if (lba != ba) | |
807 | vm_object_hold_shared(lobject); | |
808 | ||
b12defdc MD |
809 | while (lobject->type == OBJT_VNODE && |
810 | (lobject->flags & OBJ_DEAD) == 0) { | |
811 | /* | |
812 | * Extract the vp | |
813 | */ | |
814 | vp = lobject->handle; | |
815 | error = vget(vp, LK_SHARED | LK_RETRY | LK_CANRECURSE); | |
816 | if (error == 0) { | |
817 | if (lobject->handle == vp) | |
818 | break; | |
819 | vput(vp); | |
820 | } else { | |
821 | kprintf("vnode_pager_lock: vp %p error %d " | |
822 | "lockstatus %d, retrying\n", | |
823 | vp, error, | |
824 | lockstatus(&vp->v_lock, curthread)); | |
9de48ead | 825 | tsleep(lobject->handle, 0, "vnpgrl", hz); |
984263bc | 826 | } |
b12defdc | 827 | vp = NULL; |
984263bc | 828 | } |
9de48ead | 829 | if (lba != ba) |
b12defdc MD |
830 | vm_object_drop(lobject); |
831 | return (vp); | |
984263bc | 832 | } |