Rename printf -> kprintf in sys/ and add some defines where necessary
[dragonfly.git] / sys / vm / vnode_pager.c
CommitLineData
984263bc
MD
1/*
2 * Copyright (c) 1990 University of Utah.
3 * Copyright (c) 1991 The Regents of the University of California.
4 * All rights reserved.
5 * Copyright (c) 1993, 1994 John S. Dyson
6 * Copyright (c) 1995, David Greenman
7 *
8 * This code is derived from software contributed to Berkeley by
9 * the Systems Programming Group of the University of Utah Computer
10 * Science Department.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 *
40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91
41 * $FreeBSD: src/sys/vm/vnode_pager.c,v 1.116.2.7 2002/12/31 09:34:51 dillon Exp $
086c1d7e 42 * $DragonFly: src/sys/vm/vnode_pager.c,v 1.32 2006/12/23 00:41:31 swildner Exp $
984263bc
MD
43 */
44
45/*
46 * Page to/from files (vnodes).
47 */
48
49/*
50 * TODO:
51 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will
52 * greatly re-simplify the vnode_pager.
53 */
54
55#include <sys/param.h>
56#include <sys/systm.h>
5fd012e0 57#include <sys/kernel.h>
984263bc
MD
58#include <sys/proc.h>
59#include <sys/vnode.h>
60#include <sys/mount.h>
61#include <sys/buf.h>
62#include <sys/vmmeter.h>
63#include <sys/conf.h>
06ecca5a 64#include <sys/sfbuf.h>
cdd46d2e 65#include <sys/thread2.h>
984263bc
MD
66
67#include <vm/vm.h>
68#include <vm/vm_object.h>
69#include <vm/vm_page.h>
70#include <vm/vm_pager.h>
71#include <vm/vm_map.h>
72#include <vm/vnode_pager.h>
73#include <vm/vm_extern.h>
74
54078292 75static off_t vnode_pager_addr (struct vnode *vp, off_t loffset, int *run);
81b5c339 76static void vnode_pager_iodone (struct bio *bio);
1388df65
RG
77static int vnode_pager_input_smlfs (vm_object_t object, vm_page_t m);
78static int vnode_pager_input_old (vm_object_t object, vm_page_t m);
79static void vnode_pager_dealloc (vm_object_t);
80static int vnode_pager_getpages (vm_object_t, vm_page_t *, int, int);
81static void vnode_pager_putpages (vm_object_t, vm_page_t *, int, boolean_t, int *);
82static boolean_t vnode_pager_haspage (vm_object_t, vm_pindex_t, int *, int *);
984263bc
MD
83
84struct pagerops vnodepagerops = {
85 NULL,
86 vnode_pager_alloc,
87 vnode_pager_dealloc,
88 vnode_pager_getpages,
89 vnode_pager_putpages,
90 vnode_pager_haspage,
91 NULL
92};
93
94int vnode_pbuf_freecnt = -1; /* start out unlimited */
95
96/*
97 * Allocate (or lookup) pager for a vnode.
98 * Handle is a vnode pointer.
99 */
100vm_object_t
57f7b636 101vnode_pager_alloc(void *handle, off_t size, vm_prot_t prot, off_t offset)
984263bc
MD
102{
103 vm_object_t object;
104 struct vnode *vp;
105
106 /*
107 * Pageout to vnode, no can do yet.
108 */
109 if (handle == NULL)
110 return (NULL);
111
112 /*
113 * XXX hack - This initialization should be put somewhere else.
114 */
115 if (vnode_pbuf_freecnt < 0) {
116 vnode_pbuf_freecnt = nswbuf / 2 + 1;
117 }
118
119 vp = (struct vnode *) handle;
120
121 /*
122 * Prevent race condition when allocating the object. This
123 * can happen with NFS vnodes since the nfsnode isn't locked.
124 */
125 while (vp->v_flag & VOLOCK) {
126 vp->v_flag |= VOWANT;
377d4740 127 tsleep(vp, 0, "vnpobj", 0);
984263bc
MD
128 }
129 vp->v_flag |= VOLOCK;
130
131 /*
132 * If the object is being terminated, wait for it to
133 * go away.
134 */
135 while (((object = vp->v_object) != NULL) &&
136 (object->flags & OBJ_DEAD)) {
377d4740 137 tsleep(object, 0, "vadead", 0);
984263bc
MD
138 }
139
140 if (vp->v_usecount == 0)
141 panic("vnode_pager_alloc: no vnode reference");
142
143 if (object == NULL) {
144 /*
145 * And an object of the appropriate size
146 */
147 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size)));
148 object->flags = 0;
984263bc
MD
149 object->handle = handle;
150 vp->v_object = object;
57f7b636 151 vp->v_filesize = size;
984263bc
MD
152 } else {
153 object->ref_count++;
57f7b636 154 if (vp->v_filesize != size)
086c1d7e 155 kprintf("vnode_pager_alloc: Warning, filesize mismatch %lld/%lld\n", vp->v_filesize, size);
984263bc 156 }
57f7b636 157 vp->v_usecount++;
984263bc
MD
158
159 vp->v_flag &= ~VOLOCK;
160 if (vp->v_flag & VOWANT) {
161 vp->v_flag &= ~VOWANT;
162 wakeup(vp);
163 }
164 return (object);
165}
166
167static void
57e43348 168vnode_pager_dealloc(vm_object_t object)
984263bc 169{
5f910b2f 170 struct vnode *vp = object->handle;
984263bc
MD
171
172 if (vp == NULL)
173 panic("vnode_pager_dealloc: pager already dealloced");
174
175 vm_object_pip_wait(object, "vnpdea");
176
177 object->handle = NULL;
178 object->type = OBJT_DEAD;
179 vp->v_object = NULL;
57f7b636 180 vp->v_filesize = NOOFFSET;
984263bc
MD
181 vp->v_flag &= ~(VTEXT | VOBJBUF);
182}
183
54078292
MD
184/*
185 * Return whether the vnode pager has the requested page. Return the
186 * number of disk-contiguous pages before and after the requested page,
187 * not including the requested page.
188 */
984263bc 189static boolean_t
57e43348 190vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
54078292 191 int *after)
984263bc
MD
192{
193 struct vnode *vp = object->handle;
54078292
MD
194 off_t loffset;
195 off_t doffset;
196 int voff;
984263bc 197 int bsize;
54078292 198 int error;
984263bc
MD
199
200 /*
201 * If no vp or vp is doomed or marked transparent to VM, we do not
202 * have the page.
203 */
5fd012e0 204 if ((vp == NULL) || (vp->v_flag & VRECLAIMED))
984263bc
MD
205 return FALSE;
206
207 /*
208 * If filesystem no longer mounted or offset beyond end of file we do
209 * not have the page.
210 */
54078292
MD
211 loffset = IDX_TO_OFF(pindex);
212
57f7b636 213 if (vp->v_mount == NULL || loffset >= vp->v_filesize)
984263bc
MD
214 return FALSE;
215
216 bsize = vp->v_mount->mnt_stat.f_iosize;
54078292
MD
217 voff = loffset % bsize;
218
219 error = VOP_BMAP(vp, loffset - voff, NULL, &doffset, after, before);
220 if (error)
984263bc 221 return TRUE;
54078292 222 if (doffset == NOOFFSET)
984263bc 223 return FALSE;
984263bc 224
54078292
MD
225 if (before) {
226 *before = (*before + voff) >> PAGE_SHIFT;
227 }
228 if (after) {
229 *after -= voff;
57f7b636
MD
230 if (loffset + *after > vp->v_filesize)
231 *after = vp->v_filesize - loffset;
54078292
MD
232 *after >>= PAGE_SHIFT;
233 if (*after < 0)
234 *after = 0;
984263bc
MD
235 }
236 return TRUE;
237}
238
239/*
240 * Lets the VM system know about a change in size for a file.
241 * We adjust our own internal size and flush any cached pages in
242 * the associated object that are affected by the size change.
243 *
57f7b636 244 * NOTE: This routine may be invoked as a result of a pager put
984263bc 245 * operation (possibly at object termination time), so we must be careful.
57f7b636
MD
246 *
247 * NOTE: vp->v_filesize is initialized to NOOFFSET (-1), be sure that
248 * we do not blow up on the case. nsize will always be >= 0, however.
984263bc
MD
249 */
250void
57e43348 251vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
984263bc
MD
252{
253 vm_pindex_t nobjsize;
c4b46cb4 254 vm_pindex_t oobjsize;
984263bc
MD
255 vm_object_t object = vp->v_object;
256
257 if (object == NULL)
258 return;
259
260 /*
261 * Hasn't changed size
262 */
57f7b636 263 if (nsize == vp->v_filesize)
984263bc
MD
264 return;
265
c4b46cb4
MD
266 /*
267 * Has changed size. Adjust the VM object's size and v_filesize
268 * before we start scanning pages to prevent new pages from being
269 * allocated during the scan.
270 */
984263bc 271 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK);
c4b46cb4
MD
272 oobjsize = object->size;
273 object->size = nobjsize;
984263bc
MD
274
275 /*
276 * File has shrunk. Toss any cached pages beyond the new EOF.
277 */
57f7b636 278 if (nsize < vp->v_filesize) {
135bd6a8 279 vp->v_filesize = nsize;
c4b46cb4
MD
280 if (nobjsize < oobjsize) {
281 vm_object_page_remove(object, nobjsize, oobjsize,
282 FALSE);
984263bc
MD
283 }
284 /*
8d429613
MD
285 * This gets rid of garbage at the end of a page that is now
286 * only partially backed by the vnode. Since we are setting
287 * the entire page valid & clean after we are done we have
288 * to be sure that the portion of the page within the file
289 * bounds is already valid. If it isn't then making it
290 * valid would create a corrupt block.
984263bc
MD
291 */
292 if (nsize & PAGE_MASK) {
293 vm_offset_t kva;
294 vm_page_t m;
295
296 m = vm_page_lookup(object, OFF_TO_IDX(nsize));
297 if (m && m->valid) {
298 int base = (int)nsize & PAGE_MASK;
299 int size = PAGE_SIZE - base;
06ecca5a 300 struct sf_buf *sf;
984263bc
MD
301
302 /*
303 * Clear out partial-page garbage in case
304 * the page has been mapped.
305 */
4f1640d6 306 sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
06ecca5a 307 kva = sf_buf_kva(sf);
984263bc 308 bzero((caddr_t)kva + base, size);
06ecca5a 309 sf_buf_free(sf);
984263bc
MD
310
311 /*
312 * XXX work around SMP data integrity race
313 * by unmapping the page from user processes.
314 * The garbage we just cleared may be mapped
315 * to a user process running on another cpu
316 * and this code is not running through normal
317 * I/O channels which handle SMP issues for
318 * us, so unmap page to synchronize all cpus.
319 *
320 * XXX should vm_pager_unmap_page() have
321 * dealt with this?
322 */
323 vm_page_protect(m, VM_PROT_NONE);
324
325 /*
326 * Clear out partial-page dirty bits. This
327 * has the side effect of setting the valid
328 * bits, but that is ok. There are a bunch
329 * of places in the VM system where we expected
330 * m->dirty == VM_PAGE_BITS_ALL. The file EOF
331 * case is one of them. If the page is still
332 * partially dirty, make it fully dirty.
333 *
334 * note that we do not clear out the valid
335 * bits. This would prevent bogus_page
336 * replacement from working properly.
337 */
338 vm_page_set_validclean(m, base, size);
339 if (m->dirty != 0)
340 m->dirty = VM_PAGE_BITS_ALL;
341 }
342 }
135bd6a8
MD
343 } else {
344 vp->v_filesize = nsize;
984263bc 345 }
984263bc
MD
346}
347
348void
57e43348 349vnode_pager_freepage(vm_page_t m)
984263bc
MD
350{
351 vm_page_free(m);
352}
353
354/*
54078292
MD
355 * calculate the disk byte address of specified logical byte offset. The
356 * logical offset will be block-aligned. Return the number of contiguous
357 * pages that may be read from the underlying block device in *run. If
358 * *run is non-NULL, it will be set to a value of at least 1.
984263bc 359 */
54078292
MD
360static off_t
361vnode_pager_addr(struct vnode *vp, off_t loffset, int *run)
984263bc 362{
984263bc 363 struct vnode *rtvp;
54078292
MD
364 off_t doffset;
365 int bsize;
366 int error;
367 int voff;
984263bc 368
54078292 369 if (loffset < 0)
984263bc
MD
370 return -1;
371
372 if (vp->v_mount == NULL)
373 return -1;
374
54078292
MD
375 /*
376 * Align loffset to a block boundary for the BMAP, then adjust the
377 * returned disk address appropriately.
378 */
984263bc 379 bsize = vp->v_mount->mnt_stat.f_iosize;
54078292
MD
380 voff = loffset % bsize;
381
382 /*
383 * Map the block, adjust the disk offset so it represents the
384 * passed loffset rather then the block containing loffset.
385 */
386 error = VOP_BMAP(vp, loffset - voff, &rtvp, &doffset, run, NULL);
387 if (error || doffset == NOOFFSET) {
388 doffset = NOOFFSET;
389 } else {
390 doffset += voff;
391
392 /*
393 * When calculating *run, which is the number of pages
394 * worth of data which can be read linearly from disk,
395 * the minimum return value is 1 page.
396 */
397 if (run) {
398 *run = (*run - voff) >> PAGE_SHIFT;
399 if (*run < 1)
400 *run = 1;
984263bc 401 }
984263bc 402
54078292
MD
403 }
404 return (doffset);
984263bc
MD
405}
406
407/*
408 * interrupt routine for I/O completion
409 */
410static void
81b5c339 411vnode_pager_iodone(struct bio *bio)
984263bc 412{
81b5c339
MD
413 struct buf *bp = bio->bio_buf;
414
10f3fee5 415 bp->b_cmd = BUF_CMD_DONE;
984263bc
MD
416 wakeup(bp);
417}
418
419/*
420 * small block file system vnode pager input
421 */
422static int
57e43348 423vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)
984263bc
MD
424{
425 int i;
984263bc
MD
426 struct vnode *dp, *vp;
427 struct buf *bp;
428 vm_offset_t kva;
06ecca5a 429 struct sf_buf *sf;
54078292 430 off_t doffset;
984263bc
MD
431 vm_offset_t bsize;
432 int error = 0;
433
434 vp = object->handle;
435 if (vp->v_mount == NULL)
436 return VM_PAGER_BAD;
437
438 bsize = vp->v_mount->mnt_stat.f_iosize;
439
440
54078292 441 VOP_BMAP(vp, (off_t)0, &dp, NULL, NULL, NULL);
984263bc 442
06ecca5a
MD
443 sf = sf_buf_alloc(m, 0);
444 kva = sf_buf_kva(sf);
984263bc
MD
445
446 for (i = 0; i < PAGE_SIZE / bsize; i++) {
54078292 447 off_t loffset;
984263bc
MD
448
449 if (vm_page_bits(i * bsize, bsize) & m->valid)
450 continue;
451
54078292 452 loffset = IDX_TO_OFF(m->pindex) + i * bsize;
57f7b636 453 if (loffset >= vp->v_filesize) {
54078292 454 doffset = NOOFFSET;
984263bc 455 } else {
54078292 456 doffset = vnode_pager_addr(vp, loffset, NULL);
984263bc 457 }
54078292 458 if (doffset != NOOFFSET) {
984263bc
MD
459 bp = getpbuf(&vnode_pbuf_freecnt);
460
461 /* build a minimal buffer header */
984263bc 462 bp->b_data = (caddr_t) kva + i * bsize;
81b5c339 463 bp->b_bio1.bio_done = vnode_pager_iodone;
54078292 464 bp->b_bio1.bio_offset = doffset;
984263bc 465 bp->b_bcount = bsize;
9a71d53f 466 bp->b_runningbufspace = bsize;
984263bc 467 runningbufspace += bp->b_runningbufspace;
10f3fee5 468 bp->b_cmd = BUF_CMD_READ;
984263bc
MD
469
470 /* do the input */
81b5c339 471 vn_strategy(dp, &bp->b_bio1);
984263bc
MD
472
473 /* we definitely need to be at splvm here */
474
cdd46d2e 475 crit_enter();
10f3fee5 476 while (bp->b_cmd != BUF_CMD_DONE)
377d4740 477 tsleep(bp, 0, "vnsrd", 0);
cdd46d2e 478 crit_exit();
984263bc
MD
479 if ((bp->b_flags & B_ERROR) != 0)
480 error = EIO;
481
482 /*
483 * free the buffer header back to the swap buffer pool
484 */
485 relpbuf(bp, &vnode_pbuf_freecnt);
486 if (error)
487 break;
488
489 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize);
490 } else {
491 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize);
492 bzero((caddr_t) kva + i * bsize, bsize);
493 }
494 }
06ecca5a 495 sf_buf_free(sf);
984263bc
MD
496 pmap_clear_modify(m);
497 vm_page_flag_clear(m, PG_ZERO);
498 if (error) {
499 return VM_PAGER_ERROR;
500 }
501 return VM_PAGER_OK;
502
503}
504
505
506/*
507 * old style vnode pager output routine
508 */
509static int
57e43348 510vnode_pager_input_old(vm_object_t object, vm_page_t m)
984263bc
MD
511{
512 struct uio auio;
513 struct iovec aiov;
514 int error;
515 int size;
516 vm_offset_t kva;
06ecca5a 517 struct sf_buf *sf;
57f7b636 518 struct vnode *vp;
984263bc
MD
519
520 error = 0;
57f7b636 521 vp = object->handle;
984263bc
MD
522
523 /*
524 * Return failure if beyond current EOF
525 */
57f7b636 526 if (IDX_TO_OFF(m->pindex) >= vp->v_filesize) {
984263bc
MD
527 return VM_PAGER_BAD;
528 } else {
529 size = PAGE_SIZE;
57f7b636
MD
530 if (IDX_TO_OFF(m->pindex) + size > vp->v_filesize)
531 size = vp->v_filesize - IDX_TO_OFF(m->pindex);
984263bc
MD
532
533 /*
534 * Allocate a kernel virtual address and initialize so that
535 * we can use VOP_READ/WRITE routines.
536 */
06ecca5a
MD
537 sf = sf_buf_alloc(m, 0);
538 kva = sf_buf_kva(sf);
984263bc
MD
539
540 aiov.iov_base = (caddr_t) kva;
541 aiov.iov_len = size;
542 auio.uio_iov = &aiov;
543 auio.uio_iovcnt = 1;
544 auio.uio_offset = IDX_TO_OFF(m->pindex);
545 auio.uio_segflg = UIO_SYSSPACE;
546 auio.uio_rw = UIO_READ;
547 auio.uio_resid = size;
dadab5e9 548 auio.uio_td = curthread;
984263bc 549
0961aa92
MD
550 error = VOP_READ(((struct vnode *)object->handle),
551 &auio, 0, proc0.p_ucred);
984263bc 552 if (!error) {
5f910b2f 553 int count = size - auio.uio_resid;
984263bc
MD
554
555 if (count == 0)
556 error = EINVAL;
557 else if (count != PAGE_SIZE)
558 bzero((caddr_t) kva + count, PAGE_SIZE - count);
559 }
06ecca5a 560 sf_buf_free(sf);
984263bc
MD
561 }
562 pmap_clear_modify(m);
563 vm_page_undirty(m);
564 vm_page_flag_clear(m, PG_ZERO);
565 if (!error)
566 m->valid = VM_PAGE_BITS_ALL;
567 return error ? VM_PAGER_ERROR : VM_PAGER_OK;
568}
569
570/*
571 * generic vnode pager input routine
572 */
573
574/*
575 * EOPNOTSUPP is no longer legal. For local media VFS's that do not
576 * implement their own VOP_GETPAGES, their VOP_GETPAGES should call to
577 * vnode_pager_generic_getpages() to implement the previous behaviour.
578 *
579 * All other FS's should use the bypass to get to the local media
580 * backing vp's VOP_GETPAGES.
581 */
582static int
57e43348 583vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
984263bc
MD
584{
585 int rtval;
586 struct vnode *vp;
587 int bytes = count * PAGE_SIZE;
588
589 vp = object->handle;
590 /*
591 * XXX temporary diagnostic message to help track stale FS code,
592 * Returning EOPNOTSUPP from here may make things unhappy.
593 */
594 rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0);
595 if (rtval == EOPNOTSUPP) {
086c1d7e 596 kprintf("vnode_pager: *** WARNING *** stale FS getpages\n");
984263bc
MD
597 rtval = vnode_pager_generic_getpages( vp, m, bytes, reqpage);
598 }
599 return rtval;
600}
601
602
603/*
604 * This is now called from local media FS's to operate against their
605 * own vnodes if they fail to implement VOP_GETPAGES.
606 */
607int
57e43348
MD
608vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
609 int reqpage)
984263bc
MD
610{
611 vm_object_t object;
612 vm_offset_t kva;
613 off_t foff, tfoff, nextoff;
54078292
MD
614 int i, size, bsize, first;
615 off_t firstaddr;
984263bc
MD
616 struct vnode *dp;
617 int runpg;
618 int runend;
619 struct buf *bp;
984263bc
MD
620 int count;
621 int error = 0;
622
623 object = vp->v_object;
624 count = bytecount / PAGE_SIZE;
625
626 if (vp->v_mount == NULL)
627 return VM_PAGER_BAD;
628
629 bsize = vp->v_mount->mnt_stat.f_iosize;
630
631 /* get the UNDERLYING device for the file with VOP_BMAP() */
632
633 /*
634 * originally, we did not check for an error return value -- assuming
635 * an fs always has a bmap entry point -- that assumption is wrong!!!
636 */
637 foff = IDX_TO_OFF(m[reqpage]->pindex);
638
639 /*
640 * if we can't bmap, use old VOP code
641 */
54078292 642 if (VOP_BMAP(vp, (off_t)0, &dp, NULL, NULL, NULL)) {
984263bc
MD
643 for (i = 0; i < count; i++) {
644 if (i != reqpage) {
645 vnode_pager_freepage(m[i]);
646 }
647 }
12e4aaff
MD
648 mycpu->gd_cnt.v_vnodein++;
649 mycpu->gd_cnt.v_vnodepgsin++;
984263bc
MD
650 return vnode_pager_input_old(object, m[reqpage]);
651
652 /*
653 * if the blocksize is smaller than a page size, then use
654 * special small filesystem code. NFS sometimes has a small
655 * blocksize, but it can handle large reads itself.
656 */
657 } else if ((PAGE_SIZE / bsize) > 1 &&
658 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) {
659 for (i = 0; i < count; i++) {
660 if (i != reqpage) {
661 vnode_pager_freepage(m[i]);
662 }
663 }
12e4aaff
MD
664 mycpu->gd_cnt.v_vnodein++;
665 mycpu->gd_cnt.v_vnodepgsin++;
984263bc
MD
666 return vnode_pager_input_smlfs(object, m[reqpage]);
667 }
668
669 /*
670 * If we have a completely valid page available to us, we can
671 * clean up and return. Otherwise we have to re-read the
672 * media.
8d429613
MD
673 *
674 * Note that this does not work with NFS, so NFS has its own
675 * getpages routine. The problem is that NFS can have partially
676 * valid pages associated with the buffer cache due to the piecemeal
677 * write support. If we were to fall through and re-read the media
678 * as we do here, dirty data could be lost.
984263bc
MD
679 */
680
681 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) {
682 for (i = 0; i < count; i++) {
683 if (i != reqpage)
684 vnode_pager_freepage(m[i]);
685 }
686 return VM_PAGER_OK;
687 }
688 m[reqpage]->valid = 0;
689
690 /*
691 * here on direct device I/O
692 */
693
694 firstaddr = -1;
695 /*
696 * calculate the run that includes the required page
697 */
698 for(first = 0, i = 0; i < count; i = runend) {
54078292
MD
699 firstaddr = vnode_pager_addr(vp, IDX_TO_OFF(m[i]->pindex),
700 &runpg);
984263bc 701 if (firstaddr == -1) {
57f7b636 702 if (i == reqpage && foff < vp->v_filesize) {
984263bc 703 /* XXX no %qd in kernel. */
57f7b636
MD
704 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %012llx, foff: 0x%012llx, v_filesize: 0x%012llx",
705 firstaddr, foff, vp->v_filesize);
984263bc
MD
706 }
707 vnode_pager_freepage(m[i]);
708 runend = i + 1;
709 first = runend;
710 continue;
711 }
712 runend = i + runpg;
713 if (runend <= reqpage) {
714 int j;
715 for (j = i; j < runend; j++) {
716 vnode_pager_freepage(m[j]);
717 }
718 } else {
719 if (runpg < (count - first)) {
720 for (i = first + runpg; i < count; i++)
721 vnode_pager_freepage(m[i]);
722 count = first + runpg;
723 }
724 break;
725 }
726 first = runend;
727 }
728
729 /*
730 * the first and last page have been calculated now, move input pages
731 * to be zero based...
732 */
733 if (first != 0) {
734 for (i = first; i < count; i++) {
735 m[i - first] = m[i];
736 }
737 count -= first;
738 reqpage -= first;
739 }
740
741 /*
742 * calculate the file virtual address for the transfer
743 */
744 foff = IDX_TO_OFF(m[0]->pindex);
745
746 /*
747 * calculate the size of the transfer
748 */
749 size = count * PAGE_SIZE;
57f7b636
MD
750 if ((foff + size) > vp->v_filesize)
751 size = vp->v_filesize - foff;
984263bc
MD
752
753 /*
754 * round up physical size for real devices.
755 */
756 if (dp->v_type == VBLK || dp->v_type == VCHR) {
757 int secmask = dp->v_rdev->si_bsize_phys - 1;
758 KASSERT(secmask < PAGE_SIZE, ("vnode_pager_generic_getpages: sector size %d too large\n", secmask + 1));
759 size = (size + secmask) & ~secmask;
760 }
761
762 bp = getpbuf(&vnode_pbuf_freecnt);
763 kva = (vm_offset_t) bp->b_data;
764
765 /*
766 * and map the pages to be read into the kva
767 */
768 pmap_qenter(kva, m, count);
769
770 /* build a minimal buffer header */
81b5c339 771 bp->b_bio1.bio_done = vnode_pager_iodone;
54078292 772 bp->b_bio1.bio_offset = firstaddr;
984263bc 773 bp->b_bcount = size;
9a71d53f 774 bp->b_runningbufspace = size;
984263bc 775 runningbufspace += bp->b_runningbufspace;
10f3fee5 776 bp->b_cmd = BUF_CMD_READ;
984263bc 777
12e4aaff
MD
778 mycpu->gd_cnt.v_vnodein++;
779 mycpu->gd_cnt.v_vnodepgsin += count;
984263bc
MD
780
781 /* do the input */
81b5c339 782 vn_strategy(dp, &bp->b_bio1);
984263bc 783
cdd46d2e 784 crit_enter();
984263bc
MD
785 /* we definitely need to be at splvm here */
786
10f3fee5 787 while (bp->b_cmd != BUF_CMD_DONE)
377d4740 788 tsleep(bp, 0, "vnread", 0);
cdd46d2e 789 crit_exit();
984263bc
MD
790 if ((bp->b_flags & B_ERROR) != 0)
791 error = EIO;
792
793 if (!error) {
794 if (size != count * PAGE_SIZE)
795 bzero((caddr_t) kva + size, PAGE_SIZE * count - size);
796 }
797 pmap_qremove(kva, count);
798
799 /*
800 * free the buffer header back to the swap buffer pool
801 */
802 relpbuf(bp, &vnode_pbuf_freecnt);
803
804 for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) {
805 vm_page_t mt;
806
807 nextoff = tfoff + PAGE_SIZE;
808 mt = m[i];
809
57f7b636 810 if (nextoff <= vp->v_filesize) {
984263bc
MD
811 /*
812 * Read filled up entire page.
813 */
814 mt->valid = VM_PAGE_BITS_ALL;
815 vm_page_undirty(mt); /* should be an assert? XXX */
816 pmap_clear_modify(mt);
817 } else {
818 /*
819 * Read did not fill up entire page. Since this
820 * is getpages, the page may be mapped, so we have
821 * to zero the invalid portions of the page even
822 * though we aren't setting them valid.
823 *
824 * Currently we do not set the entire page valid,
825 * we just try to clear the piece that we couldn't
826 * read.
827 */
57f7b636 828 vm_page_set_validclean(mt, 0, vp->v_filesize - tfoff);
984263bc
MD
829 /* handled by vm_fault now */
830 /* vm_page_zero_invalid(mt, FALSE); */
831 }
832
833 vm_page_flag_clear(mt, PG_ZERO);
834 if (i != reqpage) {
835
836 /*
837 * whether or not to leave the page activated is up in
838 * the air, but we should put the page on a page queue
839 * somewhere. (it already is in the object). Result:
840 * It appears that empirical results show that
841 * deactivating pages is best.
842 */
843
844 /*
845 * just in case someone was asking for this page we
846 * now tell them that it is ok to use
847 */
848 if (!error) {
849 if (mt->flags & PG_WANTED)
850 vm_page_activate(mt);
851 else
852 vm_page_deactivate(mt);
853 vm_page_wakeup(mt);
854 } else {
855 vnode_pager_freepage(mt);
856 }
857 }
858 }
859 if (error) {
086c1d7e 860 kprintf("vnode_pager_getpages: I/O read error\n");
984263bc
MD
861 }
862 return (error ? VM_PAGER_ERROR : VM_PAGER_OK);
863}
864
865/*
866 * EOPNOTSUPP is no longer legal. For local media VFS's that do not
867 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to
868 * vnode_pager_generic_putpages() to implement the previous behaviour.
869 *
870 * All other FS's should use the bypass to get to the local media
871 * backing vp's VOP_PUTPAGES.
872 */
873static void
57e43348
MD
874vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count,
875 boolean_t sync, int *rtvals)
984263bc
MD
876{
877 int rtval;
878 struct vnode *vp;
879 int bytes = count * PAGE_SIZE;
880
881 /*
882 * Force synchronous operation if we are extremely low on memory
883 * to prevent a low-memory deadlock. VOP operations often need to
884 * allocate more memory to initiate the I/O ( i.e. do a BMAP
885 * operation ). The swapper handles the case by limiting the amount
886 * of asynchronous I/O, but that sort of solution doesn't scale well
887 * for the vnode pager without a lot of work.
888 *
889 * Also, the backing vnode's iodone routine may not wake the pageout
890 * daemon up. This should be probably be addressed XXX.
891 */
892
12e4aaff 893 if ((vmstats.v_free_count + vmstats.v_cache_count) < vmstats.v_pageout_free_min)
984263bc
MD
894 sync |= OBJPC_SYNC;
895
896 /*
897 * Call device-specific putpages function
898 */
899
900 vp = object->handle;
901 rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0);
902 if (rtval == EOPNOTSUPP) {
086c1d7e 903 kprintf("vnode_pager: *** WARNING *** stale FS putpages\n");
984263bc
MD
904 rtval = vnode_pager_generic_putpages( vp, m, bytes, sync, rtvals);
905 }
906}
907
908
909/*
910 * This is now called from local media FS's to operate against their
911 * own vnodes if they fail to implement VOP_PUTPAGES.
912 *
913 * This is typically called indirectly via the pageout daemon and
914 * clustering has already typically occured, so in general we ask the
915 * underlying filesystem to write the data out asynchronously rather
916 * then delayed.
917 */
918int
57e43348
MD
919vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int bytecount,
920 int flags, int *rtvals)
984263bc
MD
921{
922 int i;
923 vm_object_t object;
924 int count;
925
926 int maxsize, ncount;
927 vm_ooffset_t poffset;
928 struct uio auio;
929 struct iovec aiov;
930 int error;
931 int ioflags;
932
933 object = vp->v_object;
934 count = bytecount / PAGE_SIZE;
935
936 for (i = 0; i < count; i++)
937 rtvals[i] = VM_PAGER_AGAIN;
938
939 if ((int) m[0]->pindex < 0) {
086c1d7e 940 kprintf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%x)\n",
984263bc
MD
941 (long)m[0]->pindex, m[0]->dirty);
942 rtvals[0] = VM_PAGER_BAD;
943 return VM_PAGER_BAD;
944 }
945
946 maxsize = count * PAGE_SIZE;
947 ncount = count;
948
949 poffset = IDX_TO_OFF(m[0]->pindex);
950
951 /*
952 * If the page-aligned write is larger then the actual file we
953 * have to invalidate pages occuring beyond the file EOF. However,
954 * there is an edge case where a file may not be page-aligned where
955 * the last page is partially invalid. In this case the filesystem
956 * may not properly clear the dirty bits for the entire page (which
957 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d).
958 * With the page locked we are free to fix-up the dirty bits here.
959 *
960 * We do not under any circumstances truncate the valid bits, as
961 * this will screw up bogus page replacement.
962 */
57f7b636
MD
963 if (maxsize + poffset > vp->v_filesize) {
964 if (vp->v_filesize > poffset) {
984263bc
MD
965 int pgoff;
966
57f7b636 967 maxsize = vp->v_filesize - poffset;
984263bc
MD
968 ncount = btoc(maxsize);
969 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) {
970 vm_page_clear_dirty(m[ncount - 1], pgoff,
971 PAGE_SIZE - pgoff);
972 }
973 } else {
974 maxsize = 0;
975 ncount = 0;
976 }
977 if (ncount < count) {
978 for (i = ncount; i < count; i++) {
979 rtvals[i] = VM_PAGER_BAD;
980 }
981 }
982 }
983
984 /*
985 * pageouts are already clustered, use IO_ASYNC to force a bawrite()
986 * rather then a bdwrite() to prevent paging I/O from saturating
987 * the buffer cache. Dummy-up the sequential heuristic to cause
988 * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set,
989 * the system decides how to cluster.
990 */
991 ioflags = IO_VMIO;
992 if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL))
993 ioflags |= IO_SYNC;
994 else if ((flags & VM_PAGER_CLUSTER_OK) == 0)
995 ioflags |= IO_ASYNC;
996 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0;
997 ioflags |= IO_SEQMAX << IO_SEQSHIFT;
998
999 aiov.iov_base = (caddr_t) 0;
1000 aiov.iov_len = maxsize;
1001 auio.uio_iov = &aiov;
1002 auio.uio_iovcnt = 1;
1003 auio.uio_offset = poffset;
1004 auio.uio_segflg = UIO_NOCOPY;
1005 auio.uio_rw = UIO_WRITE;
1006 auio.uio_resid = maxsize;
dadab5e9 1007 auio.uio_td = NULL;
8a8d5d85 1008 error = VOP_WRITE(vp, &auio, ioflags, proc0.p_ucred);
12e4aaff
MD
1009 mycpu->gd_cnt.v_vnodeout++;
1010 mycpu->gd_cnt.v_vnodepgsout += ncount;
984263bc
MD
1011
1012 if (error) {
086c1d7e 1013 kprintf("vnode_pager_putpages: I/O error %d\n", error);
984263bc
MD
1014 }
1015 if (auio.uio_resid) {
086c1d7e 1016 kprintf("vnode_pager_putpages: residual I/O %d at %lu\n",
984263bc
MD
1017 auio.uio_resid, (u_long)m[0]->pindex);
1018 }
1019 for (i = 0; i < ncount; i++) {
1020 rtvals[i] = VM_PAGER_OK;
1021 }
1022 return rtvals[0];
1023}
1024
1025struct vnode *
dadab5e9 1026vnode_pager_lock(vm_object_t object)
984263bc 1027{
dadab5e9 1028 struct thread *td = curthread; /* XXX */
5fd012e0 1029 int error;
984263bc
MD
1030
1031 for (; object != NULL; object = object->backing_object) {
1032 if (object->type != OBJT_VNODE)
1033 continue;
1034 if (object->flags & OBJ_DEAD)
1035 return NULL;
1036
5fd012e0
MD
1037 for (;;) {
1038 struct vnode *vp = object->handle;
ab6f251b 1039 error = vget(vp, LK_SHARED | LK_RETRY | LK_CANRECURSE);
5fd012e0
MD
1040 if (error == 0) {
1041 if (object->handle != vp) {
1042 vput(vp);
1043 continue;
1044 }
1045 return (vp);
1046 }
dadab5e9
MD
1047 if ((object->flags & OBJ_DEAD) ||
1048 (object->type != OBJT_VNODE)) {
984263bc 1049 return NULL;
dadab5e9 1050 }
086c1d7e 1051 kprintf("vnode_pager_lock: vp %p error %d lockstatus %d, retrying\n", vp, error, lockstatus(&vp->v_lock, td));
5fd012e0 1052 tsleep(object->handle, 0, "vnpgrl", hz);
984263bc 1053 }
984263bc
MD
1054 }
1055 return NULL;
1056}