2 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * Implements new VFS/VM coherency functions. For conforming VFSs
37 * we treat the backing VM object slightly differently. Instead of
38 * maintaining a number of pages to exactly fit the size of the file
39 * we instead maintain pages to fit the entire contents of the last
40 * buffer cache buffer used by the file.
42 * For VFSs like NFS and HAMMER which use (generally speaking) fixed
43 * sized buffers this greatly reduces the complexity of VFS/VM interactions.
45 * Truncations no longer invalidate pages covered by the buffer cache
46 * beyond the file EOF which still fit within the file's last buffer.
47 * We simply unmap them and do not allow userland to fault them in.
49 * The VFS is no longer responsible for zero-filling buffers during a
50 * truncation, the last buffer will be automatically zero-filled by
53 * This code is intended to (eventually) replace vtruncbuf() and
54 * vnode_pager_setsize().
57 #include <sys/param.h>
58 #include <sys/systm.h>
61 #include <sys/fcntl.h>
63 #include <sys/kernel.h>
64 #include <sys/malloc.h>
65 #include <sys/mount.h>
67 #include <sys/socket.h>
69 #include <sys/sysctl.h>
70 #include <sys/unistd.h>
71 #include <sys/vmmeter.h>
72 #include <sys/vnode.h>
74 #include <machine/limits.h>
77 #include <vm/vm_object.h>
78 #include <vm/vm_extern.h>
79 #include <vm/vm_kern.h>
81 #include <vm/vm_map.h>
82 #include <vm/vm_page.h>
83 #include <vm/vm_pager.h>
84 #include <vm/vnode_pager.h>
85 #include <vm/vm_zone.h>
88 #include <sys/thread2.h>
89 #include <sys/sysref2.h>
91 static int nvtruncbuf_bp_trunc_cmp(struct buf *bp, void *data);
92 static int nvtruncbuf_bp_trunc(struct buf *bp, void *data);
93 static int nvtruncbuf_bp_metasync_cmp(struct buf *bp, void *data);
94 static int nvtruncbuf_bp_metasync(struct buf *bp, void *data);
97 * Truncate a file's buffer and pages to a specified length. The
98 * byte-granular length of the file is specified along with the block
99 * size of the buffer containing that offset.
101 * If the last buffer straddles the length its contents will be zero-filled
102 * as appropriate. All buffers and pages after the last buffer will be
103 * destroyed. The last buffer itself will be destroyed only if the length
104 * is exactly aligned with it.
106 * UFS typically passes the old block size prior to the actual truncation,
107 * then later resizes the block based on the new file size. NFS uses a
108 * fixed block size and doesn't care. HAMMER uses a block size based on
109 * the offset which is fixed for any particular offset.
111 * When zero-filling we must bdwrite() to avoid a window of opportunity
112 * where the kernel might throw away a clean buffer and the filesystem
113 * then attempts to bread() it again before completing (or as part of)
114 * the extension. The filesystem is still responsible for zero-filling
115 * any remainder when writing to the media in the strategy function when
116 * it is able to do so without the page being mapped. The page may still
117 * be mapped by userland here.
119 * When modifying a buffer we must clear any cached raw disk offset.
120 * bdwrite() will call BMAP on it again. Some filesystems, like HAMMER,
121 * never overwrite existing data blocks.
124 struct truncbuf_info {
126 off_t truncloffset; /* truncation point */
127 int clean; /* clean tree, else dirty tree */
131 nvtruncbuf(struct vnode *vp, off_t length, int blksize, int boff, int trivial)
133 struct truncbuf_info info;
135 const char *filename;
141 * Round up to the *next* block, then destroy the buffers in question.
142 * Since we are only removing some of the buffers we must rely on the
143 * scan count to determine whether a loop is necessary.
145 * Destroy any pages beyond the last buffer.
148 boff = (int)(length % blksize);
150 info.truncloffset = length + (blksize - boff);
152 info.truncloffset = length;
154 lwkt_gettoken(&vp->v_token);
157 count = RB_SCAN(buf_rb_tree, &vp->v_rbclean_tree,
158 nvtruncbuf_bp_trunc_cmp,
159 nvtruncbuf_bp_trunc, &info);
161 count += RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree,
162 nvtruncbuf_bp_trunc_cmp,
163 nvtruncbuf_bp_trunc, &info);
166 nvnode_pager_setsize(vp, length, blksize, boff);
169 * Zero-fill the area beyond the file EOF that still fits within
170 * the last buffer. We must mark the buffer as dirty even though
171 * the modified area is beyond EOF to avoid races where the kernel
172 * might flush the buffer before the filesystem is able to reallocate
175 * The VFS is responsible for dealing with the actual truncation.
177 * Only do this if trivial is zero, otherwise it is up to the
178 * VFS to handle the block straddling the EOF.
180 if (boff && trivial == 0) {
181 truncboffset = length - boff;
182 error = bread(vp, truncboffset, blksize, &bp);
184 bzero(bp->b_data + boff, blksize - boff);
185 if (bp->b_flags & B_DELWRI) {
186 if (bp->b_dirtyoff > boff)
187 bp->b_dirtyoff = boff;
188 if (bp->b_dirtyend > boff)
189 bp->b_dirtyend = boff;
191 bp->b_bio2.bio_offset = NOOFFSET;
199 * For safety, fsync any remaining metadata if the file is not being
200 * truncated to 0. Since the metadata does not represent the entire
201 * dirty list we have to rely on the hit count to ensure that we get
204 * This is typically applicable only to UFS. NFS and HAMMER do
205 * not store indirect blocks in the per-vnode buffer cache.
209 count = RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree,
210 nvtruncbuf_bp_metasync_cmp,
211 nvtruncbuf_bp_metasync, &info);
216 * It is possible to have in-progress I/O from buffers that were
217 * not part of the truncation. This should not happen if we
218 * are truncating to 0-length.
220 bio_track_wait(&vp->v_track_write, 0, 0);
225 spin_lock(&vp->v_spin);
226 filename = TAILQ_FIRST(&vp->v_namecache) ?
227 TAILQ_FIRST(&vp->v_namecache)->nc_name : "?";
228 spin_unlock(&vp->v_spin);
231 * Make sure no buffers were instantiated while we were trying
232 * to clean out the remaining VM pages. This could occur due
233 * to busy dirty VM pages being flushed out to disk.
237 count = RB_SCAN(buf_rb_tree, &vp->v_rbclean_tree,
238 nvtruncbuf_bp_trunc_cmp,
239 nvtruncbuf_bp_trunc, &info);
241 count += RB_SCAN(buf_rb_tree, &vp->v_rbdirty_tree,
242 nvtruncbuf_bp_trunc_cmp,
243 nvtruncbuf_bp_trunc, &info);
245 kprintf("Warning: vtruncbuf(): Had to re-clean %d "
246 "left over buffers in %s\n", count, filename);
250 lwkt_reltoken(&vp->v_token);
256 * The callback buffer is beyond the new file EOF and must be destroyed.
257 * Note that the compare function must conform to the RB_SCAN's requirements.
261 nvtruncbuf_bp_trunc_cmp(struct buf *bp, void *data)
263 struct truncbuf_info *info = data;
265 if (bp->b_loffset >= info->truncloffset)
272 nvtruncbuf_bp_trunc(struct buf *bp, void *data)
274 struct truncbuf_info *info = data;
277 * Do not try to use a buffer we cannot immediately lock,
278 * but sleep anyway to prevent a livelock. The code will
279 * loop until all buffers can be acted upon.
281 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
282 atomic_add_int(&bp->b_refs, 1);
283 if (BUF_LOCK(bp, LK_EXCLUSIVE|LK_SLEEPFAIL) == 0)
285 atomic_subtract_int(&bp->b_refs, 1);
286 } else if ((info->clean && (bp->b_flags & B_DELWRI)) ||
287 (info->clean == 0 && (bp->b_flags & B_DELWRI) == 0) ||
288 bp->b_vp != info->vp ||
289 nvtruncbuf_bp_trunc_cmp(bp, data)) {
293 bp->b_flags |= (B_INVAL | B_RELBUF | B_NOCACHE);
300 * Fsync all meta-data after truncating a file to be non-zero. Only metadata
301 * blocks (with a negative loffset) are scanned.
302 * Note that the compare function must conform to the RB_SCAN's requirements.
305 nvtruncbuf_bp_metasync_cmp(struct buf *bp, void *data __unused)
307 if (bp->b_loffset < 0)
313 nvtruncbuf_bp_metasync(struct buf *bp, void *data)
315 struct truncbuf_info *info = data;
318 * Do not try to use a buffer we cannot immediately lock,
319 * but sleep anyway to prevent a livelock. The code will
320 * loop until all buffers can be acted upon.
322 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT)) {
323 atomic_add_int(&bp->b_refs, 1);
324 if (BUF_LOCK(bp, LK_EXCLUSIVE|LK_SLEEPFAIL) == 0)
326 atomic_subtract_int(&bp->b_refs, 1);
327 } else if ((bp->b_flags & B_DELWRI) == 0 ||
328 bp->b_vp != info->vp ||
329 nvtruncbuf_bp_metasync_cmp(bp, data)) {
339 * Extend a file's buffer and pages to a new, larger size. The block size
340 * at both the old and new length must be passed, but buffer cache operations
341 * will only be performed on the old block. The new nlength/nblksize will
342 * be used to properly set the VM object size.
344 * To make this explicit we require the old length to passed even though
345 * we can acquire it from vp->v_filesize, which also avoids potential
346 * corruption if the filesystem and vp get desynchronized somehow.
348 * If the caller intends to immediately write into the newly extended
349 * space pass trivial == 1. If trivial is 0 the original buffer will be
350 * zero-filled as necessary to clean out any junk in the extended space.
351 * If non-zero the original buffer (straddling EOF) is not touched.
353 * When zero-filling we must bdwrite() to avoid a window of opportunity
354 * where the kernel might throw away a clean buffer and the filesystem
355 * then attempts to bread() it again before completing (or as part of)
356 * the extension. The filesystem is still responsible for zero-filling
357 * any remainder when writing to the media in the strategy function when
358 * it is able to do so without the page being mapped. The page may still
359 * be mapped by userland here.
361 * When modifying a buffer we must clear any cached raw disk offset.
362 * bdwrite() will call BMAP on it again. Some filesystems, like HAMMER,
363 * never overwrite existing data blocks.
366 nvextendbuf(struct vnode *vp, off_t olength, off_t nlength,
367 int oblksize, int nblksize, int oboff, int nboff, int trivial)
374 nvnode_pager_setsize(vp, nlength, nblksize, nboff);
377 oboff = (int)(olength % oblksize);
378 truncboffset = olength - oboff;
381 error = bread(vp, truncboffset, oblksize, &bp);
383 bzero(bp->b_data + oboff, oblksize - oboff);
384 bp->b_bio2.bio_offset = NOOFFSET;
393 * Set vp->v_filesize and vp->v_object->size, destroy pages beyond
394 * the last buffer when truncating.
396 * This function does not do any zeroing or invalidating of partially
397 * overlapping pages. Zeroing is the responsibility of nvtruncbuf().
398 * However, it does unmap VM pages from the user address space on a
399 * page-granular (verses buffer cache granular) basis.
401 * If boff is passed as -1 the base offset of the buffer cache buffer is
402 * calculated from length and blksize. Filesystems such as UFS which deal
403 * with fragments have to specify a boff >= 0 since the base offset cannot
404 * be calculated from length and blksize.
406 * For UFS blksize is the 'new' blocksize, used only to determine how large
407 * the VM object must become.
410 nvnode_pager_setsize(struct vnode *vp, off_t length, int blksize, int boff)
412 vm_pindex_t nobjsize;
413 vm_pindex_t oobjsize;
420 * Degenerate conditions
422 if ((object = vp->v_object) == NULL)
424 vm_object_hold(object);
425 if (length == vp->v_filesize) {
426 vm_object_drop(object);
431 * Calculate the size of the VM object, coverage includes
432 * the buffer straddling EOF. If EOF is buffer-aligned
435 * Buffers do not have to be page-aligned. Make sure
436 * nobjsize is beyond the last page of the buffer.
439 boff = (int)(length % blksize);
440 truncboffset = length - boff;
441 oobjsize = object->size;
443 nobjsize = OFF_TO_IDX(truncboffset + blksize + PAGE_MASK);
445 nobjsize = OFF_TO_IDX(truncboffset + PAGE_MASK);
446 object->size = nobjsize;
448 if (length < vp->v_filesize) {
450 * File has shrunk, toss any cached pages beyond
451 * the end of the buffer (blksize aligned) for the
454 vp->v_filesize = length;
455 if (nobjsize < oobjsize) {
456 vm_object_page_remove(object, nobjsize, oobjsize,
461 * Unmap any pages (page aligned) beyond the new EOF.
462 * The pages remain part of the (last) buffer and are not
465 pi = OFF_TO_IDX(length + PAGE_MASK);
466 while (pi < nobjsize) {
467 m = vm_page_lookup_busy_wait(object, pi, FALSE, "vmpg");
469 vm_page_protect(m, VM_PROT_NONE);
478 vp->v_filesize = length;
480 vm_object_drop(object);