2 * Copyright (c) 2011-2018 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * TRANSACTION AND FLUSH HANDLING
38 * Deceptively simple but actually fairly difficult to implement properly is
39 * how I would describe it.
41 * Flushing generally occurs bottom-up but requires a top-down scan to
42 * locate chains with MODIFIED and/or UPDATE bits set. The ONFLUSH flag
43 * tells how to recurse downward to find these chains.
46 #include <sys/cdefs.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/types.h>
57 #define HAMMER2_FLUSH_DEPTH_LIMIT 60 /* stack recursion limit */
61 * Recursively flush the specified chain. The chain is locked and
62 * referenced by the caller and will remain so on return. The chain
63 * will remain referenced throughout but can temporarily lose its
64 * lock during the recursion to avoid unnecessarily stalling user
67 struct hammer2_flush_info {
68 hammer2_chain_t *parent;
70 int error; /* cumulative error */
72 #ifdef HAMMER2_SCAN_DEBUG
80 hammer2_chain_t *debug;
83 typedef struct hammer2_flush_info hammer2_flush_info_t;
85 static int hammer2_flush_core(hammer2_flush_info_t *info,
86 hammer2_chain_t *chain, int flags);
87 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
90 * Any per-pfs transaction initialization goes here.
93 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
98 * Transaction support for any modifying operation. Transactions are used
99 * in the pmp layer by the frontend and in the spmp layer by the backend.
101 * 0 - Normal transaction. Interlocks against just the
102 * COPYQ portion of an ISFLUSH transaction.
104 * TRANS_ISFLUSH - Flush transaction. Interlocks against other flush
107 * When COPYQ is also specified, waits for the count
110 * TRANS_BUFCACHE - Buffer cache transaction. No interlock.
112 * TRANS_SIDEQ - Run the sideq (only tested in trans_done())
114 * Initializing a new transaction allocates a transaction ID. Typically
115 * passed a pmp (hmp passed as NULL), indicating a cluster transaction. Can
116 * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
117 * media target. The latter mode is used by the recovery code.
120 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
127 oflags = pmp->trans.flags;
131 if (flags & HAMMER2_TRANS_ISFLUSH) {
133 * Interlock against other flush transactions.
135 if (oflags & HAMMER2_TRANS_ISFLUSH) {
136 nflags = oflags | HAMMER2_TRANS_WAITING;
139 nflags = (oflags | flags) + 1;
141 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
143 * Requesting strategy transaction from buffer-cache,
144 * or a VM getpages/putpages through the buffer cache.
145 * We must allow such transactions in all situations
146 * to avoid deadlocks.
148 nflags = (oflags | flags) + 1;
151 * Normal transaction. We currently only interlock
152 * against COPYQ. We do not interlock against
153 * BUFCACHE or ISFLUSH. COPYQ is used to interlock
154 * the transfer of SIDEQ into SYNCQ.
156 * Note that vnode locks may be held going into
159 * NOTE: Remember that non-modifying operations
160 * such as read, stat, readdir, etc, do
161 * not use transactions.
163 if (oflags & HAMMER2_TRANS_COPYQ) {
164 nflags = oflags | HAMMER2_TRANS_WAITING;
167 nflags = (oflags | flags) + 1;
171 tsleep_interlock(&pmp->trans.sync_wait, 0);
172 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
175 tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
186 * When entering a FLUSH transaction with COPYQ set, wait for the
187 * transaction count to drop to 1 (our flush transaction only)
190 * This waits for all non-flush transactions to complete and blocks
191 * new non-flush transactions from starting until COPYQ is cleared.
192 * (the flush will then proceed after clearing COPYQ). This should
193 * be a very short stall on modifying operations.
195 while ((flags & HAMMER2_TRANS_ISFLUSH) &&
196 (flags & HAMMER2_TRANS_COPYQ)) {
197 oflags = pmp->trans.flags;
199 if ((oflags & HAMMER2_TRANS_MASK) == 1)
201 nflags = oflags | HAMMER2_TRANS_WAITING;
202 tsleep_interlock(&pmp->trans.sync_wait, 0);
203 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
204 tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
211 * Start a sub-transaction, there is no 'subdone' function. This will
212 * issue a new modify_tid (mtid) for the current transaction, which is a
213 * CLC (cluster level change) id and not a per-node id.
215 * This function must be called for each XOP when multiple XOPs are run in
216 * sequence within a transaction.
218 * Callers typically update the inode with the transaction mtid manually
219 * to enforce sequencing.
222 hammer2_trans_sub(hammer2_pfs_t *pmp)
226 mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
232 hammer2_trans_setflags(hammer2_pfs_t *pmp, uint32_t flags)
234 atomic_set_int(&pmp->trans.flags, flags);
238 * Typically used to clear trans flags asynchronously. If TRANS_WAITING
239 * is in the mask, and was previously set, this function will wake up
243 hammer2_trans_clearflags(hammer2_pfs_t *pmp, uint32_t flags)
249 oflags = pmp->trans.flags;
251 nflags = oflags & ~flags;
252 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
253 if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING)
254 wakeup(&pmp->trans.sync_wait);
263 hammer2_trans_done(hammer2_pfs_t *pmp, uint32_t flags)
269 * Modifying ops on the front-end can cause dirty inodes to
270 * build up in the sideq. We don't flush these on inactive/reclaim
271 * due to potential deadlocks, so we have to deal with them from
272 * inside other nominal modifying front-end transactions.
274 if ((flags & HAMMER2_TRANS_SIDEQ) &&
275 pmp->sideq_count > (pmp->inum_count >> 3) &&
277 speedup_syncer(pmp->mp);
281 * Clean-up the transaction. Wakeup any waiters when finishing
282 * a flush transaction or transitioning the non-flush transaction
283 * count from 2->1 while a flush transaction is pending.
286 oflags = pmp->trans.flags;
288 KKASSERT(oflags & HAMMER2_TRANS_MASK);
290 nflags = (oflags - 1) & ~flags;
291 if (flags & HAMMER2_TRANS_ISFLUSH) {
292 nflags &= ~HAMMER2_TRANS_WAITING;
294 if ((oflags & (HAMMER2_TRANS_ISFLUSH|HAMMER2_TRANS_MASK)) ==
295 (HAMMER2_TRANS_ISFLUSH|2)) {
296 nflags &= ~HAMMER2_TRANS_WAITING;
298 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
299 if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING)
300 wakeup(&pmp->trans.sync_wait);
309 * Obtain new, unique inode number (not serialized by caller).
312 hammer2_trans_newinum(hammer2_pfs_t *pmp)
316 tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
322 * Assert that a strategy call is ok here. Currently we allow strategy
323 * calls in all situations, including during flushes. Previously:
324 * (old) (1) In a normal transaction.
325 * (old) (2) In a flush transaction only if PREFLUSH is also set.
328 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
331 KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 ||
332 (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH));
337 * Flush the chain and all modified sub-chains through the specified
338 * synchronization point, propagating blockref updates back up. As
339 * part of this propagation, mirror_tid and inode/data usage statistics
340 * propagates back upward.
342 * Returns a HAMMER2 error code, 0 if no error. Note that I/O errors from
343 * buffers dirtied during the flush operation can occur later.
345 * modify_tid (clc - cluster level change) is not propagated.
347 * update_tid (clc) is used for validation and is not propagated by this
350 * This routine can be called from several places but the most important
351 * is from VFS_SYNC (frontend) via hammer2_xop_inode_flush (backend).
353 * chain is locked on call and will remain locked on return. The chain's
354 * UPDATE flag indicates that its parent's block table (which is not yet
355 * part of the flush) should be updated.
358 * HAMMER2_FLUSH_TOP Indicates that this is the top of the flush.
359 * Is cleared for the recursion.
361 * HAMMER2_FLUSH_ALL Recurse everything
363 * HAMMER2_FLUSH_INODE_STOP
364 * Stop at PFS inode or normal inode boundary
367 hammer2_flush(hammer2_chain_t *chain, int flags)
369 hammer2_flush_info_t info;
374 * Execute the recursive flush and handle deferrals.
376 * Chains can be ridiculously long (thousands deep), so to
377 * avoid blowing out the kernel stack the recursive flush has a
378 * depth limit. Elements at the limit are placed on a list
379 * for re-execution after the stack has been popped.
381 bzero(&info, sizeof(info));
382 info.flags = flags & ~HAMMER2_FLUSH_TOP;
385 * Calculate parent (can be NULL), if not NULL the flush core
386 * expects the parent to be referenced so it can easily lock/unlock
387 * it without it getting ripped up.
389 if ((info.parent = chain->parent) != NULL)
390 hammer2_chain_ref(info.parent);
393 * Extra ref needed because flush_core expects it when replacing
396 hammer2_chain_ref(chain);
402 * [re]flush chain as the deep recursion may have generated
403 * additional modifications.
405 if (info.parent != chain->parent) {
406 if (hammer2_debug & 0x0040) {
407 kprintf("LOST CHILD4 %p->%p "
408 "(actual parent %p)\n",
409 info.parent, chain, chain->parent);
411 hammer2_chain_drop(info.parent);
412 info.parent = chain->parent;
413 hammer2_chain_ref(info.parent);
415 if (hammer2_flush_core(&info, chain, flags) == 0)
418 if (++loops % 1000 == 0) {
419 kprintf("hammer2_flush: excessive loops on %p\n",
421 if (hammer2_debug & 0x100000)
425 #ifdef HAMMER2_SCAN_DEBUG
426 if (info.scan_count >= 10)
427 kprintf("hammer2_flush: scan_count %ld (%ld,%ld,%ld,%ld) "
428 "bt(%ld,%ld,%ld,%ld,%ld,%ld)\n",
441 hammer2_chain_drop(chain);
443 hammer2_chain_drop(info.parent);
448 * This is the core of the chain flushing code. The chain is locked by the
449 * caller and must also have an extra ref on it by the caller, and remains
450 * locked and will have an extra ref on return. info.parent is referenced
453 * Upon return, the caller can test the UPDATE bit on the chain to determine
454 * if the parent needs updating.
456 * If non-zero is returned, the chain's parent changed during the flush and
457 * the caller must retry the operation.
459 * (1) Determine if this node is a candidate for the flush, return if it is
460 * not. fchain and vchain are always candidates for the flush.
462 * (2) If we recurse too deep the chain is entered onto the deferral list and
463 * the current flush stack is aborted until after the deferral list is
466 * (3) Recursively flush live children (rbtree). This can create deferrals.
467 * A successful flush clears the MODIFIED and UPDATE bits on the children
468 * and typically causes the parent to be marked MODIFIED as the children
469 * update the parent's block table. A parent might already be marked
470 * MODIFIED due to a deletion (whos blocktable update in the parent is
471 * handled by the frontend), or if the parent itself is modified by the
472 * frontend for other reasons.
474 * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
475 * Deleted-but-open inodes can still be individually flushed via the
478 * (5) Delete parents on the way back up if they are normal indirect blocks
479 * and have no children.
481 * (6) Note that an unmodified child may still need the block table in its
482 * parent updated (e.g. rename/move). The child will have UPDATE set
485 * WARNING ON BREF MODIFY_TID/MIRROR_TID
487 * blockref.modify_tid is consistent only within a PFS, and will not be
488 * consistent during synchronization. mirror_tid is consistent across the
489 * block device regardless of the PFS.
492 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
495 hammer2_chain_t *parent;
503 * (1) Optimize downward recursion to locate nodes needing action.
504 * Nothing to do if none of these flags are set.
506 if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) {
507 if (hammer2_debug & 0x200) {
508 if (info->debug == NULL)
518 * NOTE: parent can be NULL, usually due to destroy races.
520 parent = info->parent;
521 KKASSERT(chain->parent == parent);
524 * Downward search recursion
526 * We must be careful on cold stops. If CHAIN_UPDATE is set and
527 * we stop cold, the update can wind up never being applied. This
528 * situation most typically occurs on inode boundaries due to the way
529 * hammer2_vfs_sync() breaks-up the flush. As a safety, we
530 * flush-through such situations. XXX removed
532 if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) &&
533 /* (chain->flags & HAMMER2_CHAIN_UPDATE) == 0 && */
534 (flags & HAMMER2_FLUSH_ALL) == 0 &&
535 (flags & HAMMER2_FLUSH_TOP) == 0 &&
536 chain->pmp && chain->pmp->mp) {
538 * If FLUSH_ALL is not specified the caller does not want
539 * to recurse through PFS roots that have been mounted.
541 * (If the PFS has not been mounted there may not be
542 * anything monitoring its chains and its up to us
545 * The typical sequence is to flush dirty PFS's starting at
546 * their root downward, then flush the device root (vchain).
547 * It is this second flush that typically leaves out the
550 * However we must still process the PFSROOT chains for block
551 * table updates in their parent (which IS part of our flush).
553 * NOTE: The volume root, vchain, does not set PFSBOUNDARY.
555 * NOTE: We must re-set ONFLUSH in the parent to retain if
556 * this chain (that we are skipping) requires work.
558 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
559 HAMMER2_CHAIN_DESTROY |
560 HAMMER2_CHAIN_MODIFIED)) {
561 hammer2_chain_setflush(parent);
564 } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
565 /* (chain->flags & HAMMER2_CHAIN_UPDATE) == 0 && */
566 (flags & HAMMER2_FLUSH_INODE_STOP) &&
567 (flags & HAMMER2_FLUSH_ALL) == 0 &&
568 (flags & HAMMER2_FLUSH_TOP) == 0 &&
569 chain->pmp && chain->pmp->mp) {
571 * When FLUSH_INODE_STOP is specified we are being asked not
572 * to include any inode changes for inodes we encounter,
573 * with the exception of the inode that the flush began with.
574 * So: INODE, INODE_STOP, and TOP==0 basically.
579 * If FLUSH_INODE_STOP is specified and both ALL and TOP
580 * are clear, we must not flush the chain. The chain should
581 * have already been flushed and any further ONFLUSH/UPDATE
582 * setting will be related to the next flush.
584 * This features allows us to flush inodes independently of
585 * each other and meta-data above the inodes separately.
587 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
588 HAMMER2_CHAIN_DESTROY |
589 HAMMER2_CHAIN_MODIFIED)) {
591 hammer2_chain_setflush(parent);
594 } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
596 * Recursion depth reached.
598 panic("hammer2: flush depth limit");
599 } else if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
600 HAMMER2_CHAIN_DESTROY)) {
602 * Downward recursion search (actual flush occurs bottom-up).
603 * pre-clear ONFLUSH. It can get set again due to races or
604 * flush errors, which we want so the scan finds us again in
607 * We must also recurse if DESTROY is set so we can finally
608 * get rid of the related children, otherwise the node will
609 * just get re-flushed on lastdrop.
611 * WARNING! The recursion will unlock/relock info->parent
612 * (which is 'chain'), potentially allowing it
615 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
616 save_error = info->error;
618 info->parent = chain;
621 * We may have to do this twice to catch any indirect
622 * block maintenance that occurs.
624 hammer2_spin_ex(&chain->core.spin);
625 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
626 NULL, hammer2_flush_recurse, info);
627 if (chain->flags & HAMMER2_CHAIN_ONFLUSH) {
628 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
629 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
630 NULL, hammer2_flush_recurse, info);
632 hammer2_spin_unex(&chain->core.spin);
633 info->parent = parent;
636 * Re-set the flush bits if the flush was incomplete or
637 * an error occurred. If an error occurs it is typically
638 * an allocation error. Errors do not cause deferrals.
641 hammer2_chain_setflush(chain);
642 info->error |= save_error;
645 * If we lost the parent->chain association we have to
646 * stop processing this chain because it is no longer
647 * in this recursion. If it moved, it will be handled
648 * by the ONFLUSH flag elsewhere.
650 if (chain->parent != parent) {
651 kprintf("LOST CHILD2 %p->%p (actual parent %p)\n",
652 parent, chain, chain->parent);
658 * Now we are in the bottom-up part of the recursion.
660 * We continue to try to update the chain on lower-level errors, but
661 * the flush code may decide not to flush the volume root.
663 * XXX should we continue to try to update the chain if an error
668 * Both parent and chain must be locked in order to flush chain,
669 * in order to properly update the parent under certain conditions.
671 * In addition, we can't safely unlock/relock the chain once we
672 * start flushing the chain itself, which we would have to do later
673 * on in order to lock the parent if we didn't do that now.
675 hammer2_chain_ref_hold(chain);
676 hammer2_chain_unlock(chain);
678 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
679 hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
680 hammer2_chain_drop_unhold(chain);
683 * Can't process if we can't access their content.
685 if ((parent && parent->error) || chain->error) {
686 kprintf("hammer2: chain error during flush\n");
687 info->error |= chain->error;
689 info->error |= parent->error;
690 hammer2_chain_unlock(parent);
695 if (chain->parent != parent) {
696 if (hammer2_debug & 0x0040) {
697 kprintf("LOST CHILD3 %p->%p (actual parent %p)\n",
698 parent, chain, chain->parent);
700 KKASSERT(parent != NULL);
701 hammer2_chain_unlock(parent);
707 * Propagate the DESTROY flag downwards. This dummies up the flush
708 * code and tries to invalidate related buffer cache buffers to
709 * avoid the disk write.
711 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
712 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
715 * Dispose of the modified bit.
717 * If parent is present, the UPDATE bit should already be set.
718 * UPDATE should already be set.
719 * bref.mirror_tid should already be set.
721 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
722 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
723 chain->parent == NULL);
724 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
725 atomic_add_long(&hammer2_count_modified_chains, -1);
728 * Manage threads waiting for excessive dirty memory to
732 hammer2_pfs_memory_wakeup(chain->pmp);
735 if ((chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
736 chain != &hmp->vchain &&
737 chain != &hmp->fchain) {
739 * Set UPDATE bit indicating that the parent block
740 * table requires updating.
742 atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
747 * Issue the flush. This is indirect via the DIO.
749 * NOTE: A DELETED node that reaches this point must be
750 * flushed for synchronization point consistency.
752 * NOTE: Even though MODIFIED was already set, the related DIO
753 * might not be dirty due to a system buffer cache
754 * flush and must be set dirty if we are going to make
755 * further modifications to the buffer. Chains with
756 * embedded data don't need this.
758 if (hammer2_debug & 0x1000) {
759 kprintf("Flush %p.%d %016jx/%d data=%016jx\n",
760 chain, chain->bref.type,
761 (uintmax_t)chain->bref.key,
763 (uintmax_t)chain->bref.data_off);
765 if (hammer2_debug & 0x2000) {
766 Debugger("Flush hell");
770 * Update chain CRCs for flush.
772 * NOTE: Volume headers are NOT flushed here as they require
773 * special processing.
775 switch(chain->bref.type) {
776 case HAMMER2_BREF_TYPE_FREEMAP:
778 * Update the volume header's freemap_tid to the
779 * freemap's flushing mirror_tid.
781 * (note: embedded data, do not call setdirty)
783 KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
784 KKASSERT(chain == &hmp->fchain);
785 hmp->voldata.freemap_tid = chain->bref.mirror_tid;
786 if (hammer2_debug & 0x8000) {
787 /* debug only, avoid syslogd loop */
788 kprintf("sync freemap mirror_tid %08jx\n",
789 (intmax_t)chain->bref.mirror_tid);
793 * The freemap can be flushed independently of the
794 * main topology, but for the case where it is
795 * flushed in the same transaction, and flushed
796 * before vchain (a case we want to allow for
797 * performance reasons), make sure modifications
798 * made during the flush under vchain use a new
801 * Otherwise the mount recovery code will get confused.
803 ++hmp->voldata.mirror_tid;
805 case HAMMER2_BREF_TYPE_VOLUME:
807 * The free block table is flushed by
808 * hammer2_vfs_sync() before it flushes vchain.
809 * We must still hold fchain locked while copying
810 * voldata to volsync, however.
812 * These do not error per-say since their data does
813 * not need to be re-read from media on lock.
815 * (note: embedded data, do not call setdirty)
817 hammer2_chain_lock(&hmp->fchain,
818 HAMMER2_RESOLVE_ALWAYS);
819 hammer2_voldata_lock(hmp);
820 if (hammer2_debug & 0x8000) {
821 /* debug only, avoid syslogd loop */
822 kprintf("sync volume mirror_tid %08jx\n",
823 (intmax_t)chain->bref.mirror_tid);
827 * Update the volume header's mirror_tid to the
828 * main topology's flushing mirror_tid. It is
829 * possible that voldata.mirror_tid is already
830 * beyond bref.mirror_tid due to the bump we made
831 * above in BREF_TYPE_FREEMAP.
833 if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
834 hmp->voldata.mirror_tid =
835 chain->bref.mirror_tid;
839 * The volume header is flushed manually by the
840 * syncer, not here. All we do here is adjust the
843 KKASSERT(chain->data != NULL);
844 KKASSERT(chain->dio == NULL);
846 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
848 (char *)&hmp->voldata +
849 HAMMER2_VOLUME_ICRC1_OFF,
850 HAMMER2_VOLUME_ICRC1_SIZE);
851 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
853 (char *)&hmp->voldata +
854 HAMMER2_VOLUME_ICRC0_OFF,
855 HAMMER2_VOLUME_ICRC0_SIZE);
856 hmp->voldata.icrc_volheader =
858 (char *)&hmp->voldata +
859 HAMMER2_VOLUME_ICRCVH_OFF,
860 HAMMER2_VOLUME_ICRCVH_SIZE);
862 if (hammer2_debug & 0x8000) {
863 /* debug only, avoid syslogd loop */
864 kprintf("syncvolhdr %016jx %016jx\n",
865 hmp->voldata.mirror_tid,
866 hmp->vchain.bref.mirror_tid);
868 hmp->volsync = hmp->voldata;
869 atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
870 hammer2_voldata_unlock(hmp);
871 hammer2_chain_unlock(&hmp->fchain);
873 case HAMMER2_BREF_TYPE_DATA:
875 * Data elements have already been flushed via the
876 * logical file buffer cache. Their hash was set in
877 * the bref by the vop_write code. Do not re-dirty.
879 * Make sure any device buffer(s) have been flushed
880 * out here (there aren't usually any to flush) XXX.
883 case HAMMER2_BREF_TYPE_INDIRECT:
884 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
885 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
887 * Buffer I/O will be cleaned up when the volume is
888 * flushed (but the kernel is free to flush it before
891 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
892 hammer2_chain_setcheck(chain, chain->data);
894 case HAMMER2_BREF_TYPE_DIRENT:
896 * A directory entry can use the check area to store
897 * the filename for filenames <= 64 bytes, don't blow
900 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
902 hammer2_chain_setcheck(chain, chain->data);
904 case HAMMER2_BREF_TYPE_INODE:
906 * NOTE: We must call io_setdirty() to make any late
907 * changes to the inode data, the system might
908 * have already flushed the buffer.
910 if (chain->data->ipdata.meta.op_flags &
911 HAMMER2_OPFLAG_PFSROOT) {
913 * non-NULL pmp if mounted as a PFS. We must
914 * sync fields cached in the pmp? XXX
916 hammer2_inode_data_t *ipdata;
918 hammer2_io_setdirty(chain->dio);
919 ipdata = &chain->data->ipdata;
921 ipdata->meta.pfs_inum =
922 chain->pmp->inode_tid;
925 /* can't be mounted as a PFS */
928 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
929 hammer2_chain_setcheck(chain, chain->data);
932 KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
933 panic("hammer2_flush_core: unsupported "
940 * If the chain was destroyed try to avoid unnecessary I/O
941 * that might not have yet occurred. Remove the data range
942 * from dedup candidacy and attempt to invalidation that
943 * potentially dirty portion of the I/O buffer.
945 if (chain->flags & HAMMER2_CHAIN_DESTROY) {
946 hammer2_io_dedup_delete(hmp,
948 chain->bref.data_off,
953 hammer2_io_inval(chain->dio,
954 chain->bref.data_off,
956 } else if ((dio = hammer2_io_getquick(hmp,
957 chain->bref.data_off,
960 hammer2_io_inval(dio,
961 chain->bref.data_off,
963 hammer2_io_putblk(&dio);
970 * If UPDATE is set the parent block table may need to be updated.
971 * This can fail if the hammer2_chain_modify() fails.
973 * NOTE: UPDATE may be set on vchain or fchain in which case
974 * parent could be NULL, or on an inode that has not yet
975 * been inserted into the radix tree. It's easiest to allow
976 * the case and test for NULL. parent can also wind up being
977 * NULL due to a deletion so we need to handle the case anyway.
979 * NOTE: UPDATE can be set when chains are renamed into or out of
980 * an indirect block, without the chain itself being flagged
983 * If no parent exists we can just clear the UPDATE bit. If the
984 * chain gets reattached later on the bit will simply get set
987 if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL)
988 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
991 * When flushing an inode outside of a FLUSH_FSSYNC we must NOT
992 * update the parent block table to point at the flushed inode.
993 * The block table should only ever be updated by the filesystem
994 * sync code. If we do, inode<->inode dependencies (such as
995 * directory entries vs inode nlink count) can wind up not being
996 * flushed together and result in a broken topology if a crash/reboot
997 * occurs at the wrong time.
999 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
1000 (flags & HAMMER2_FLUSH_INODE_STOP) &&
1001 (flags & HAMMER2_FLUSH_FSSYNC) == 0 &&
1002 (flags & HAMMER2_FLUSH_ALL) == 0 &&
1003 chain->pmp && chain->pmp->mp) {
1004 #ifdef HAMMER2_DEBUG_SYNC
1005 kprintf("inum %ld do not update parent, non-fssync\n",
1006 (long)chain->bref.key);
1010 #ifdef HAMMER2_DEBUG_SYNC
1011 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE)
1012 kprintf("inum %ld update parent\n", (long)chain->bref.key);
1016 * The chain may need its blockrefs updated in the parent, normal
1019 if (chain->flags & HAMMER2_CHAIN_UPDATE) {
1020 hammer2_blockref_t *base;
1024 * Clear UPDATE flag, mark parent modified, update its
1025 * modify_tid if necessary, and adjust the parent blockmap.
1027 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1032 * Avoid actually modifying and updating the parent if it
1033 * was flagged for destruction. This can greatly reduce
1034 * disk I/O in large tree removals because the
1035 * hammer2_io_setinval() call in the upward recursion
1036 * (see MODIFIED code above) can only handle a few cases.
1038 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
1039 if (parent->bref.modify_tid < chain->bref.modify_tid) {
1040 parent->bref.modify_tid =
1041 chain->bref.modify_tid;
1043 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED |
1044 HAMMER2_CHAIN_BMAPUPD);
1049 * The flusher is responsible for deleting empty indirect
1050 * blocks at this point. If we don't do this, no major harm
1051 * will be done but the empty indirect blocks will stay in
1052 * the topology and make it a messy and inefficient.
1054 * The flusher is also responsible for collapsing the
1055 * content of an indirect block into its parent whenever
1056 * possible (with some hysteresis). Not doing this will also
1057 * not harm the topology, but would make it messy and
1060 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT) {
1061 if (hammer2_chain_indirect_maintenance(parent, chain))
1066 * We are updating the parent's blockmap, the parent must
1067 * be set modified. If this fails we re-set the UPDATE flag
1070 * NOTE! A modification error can be ENOSPC. We still want
1071 * to flush modified chains recursively, not break out,
1072 * so we just skip the update in this situation and
1073 * continue. That is, we still need to try to clean
1074 * out dirty chains and buffers.
1076 * This may not help bulkfree though. XXX
1078 save_error = hammer2_chain_modify(parent, 0, 0, 0);
1080 info->error |= save_error;
1081 kprintf("hammer2_flush: %016jx.%02x error=%08x\n",
1082 parent->bref.data_off, parent->bref.type,
1084 atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1087 if (parent->bref.modify_tid < chain->bref.modify_tid)
1088 parent->bref.modify_tid = chain->bref.modify_tid;
1091 * Calculate blockmap pointer
1093 switch(parent->bref.type) {
1094 case HAMMER2_BREF_TYPE_INODE:
1096 * Access the inode's block array. However, there is
1097 * no block array if the inode is flagged DIRECTDATA.
1100 (parent->data->ipdata.meta.op_flags &
1101 HAMMER2_OPFLAG_DIRECTDATA) == 0) {
1102 base = &parent->data->
1103 ipdata.u.blockset.blockref[0];
1107 count = HAMMER2_SET_COUNT;
1109 case HAMMER2_BREF_TYPE_INDIRECT:
1110 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1112 base = &parent->data->npdata[0];
1115 count = parent->bytes / sizeof(hammer2_blockref_t);
1117 case HAMMER2_BREF_TYPE_VOLUME:
1118 base = &chain->hmp->voldata.sroot_blockset.blockref[0];
1119 count = HAMMER2_SET_COUNT;
1121 case HAMMER2_BREF_TYPE_FREEMAP:
1122 base = &parent->data->npdata[0];
1123 count = HAMMER2_SET_COUNT;
1128 panic("hammer2_flush_core: "
1129 "unrecognized blockref type: %d",
1134 * Blocktable updates
1136 * We synchronize pending statistics at this time. Delta
1137 * adjustments designated for the current and upper level
1140 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) {
1141 if (chain->flags & HAMMER2_CHAIN_BMAPPED) {
1142 hammer2_spin_ex(&parent->core.spin);
1143 hammer2_base_delete(parent, base, count, chain,
1145 hammer2_spin_unex(&parent->core.spin);
1146 /* base_delete clears both bits */
1148 atomic_clear_int(&chain->flags,
1149 HAMMER2_CHAIN_BMAPUPD);
1152 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) {
1153 hammer2_spin_ex(&parent->core.spin);
1154 hammer2_base_insert(parent, base, count,
1155 chain, &chain->bref);
1156 hammer2_spin_unex(&parent->core.spin);
1157 /* base_insert sets BMAPPED */
1162 hammer2_chain_unlock(parent);
1165 * Final cleanup after flush
1168 KKASSERT(chain->refs > 0);
1169 if (hammer2_debug & 0x200) {
1170 if (info->debug == chain)
1177 * Flush recursion helper, called from flush_core, calls flush_core.
1179 * Flushes the children of the caller's chain (info->parent), restricted
1180 * by sync_tid. Set info->domodify if the child's blockref must propagate
1181 * back up to the parent.
1183 * This function may set info->error as a side effect.
1185 * Ripouts can move child from rbtree to dbtree or dbq but the caller's
1186 * flush scan order prevents any chains from being lost. A child can be
1187 * executes more than once.
1189 * WARNING! If we do not call hammer2_flush_core() we must update
1190 * bref.mirror_tid ourselves to indicate that the flush has
1191 * processed the child.
1193 * WARNING! parent->core spinlock is held on entry and return.
1196 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
1198 hammer2_flush_info_t *info = data;
1199 hammer2_chain_t *parent = info->parent;
1201 #ifdef HAMMER2_SCAN_DEBUG
1203 if (child->flags & HAMMER2_CHAIN_MODIFIED)
1204 ++info->scan_mod_count;
1205 if (child->flags & HAMMER2_CHAIN_UPDATE)
1206 ++info->scan_upd_count;
1207 if (child->flags & HAMMER2_CHAIN_ONFLUSH)
1208 ++info->scan_onf_count;
1212 * (child can never be fchain or vchain so a special check isn't
1215 * We must ref the child before unlocking the spinlock.
1217 * The caller has added a ref to the parent so we can temporarily
1218 * unlock it in order to lock the child. However, if it no longer
1219 * winds up being the child of the parent we must skip this child.
1221 * NOTE! chain locking errors are fatal. They are never out-of-space
1224 hammer2_chain_ref(child);
1225 hammer2_spin_unex(&parent->core.spin);
1227 hammer2_chain_ref_hold(parent);
1228 hammer2_chain_unlock(parent);
1229 hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
1230 if (child->parent != parent) {
1231 kprintf("LOST CHILD1 %p->%p (actual parent %p)\n",
1232 parent, child, child->parent);
1236 kprintf("CHILD ERROR DURING FLUSH LOCK %p->%p\n",
1238 info->error |= child->error;
1243 * Must propagate the DESTROY flag downwards, otherwise the
1244 * parent could end up never being removed because it will
1245 * be requeued to the flusher if it survives this run due to
1248 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
1249 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROY);
1250 #ifdef HAMMER2_SCAN_DEBUG
1251 if (child->flags & HAMMER2_CHAIN_DESTROY)
1252 ++info->scan_del_count;
1255 * Special handling of the root inode. Because the root inode
1256 * contains an index of all the inodes in the PFS in addition to
1257 * its normal directory entries, any flush that is not part of a
1258 * filesystem sync must only flush the directory entries, and not
1261 * The child might be an indirect block, but H2 guarantees that
1262 * the key-range will fully partition the inode index from the
1263 * directory entries so the case just works naturally.
1265 if ((parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
1266 (child->flags & HAMMER2_CHAIN_DESTROY) == 0 &&
1267 parent->bref.type == HAMMER2_BREF_TYPE_INODE &&
1268 (info->flags & HAMMER2_FLUSH_FSSYNC) == 0) {
1269 if ((child->bref.key & HAMMER2_DIRHASH_VISIBLE) == 0) {
1270 if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1271 hammer2_chain_setflush(parent);
1273 kprintf("inum %ld do not dive root inode\n",
1274 (long)parent->bref.key);
1280 * Recurse and collect deferral data. We're in the media flush,
1281 * this can cross PFS boundaries.
1283 if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1284 #ifdef HAMMER2_SCAN_DEBUG
1285 if (child->bref.type < 7)
1286 ++info->scan_btype[child->bref.type];
1289 hammer2_flush_core(info, child, info->flags);
1291 } else if (hammer2_debug & 0x200) {
1292 if (info->debug == NULL)
1293 info->debug = child;
1295 hammer2_flush_core(info, child, info->flags);
1297 if (info->debug == child)
1303 * Relock to continue the loop.
1305 hammer2_chain_unlock(child);
1306 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1307 hammer2_chain_drop_unhold(parent);
1308 if (parent->error) {
1309 kprintf("PARENT ERROR DURING FLUSH LOCK %p->%p\n",
1311 info->error |= parent->error;
1313 hammer2_chain_drop(child);
1314 KKASSERT(info->parent == parent);
1315 hammer2_spin_ex(&parent->core.spin);
1321 * flush helper (backend threaded)
1323 * Flushes chain topology for the specified inode.
1325 * HAMMER2_XOP_INODE_STOP The flush recursion stops at inode boundaries.
1326 * Inodes belonging to the same flush are flushed
1329 * HAMMER2_XOP_PARENTONFLUSH After flushing if the starting chain indicates
1330 * a parent update is needed, we setflush the
1331 * parent to propogate the flush request across
1334 * chain->parent can be NULL, usually due to destroy races or detached inodes.
1336 * Primarily called from vfs_sync().
1339 hammer2_xop_inode_flush(hammer2_xop_t *arg, void *scratch __unused, int clindex)
1341 hammer2_xop_flush_t *xop = &arg->xop_flush;
1342 hammer2_chain_t *chain;
1343 hammer2_inode_t *ip;
1346 int flush_error = 0;
1347 int fsync_error = 0;
1348 int total_error = 0;
1353 xflags = HAMMER2_FLUSH_TOP;
1354 if (xop->head.flags & HAMMER2_XOP_INODE_STOP)
1355 xflags |= HAMMER2_FLUSH_INODE_STOP;
1356 if (xop->head.flags & HAMMER2_XOP_FSSYNC)
1357 xflags |= HAMMER2_FLUSH_FSSYNC;
1364 chain = hammer2_inode_chain(ip, clindex, HAMMER2_RESOLVE_ALWAYS);
1367 if (chain->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1369 * Due to flush partitioning the chain topology
1370 * above the inode's chain may no longer be flagged.
1371 * When asked to flush an inode, remark the topology
1372 * leading to that inode.
1375 hammer2_chain_setflush(chain->parent);
1376 hammer2_flush(chain, xflags);
1379 if (ip == pmp->iroot && pmp != hmp->spmp) {
1380 hammer2_spin_ex(&pmp->inum_spin);
1381 pmp->pfs_iroot_blocksets[clindex] =
1382 chain->data->ipdata.u.blockset;
1383 hammer2_spin_unex(&pmp->inum_spin);
1388 * Propogate upwards but only cross an inode boundary
1389 * for inodes associated with the current filesystem
1392 if ((xop->head.flags & HAMMER2_XOP_PARENTONFLUSH) ||
1393 chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
1394 parent = chain->parent;
1396 hammer2_chain_setflush(parent);
1400 if (chain->flags & HAMMER2_CHAIN_PFSBOUNDARY)
1402 hammer2_chain_unlock(chain);
1403 hammer2_chain_drop(chain);
1410 * Only flush the volume header if asked to, plus the inode must also
1413 if ((xop->head.flags & HAMMER2_XOP_VOLHDR) == 0)
1419 * Flush volume roots. Avoid replication, we only want to
1420 * flush each hammer2_dev (hmp) once.
1422 for (j = clindex - 1; j >= 0; --j) {
1423 if ((chain = ip->cluster.array[j].chain) != NULL) {
1424 if (chain->hmp == hmp) {
1425 chain = NULL; /* safety */
1430 chain = NULL; /* safety */
1433 * spmp transaction. The super-root is never directly mounted so
1434 * there shouldn't be any vnodes, let alone any dirty vnodes
1435 * associated with it, so we shouldn't have to mess around with any
1436 * vnode flushes here.
1438 hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1441 * We must flush the superroot down to the PFS iroot. Remember
1442 * that hammer2_chain_setflush() stops at inode boundaries, so
1443 * the pmp->iroot has been flushed and flagged down to the superroot,
1444 * but the volume root (vchain) probably has not yet been flagged.
1446 if (hmp->spmp->iroot) {
1447 chain = hmp->spmp->iroot->cluster.array[0].chain;
1449 hammer2_chain_ref(chain);
1450 hammer2_chain_lock(chain, HAMMER2_RESOLVE_ALWAYS);
1452 hammer2_flush(chain,
1454 HAMMER2_FLUSH_INODE_STOP |
1455 HAMMER2_FLUSH_FSSYNC);
1456 hammer2_chain_unlock(chain);
1457 hammer2_chain_drop(chain);
1462 * Media mounts have two 'roots', vchain for the topology
1463 * and fchain for the free block table. Flush both.
1465 * Note that the topology and free block table are handled
1466 * independently, so the free block table can wind up being
1467 * ahead of the topology. We depend on the bulk free scan
1468 * code to deal with any loose ends.
1470 * vchain and fchain do not error on-lock since their data does
1471 * not have to be re-read from media.
1473 hammer2_chain_ref(&hmp->vchain);
1474 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1475 hammer2_chain_ref(&hmp->fchain);
1476 hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1477 if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1479 * This will also modify vchain as a side effect,
1480 * mark vchain as modified now.
1482 hammer2_voldata_modify(hmp);
1483 chain = &hmp->fchain;
1484 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1485 KKASSERT(chain == &hmp->fchain);
1487 hammer2_chain_unlock(&hmp->fchain);
1488 hammer2_chain_unlock(&hmp->vchain);
1489 hammer2_chain_drop(&hmp->fchain);
1490 /* vchain dropped down below */
1492 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1493 if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1494 chain = &hmp->vchain;
1495 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1496 KKASSERT(chain == &hmp->vchain);
1498 hammer2_chain_unlock(&hmp->vchain);
1499 hammer2_chain_drop(&hmp->vchain);
1502 * We can't safely flush the volume header until we have
1503 * flushed any device buffers which have built up.
1505 * XXX this isn't being incremental
1507 vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
1508 fsync_error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
1509 vn_unlock(hmp->devvp);
1510 if (fsync_error || flush_error) {
1511 kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n",
1512 fsync_error, flush_error, hmp->devrepname);
1516 * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1517 * volume header needs synchronization via hmp->volsync.
1519 * XXX synchronize the flag & data with only this flush XXX
1521 if (fsync_error == 0 && flush_error == 0 &&
1522 (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1527 * Synchronize the disk before flushing the volume
1531 bp->b_bio1.bio_offset = 0;
1534 bp->b_cmd = BUF_CMD_FLUSH;
1535 bp->b_bio1.bio_done = biodone_sync;
1536 bp->b_bio1.bio_flags |= BIO_SYNC;
1537 vn_strategy(hmp->devvp, &bp->b_bio1);
1538 fsync_error = biowait(&bp->b_bio1, "h2vol");
1542 * Then we can safely flush the version of the
1543 * volume header synchronized by the flush code.
1545 j = hmp->volhdrno + 1;
1548 if (j >= HAMMER2_NUM_VOLHDRS)
1550 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1551 hmp->volsync.volu_size) {
1554 if (hammer2_debug & 0x8000) {
1555 /* debug only, avoid syslogd loop */
1556 kprintf("sync volhdr %d %jd\n",
1557 j, (intmax_t)hmp->volsync.volu_size);
1559 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1560 HAMMER2_PBUFSIZE, GETBLK_KVABIO, 0);
1561 atomic_clear_int(&hmp->vchain.flags,
1562 HAMMER2_CHAIN_VOLUMESYNC);
1564 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1565 vol_error = bwrite(bp);
1568 fsync_error = vol_error;
1571 total_error = flush_error;
1573 total_error = hammer2_errno_to_error(fsync_error);
1576 hammer2_trans_done(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1578 hammer2_xop_feed(&xop->head, NULL, clindex, total_error);