2 * Copyright (c) 2011-2015 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * TRANSACTION AND FLUSH HANDLING
38 * Deceptively simple but actually fairly difficult to implement properly is
39 * how I would describe it.
41 * Flushing generally occurs bottom-up but requires a top-down scan to
42 * locate chains with MODIFIED and/or UPDATE bits set. The ONFLUSH flag
43 * tells how to recurse downward to find these chains.
46 #include <sys/cdefs.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/types.h>
57 #define HAMMER2_FLUSH_DEPTH_LIMIT 10 /* stack recursion limit */
61 * Recursively flush the specified chain. The chain is locked and
62 * referenced by the caller and will remain so on return. The chain
63 * will remain referenced throughout but can temporarily lose its
64 * lock during the recursion to avoid unnecessarily stalling user
67 struct hammer2_flush_info {
68 hammer2_chain_t *parent;
73 struct h2_flush_list flushq;
74 hammer2_chain_t *debug;
77 typedef struct hammer2_flush_info hammer2_flush_info_t;
79 static void hammer2_flush_core(hammer2_flush_info_t *info,
80 hammer2_chain_t *chain, int flags);
81 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
84 * Any per-pfs transaction initialization goes here.
87 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
92 * Transaction support for any modifying operation. Transactions are used
93 * in the pmp layer by the frontend and in the spmp layer by the backend.
95 * 0 - Normal transaction, interlocked against flush
98 * TRANS_ISFLUSH - Flush transaction, interlocked against normal
101 * TRANS_BUFCACHE - Buffer cache transaction, no interlock.
103 * Initializing a new transaction allocates a transaction ID. Typically
104 * passed a pmp (hmp passed as NULL), indicating a cluster transaction. Can
105 * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
106 * media target. The latter mode is used by the recovery code.
108 * TWO TRANSACTION IDs can run concurrently, where one is a flush and the
109 * other is a set of any number of concurrent filesystem operations. We
110 * can either have <running_fs_ops> + <waiting_flush> + <blocked_fs_ops>
111 * or we can have <running_flush> + <concurrent_fs_ops>.
113 * During a flush, new fs_ops are only blocked until the fs_ops prior to
114 * the flush complete. The new fs_ops can then run concurrent with the flush.
116 * Buffer-cache transactions operate as fs_ops but never block. A
117 * buffer-cache flush will run either before or after the current pending
118 * flush depending on its state.
121 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
128 oflags = pmp->trans.flags;
132 if (flags & HAMMER2_TRANS_ISFLUSH) {
134 * Requesting flush transaction. Wait for all
135 * currently running transactions to finish.
137 if (oflags & HAMMER2_TRANS_MASK) {
138 nflags = oflags | HAMMER2_TRANS_FPENDING |
139 HAMMER2_TRANS_WAITING;
142 nflags = (oflags | flags) + 1;
144 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
146 * Requesting strategy transaction. Generally
147 * allowed in all situations unless a flush
148 * is running without the preflush flag.
150 if ((oflags & (HAMMER2_TRANS_ISFLUSH |
151 HAMMER2_TRANS_PREFLUSH)) ==
152 HAMMER2_TRANS_ISFLUSH) {
153 nflags = oflags | HAMMER2_TRANS_WAITING;
156 nflags = (oflags | flags) + 1;
160 * Requesting normal transaction. Wait for any
161 * flush to finish before allowing.
163 if (oflags & HAMMER2_TRANS_ISFLUSH) {
164 nflags = oflags | HAMMER2_TRANS_WAITING;
167 nflags = (oflags | flags) + 1;
171 tsleep_interlock(&pmp->trans.sync_wait, 0);
172 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
175 tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
185 * Start a sub-transaction, there is no 'subdone' function. This will
186 * issue a new modify_tid (mtid) for the current transaction, which is a
187 * CLC (cluster level change) id and not a per-node id.
189 * This function must be called for each XOP when multiple XOPs are run in
190 * sequence within a transaction.
192 * Callers typically update the inode with the transaction mtid manually
193 * to enforce sequencing.
196 hammer2_trans_sub(hammer2_pfs_t *pmp)
200 mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
206 * Clears the PREFLUSH stage, called during a flush transaction after all
207 * logical buffer I/O has completed.
210 hammer2_trans_clear_preflush(hammer2_pfs_t *pmp)
212 atomic_clear_int(&pmp->trans.flags, HAMMER2_TRANS_PREFLUSH);
216 hammer2_trans_done(hammer2_pfs_t *pmp)
222 oflags = pmp->trans.flags;
224 KKASSERT(oflags & HAMMER2_TRANS_MASK);
225 if ((oflags & HAMMER2_TRANS_MASK) == 1) {
227 * This was the last transaction
229 nflags = (oflags - 1) & ~(HAMMER2_TRANS_ISFLUSH |
230 HAMMER2_TRANS_BUFCACHE |
231 HAMMER2_TRANS_PREFLUSH |
232 HAMMER2_TRANS_FPENDING |
233 HAMMER2_TRANS_WAITING);
236 * Still transactions pending
240 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
241 if ((nflags & HAMMER2_TRANS_MASK) == 0 &&
242 (oflags & HAMMER2_TRANS_WAITING)) {
243 wakeup(&pmp->trans.sync_wait);
254 * Obtain new, unique inode number (not serialized by caller).
257 hammer2_trans_newinum(hammer2_pfs_t *pmp)
261 tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
267 * Assert that a strategy call is ok here. Strategy calls are legal
269 * (1) In a normal transaction.
270 * (2) In a flush transaction only if PREFLUSH is also set.
273 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
275 KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 ||
276 (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH));
281 * Chains undergoing destruction are removed from the in-memory topology.
282 * To avoid getting lost these chains are placed on the delayed flush
283 * queue which will properly dispose of them.
285 * We do this instead of issuing an immediate flush in order to give
286 * recursive deletions (rm -rf, etc) a chance to remove more of the
287 * hierarchy, potentially allowing an enormous amount of write I/O to
291 hammer2_delayed_flush(hammer2_chain_t *chain)
293 if ((chain->flags & HAMMER2_CHAIN_DELAYED) == 0) {
294 hammer2_spin_ex(&chain->hmp->list_spin);
295 if ((chain->flags & (HAMMER2_CHAIN_DELAYED |
296 HAMMER2_CHAIN_DEFERRED)) == 0) {
297 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELAYED |
298 HAMMER2_CHAIN_DEFERRED);
299 TAILQ_INSERT_TAIL(&chain->hmp->flushq,
301 hammer2_chain_ref(chain);
303 hammer2_spin_unex(&chain->hmp->list_spin);
308 * Flush the chain and all modified sub-chains through the specified
309 * synchronization point, propagating blockref updates back up. As
310 * part of this propagation, mirror_tid and inode/data usage statistics
311 * propagates back upward.
313 * modify_tid (clc - cluster level change) is not propagated.
315 * update_tid (clc) is used for validation and is not propagated by this
318 * This routine can be called from several places but the most important
319 * is from VFS_SYNC (frontend) via hammer2_inode_xop_flush (backend).
321 * chain is locked on call and will remain locked on return. The chain's
322 * UPDATE flag indicates that its parent's block table (which is not yet
323 * part of the flush) should be updated. The chain may be replaced by
324 * the call if it was modified.
327 hammer2_flush(hammer2_chain_t *chain, int flags)
329 hammer2_chain_t *scan;
330 hammer2_flush_info_t info;
335 * Execute the recursive flush and handle deferrals.
337 * Chains can be ridiculously long (thousands deep), so to
338 * avoid blowing out the kernel stack the recursive flush has a
339 * depth limit. Elements at the limit are placed on a list
340 * for re-execution after the stack has been popped.
342 bzero(&info, sizeof(info));
343 TAILQ_INIT(&info.flushq);
344 info.cache_index = -1;
345 info.flags = flags & ~HAMMER2_FLUSH_TOP;
348 * Calculate parent (can be NULL), if not NULL the flush core
349 * expects the parent to be referenced so it can easily lock/unlock
350 * it without it getting ripped up.
352 if ((info.parent = chain->parent) != NULL)
353 hammer2_chain_ref(info.parent);
356 * Extra ref needed because flush_core expects it when replacing
359 hammer2_chain_ref(chain);
365 * Move hmp->flushq to info.flushq if non-empty so it can
368 if (TAILQ_FIRST(&hmp->flushq) != NULL) {
369 hammer2_spin_ex(&chain->hmp->list_spin);
370 TAILQ_CONCAT(&info.flushq, &hmp->flushq, flush_node);
371 hammer2_spin_unex(&chain->hmp->list_spin);
375 * Unwind deep recursions which had been deferred. This
376 * can leave the FLUSH_* bits set for these chains, which
377 * will be handled when we [re]flush chain after the unwind.
379 while ((scan = TAILQ_FIRST(&info.flushq)) != NULL) {
380 KKASSERT(scan->flags & HAMMER2_CHAIN_DEFERRED);
381 TAILQ_REMOVE(&info.flushq, scan, flush_node);
382 atomic_clear_int(&scan->flags, HAMMER2_CHAIN_DEFERRED |
383 HAMMER2_CHAIN_DELAYED);
386 * Now that we've popped back up we can do a secondary
387 * recursion on the deferred elements.
389 * NOTE: hammer2_flush() may replace scan.
391 if (hammer2_debug & 0x0040)
392 kprintf("deferred flush %p\n", scan);
393 hammer2_chain_lock(scan, HAMMER2_RESOLVE_MAYBE);
394 hammer2_flush(scan, flags & ~HAMMER2_FLUSH_TOP);
395 hammer2_chain_unlock(scan);
396 hammer2_chain_drop(scan); /* ref from deferral */
402 info.diddeferral = 0;
403 hammer2_flush_core(&info, chain, flags);
406 * Only loop if deep recursions have been deferred.
408 if (TAILQ_EMPTY(&info.flushq))
411 if (++loops % 1000 == 0) {
412 kprintf("hammer2_flush: excessive loops on %p\n",
414 if (hammer2_debug & 0x100000)
418 hammer2_chain_drop(chain);
420 hammer2_chain_drop(info.parent);
424 * This is the core of the chain flushing code. The chain is locked by the
425 * caller and must also have an extra ref on it by the caller, and remains
426 * locked and will have an extra ref on return. Upon return, the caller can
427 * test the UPDATE bit on the child to determine if the parent needs updating.
429 * (1) Determine if this node is a candidate for the flush, return if it is
430 * not. fchain and vchain are always candidates for the flush.
432 * (2) If we recurse too deep the chain is entered onto the deferral list and
433 * the current flush stack is aborted until after the deferral list is
436 * (3) Recursively flush live children (rbtree). This can create deferrals.
437 * A successful flush clears the MODIFIED and UPDATE bits on the children
438 * and typically causes the parent to be marked MODIFIED as the children
439 * update the parent's block table. A parent might already be marked
440 * MODIFIED due to a deletion (whos blocktable update in the parent is
441 * handled by the frontend), or if the parent itself is modified by the
442 * frontend for other reasons.
444 * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
445 * Deleted-but-open inodes can still be individually flushed via the
448 * (5) Note that an unmodified child may still need the block table in its
449 * parent updated (e.g. rename/move). The child will have UPDATE set
452 * WARNING ON BREF MODIFY_TID/MIRROR_TID
454 * blockref.modify_tid is consistent only within a PFS, and will not be
455 * consistent during synchronization. mirror_tid is consistent across the
456 * block device regardless of the PFS.
459 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
462 hammer2_chain_t *parent;
467 * (1) Optimize downward recursion to locate nodes needing action.
468 * Nothing to do if none of these flags are set.
470 if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) {
471 if (hammer2_debug & 0x200) {
472 if (info->debug == NULL)
480 diddeferral = info->diddeferral;
481 parent = info->parent; /* can be NULL */
484 * Downward search recursion
486 if (chain->flags & (HAMMER2_CHAIN_DEFERRED | HAMMER2_CHAIN_DELAYED)) {
491 } else if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) &&
492 (flags & HAMMER2_FLUSH_ALL) == 0 &&
493 (flags & HAMMER2_FLUSH_TOP) == 0) {
495 * We do not recurse through PFSROOTs. PFSROOT flushes are
496 * handled by the related pmp's (whether mounted or not,
497 * including during recovery).
499 * But we must still process the PFSROOT chains for block
500 * table updates in their parent (which IS part of our flush).
502 * Note that the volume root, vchain, does not set this flag.
503 * Note the logic here requires that this test be done before
504 * the depth-limit test, else it might become the top on a
508 } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
510 * Recursion depth reached.
512 KKASSERT((chain->flags & HAMMER2_CHAIN_DELAYED) == 0);
513 hammer2_chain_ref(chain);
514 TAILQ_INSERT_TAIL(&info->flushq, chain, flush_node);
515 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEFERRED);
517 } else if (chain->flags & HAMMER2_CHAIN_ONFLUSH) {
519 * Downward recursion search (actual flush occurs bottom-up).
520 * pre-clear ONFLUSH. It can get set again due to races,
521 * which we want so the scan finds us again in the next flush.
523 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
524 info->parent = chain;
525 hammer2_spin_ex(&chain->core.spin);
526 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
527 NULL, hammer2_flush_recurse, info);
528 hammer2_spin_unex(&chain->core.spin);
529 info->parent = parent;
530 if (info->diddeferral)
531 hammer2_chain_setflush(chain);
535 * Now we are in the bottom-up part of the recursion.
537 * Do not update chain if lower layers were deferred.
539 if (info->diddeferral)
543 * Propagate the DESTROY flag downwards. This dummies up the flush
544 * code and tries to invalidate related buffer cache buffers to
545 * avoid the disk write.
547 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
548 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
551 * Chain was already modified or has become modified, flush it out.
554 if ((hammer2_debug & 0x200) &&
556 (chain->flags & (HAMMER2_CHAIN_MODIFIED | HAMMER2_CHAIN_UPDATE))) {
557 hammer2_chain_t *scan = chain;
559 kprintf("DISCONNECTED FLUSH %p->%p\n", info->debug, chain);
561 kprintf(" chain %p [%08x] bref=%016jx:%02x\n",
563 scan->bref.key, scan->bref.type);
564 if (scan == info->debug)
570 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
572 * Dispose of the modified bit.
574 * UPDATE should already be set.
575 * bref.mirror_tid should already be set.
577 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
578 chain == &hmp->vchain);
579 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
582 * Manage threads waiting for excessive dirty memory to
586 hammer2_pfs_memory_wakeup(chain->pmp);
588 if ((chain->flags & HAMMER2_CHAIN_UPDATE) ||
589 chain == &hmp->vchain ||
590 chain == &hmp->fchain) {
592 * Drop the ref from the MODIFIED bit we cleared,
595 hammer2_chain_drop(chain);
598 * Drop the ref from the MODIFIED bit we cleared and
599 * set a ref for the UPDATE bit we are setting. Net
602 atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
606 * Issue the flush. This is indirect via the DIO.
608 * NOTE: A DELETED node that reaches this point must be
609 * flushed for synchronization point consistency.
611 * NOTE: Even though MODIFIED was already set, the related DIO
612 * might not be dirty due to a system buffer cache
613 * flush and must be set dirty if we are going to make
614 * further modifications to the buffer. Chains with
615 * embedded data don't need this.
617 if (hammer2_debug & 0x1000) {
618 kprintf("Flush %p.%d %016jx/%d data=%016jx",
619 chain, chain->bref.type,
620 (uintmax_t)chain->bref.key,
622 (uintmax_t)chain->bref.data_off);
624 if (hammer2_debug & 0x2000) {
625 Debugger("Flush hell");
629 * Update chain CRCs for flush.
631 * NOTE: Volume headers are NOT flushed here as they require
632 * special processing.
634 switch(chain->bref.type) {
635 case HAMMER2_BREF_TYPE_FREEMAP:
637 * Update the volume header's freemap_tid to the
638 * freemap's flushing mirror_tid.
640 * (note: embedded data, do not call setdirty)
642 KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
643 KKASSERT(chain == &hmp->fchain);
644 hmp->voldata.freemap_tid = chain->bref.mirror_tid;
645 if (hammer2_debug & 0x8000) {
646 /* debug only, avoid syslogd loop */
647 kprintf("sync freemap mirror_tid %08jx\n",
648 (intmax_t)chain->bref.mirror_tid);
652 * The freemap can be flushed independently of the
653 * main topology, but for the case where it is
654 * flushed in the same transaction, and flushed
655 * before vchain (a case we want to allow for
656 * performance reasons), make sure modifications
657 * made during the flush under vchain use a new
660 * Otherwise the mount recovery code will get confused.
662 ++hmp->voldata.mirror_tid;
664 case HAMMER2_BREF_TYPE_VOLUME:
666 * The free block table is flushed by
667 * hammer2_vfs_sync() before it flushes vchain.
668 * We must still hold fchain locked while copying
669 * voldata to volsync, however.
671 * (note: embedded data, do not call setdirty)
673 hammer2_chain_lock(&hmp->fchain,
674 HAMMER2_RESOLVE_ALWAYS);
675 hammer2_voldata_lock(hmp);
676 if (hammer2_debug & 0x8000) {
677 /* debug only, avoid syslogd loop */
678 kprintf("sync volume mirror_tid %08jx\n",
679 (intmax_t)chain->bref.mirror_tid);
683 * Update the volume header's mirror_tid to the
684 * main topology's flushing mirror_tid. It is
685 * possible that voldata.mirror_tid is already
686 * beyond bref.mirror_tid due to the bump we made
687 * above in BREF_TYPE_FREEMAP.
689 if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
690 hmp->voldata.mirror_tid =
691 chain->bref.mirror_tid;
695 * The volume header is flushed manually by the
696 * syncer, not here. All we do here is adjust the
699 KKASSERT(chain->data != NULL);
700 KKASSERT(chain->dio == NULL);
702 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
704 (char *)&hmp->voldata +
705 HAMMER2_VOLUME_ICRC1_OFF,
706 HAMMER2_VOLUME_ICRC1_SIZE);
707 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
709 (char *)&hmp->voldata +
710 HAMMER2_VOLUME_ICRC0_OFF,
711 HAMMER2_VOLUME_ICRC0_SIZE);
712 hmp->voldata.icrc_volheader =
714 (char *)&hmp->voldata +
715 HAMMER2_VOLUME_ICRCVH_OFF,
716 HAMMER2_VOLUME_ICRCVH_SIZE);
718 if (hammer2_debug & 0x8000) {
719 /* debug only, avoid syslogd loop */
720 kprintf("syncvolhdr %016jx %016jx\n",
721 hmp->voldata.mirror_tid,
722 hmp->vchain.bref.mirror_tid);
724 hmp->volsync = hmp->voldata;
725 atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
726 hammer2_voldata_unlock(hmp);
727 hammer2_chain_unlock(&hmp->fchain);
729 case HAMMER2_BREF_TYPE_DATA:
731 * Data elements have already been flushed via the
732 * logical file buffer cache. Their hash was set in
733 * the bref by the vop_write code. Do not re-dirty.
735 * Make sure any device buffer(s) have been flushed
736 * out here (there aren't usually any to flush) XXX.
739 case HAMMER2_BREF_TYPE_INDIRECT:
740 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
741 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
743 * Buffer I/O will be cleaned up when the volume is
744 * flushed (but the kernel is free to flush it before
747 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
748 hammer2_chain_setcheck(chain, chain->data);
750 case HAMMER2_BREF_TYPE_INODE:
752 * NOTE: We must call io_setdirty() to make any late
753 * changes to the inode data, the system might
754 * have already flushed the buffer.
756 if (chain->data->ipdata.meta.op_flags &
757 HAMMER2_OPFLAG_PFSROOT) {
759 * non-NULL pmp if mounted as a PFS. We must
760 * sync fields cached in the pmp? XXX
762 hammer2_inode_data_t *ipdata;
764 hammer2_io_setdirty(chain->dio);
765 ipdata = &chain->data->ipdata;
767 ipdata->meta.pfs_inum =
768 chain->pmp->inode_tid;
771 /* can't be mounted as a PFS */
774 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
775 hammer2_chain_setcheck(chain, chain->data);
778 KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
779 panic("hammer2_flush_core: unsupported "
786 * If the chain was destroyed try to avoid unnecessary I/O.
787 * (this only really works if the DIO system buffer is the
788 * same size as chain->bytes).
790 if ((chain->flags & HAMMER2_CHAIN_DESTROY) &&
791 (chain->flags & HAMMER2_CHAIN_DEDUP) == 0 &&
793 hammer2_io_setinval(chain->dio, chain->bytes);
798 * If UPDATE is set the parent block table may need to be updated.
800 * NOTE: UPDATE may be set on vchain or fchain in which case
801 * parent could be NULL. It's easiest to allow the case
802 * and test for NULL. parent can also wind up being NULL
803 * due to a deletion so we need to handle the case anyway.
805 * If no parent exists we can just clear the UPDATE bit. If the
806 * chain gets reattached later on the bit will simply get set
809 if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL) {
810 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
811 hammer2_chain_drop(chain);
815 * The chain may need its blockrefs updated in the parent. This
816 * requires some fancy footwork.
818 if (chain->flags & HAMMER2_CHAIN_UPDATE) {
819 hammer2_blockref_t *base;
823 * Both parent and chain must be locked. This requires
824 * temporarily unlocking the chain. We have to deal with
825 * the case where the chain might be reparented or modified
826 * while it was unlocked.
828 hammer2_chain_unlock(chain);
829 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
830 hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
831 if (chain->parent != parent) {
832 kprintf("PARENT MISMATCH ch=%p p=%p/%p\n",
833 chain, chain->parent, parent);
834 hammer2_chain_unlock(parent);
839 * Check race condition. If someone got in and modified
840 * it again while it was unlocked, we have to loop up.
842 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
843 hammer2_chain_unlock(parent);
844 kprintf("hammer2_flush: chain %p flush-mod race\n",
850 * Clear UPDATE flag, mark parent modified, update its
851 * modify_tid if necessary, and adjust the parent blockmap.
853 if (chain->flags & HAMMER2_CHAIN_UPDATE) {
854 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
855 hammer2_chain_drop(chain);
861 * Avoid actually modifying and updating the parent if it
862 * was flagged for destruction. This can greatly reduce
863 * disk I/O in large tree removals because the
864 * hammer2_io_setinval() call in the upward recursion
865 * (see MODIFIED code above) can only handle a few cases.
867 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
868 if (parent->bref.modify_tid < chain->bref.modify_tid) {
869 parent->bref.modify_tid =
870 chain->bref.modify_tid;
872 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED |
873 HAMMER2_CHAIN_BMAPUPD);
874 hammer2_chain_unlock(parent);
879 * We are updating the parent's blockmap, the parent must
882 hammer2_chain_modify(parent, 0, 0, 0);
883 if (parent->bref.modify_tid < chain->bref.modify_tid)
884 parent->bref.modify_tid = chain->bref.modify_tid;
887 * Calculate blockmap pointer
889 switch(parent->bref.type) {
890 case HAMMER2_BREF_TYPE_INODE:
892 * Access the inode's block array. However, there is
893 * no block array if the inode is flagged DIRECTDATA.
896 (parent->data->ipdata.meta.op_flags &
897 HAMMER2_OPFLAG_DIRECTDATA) == 0) {
898 base = &parent->data->
899 ipdata.u.blockset.blockref[0];
903 count = HAMMER2_SET_COUNT;
905 case HAMMER2_BREF_TYPE_INDIRECT:
906 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
908 base = &parent->data->npdata[0];
911 count = parent->bytes / sizeof(hammer2_blockref_t);
913 case HAMMER2_BREF_TYPE_VOLUME:
914 base = &chain->hmp->voldata.sroot_blockset.blockref[0];
915 count = HAMMER2_SET_COUNT;
917 case HAMMER2_BREF_TYPE_FREEMAP:
918 base = &parent->data->npdata[0];
919 count = HAMMER2_SET_COUNT;
924 panic("hammer2_flush_core: "
925 "unrecognized blockref type: %d",
932 * We synchronize pending statistics at this time. Delta
933 * adjustments designated for the current and upper level
936 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) {
937 if (chain->flags & HAMMER2_CHAIN_BMAPPED) {
938 hammer2_spin_ex(&parent->core.spin);
939 hammer2_base_delete(parent, base, count,
940 &info->cache_index, chain);
941 hammer2_spin_unex(&parent->core.spin);
942 /* base_delete clears both bits */
944 atomic_clear_int(&chain->flags,
945 HAMMER2_CHAIN_BMAPUPD);
948 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) {
949 hammer2_spin_ex(&parent->core.spin);
950 hammer2_base_insert(parent, base, count,
951 &info->cache_index, chain);
952 hammer2_spin_unex(&parent->core.spin);
953 /* base_insert sets BMAPPED */
955 hammer2_chain_unlock(parent);
961 * Final cleanup after flush
964 KKASSERT(chain->refs > 0);
965 if (hammer2_debug & 0x200) {
966 if (info->debug == chain)
972 * Flush recursion helper, called from flush_core, calls flush_core.
974 * Flushes the children of the caller's chain (info->parent), restricted
975 * by sync_tid. Set info->domodify if the child's blockref must propagate
976 * back up to the parent.
978 * Ripouts can move child from rbtree to dbtree or dbq but the caller's
979 * flush scan order prevents any chains from being lost. A child can be
980 * executes more than once.
982 * WARNING! If we do not call hammer2_flush_core() we must update
983 * bref.mirror_tid ourselves to indicate that the flush has
984 * processed the child.
986 * WARNING! parent->core spinlock is held on entry and return.
989 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
991 hammer2_flush_info_t *info = data;
992 hammer2_chain_t *parent = info->parent;
995 * (child can never be fchain or vchain so a special check isn't
998 * We must ref the child before unlocking the spinlock.
1000 * The caller has added a ref to the parent so we can temporarily
1001 * unlock it in order to lock the child.
1003 hammer2_chain_ref(child);
1004 hammer2_spin_unex(&parent->core.spin);
1006 hammer2_chain_unlock(parent);
1007 hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
1010 * Recurse and collect deferral data. We're in the media flush,
1011 * this can cross PFS boundaries.
1013 if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1015 hammer2_flush_core(info, child, info->flags);
1017 } else if (hammer2_debug & 0x200) {
1018 if (info->debug == NULL)
1019 info->debug = child;
1021 hammer2_flush_core(info, child, info->flags);
1023 if (info->debug == child)
1028 * Relock to continue the loop
1030 hammer2_chain_unlock(child);
1031 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1032 hammer2_chain_drop(child);
1033 KKASSERT(info->parent == parent);
1034 hammer2_spin_ex(&parent->core.spin);
1040 * flush helper (direct)
1042 * Quickly flushes any dirty chains for a device. This will update our
1043 * concept of the volume root but does NOT flush the actual volume root
1044 * and does not flush dirty device buffers.
1046 * This function is primarily used by the bulkfree code to allow it to
1047 * create a snapshot for the pass. It doesn't care about any pending
1048 * work (dirty vnodes, dirty inodes, dirty logical buffers) for which blocks
1049 * have not yet been allocated.
1052 hammer2_flush_quick(hammer2_dev_t *hmp)
1054 hammer2_chain_t *chain;
1056 hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1058 hammer2_chain_ref(&hmp->vchain);
1059 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1060 if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1061 chain = &hmp->vchain;
1062 hammer2_flush(chain, HAMMER2_FLUSH_TOP |
1064 KKASSERT(chain == &hmp->vchain);
1066 hammer2_chain_unlock(&hmp->vchain);
1067 hammer2_chain_drop(&hmp->vchain);
1069 hammer2_trans_done(hmp->spmp); /* spmp trans */
1073 * flush helper (backend threaded)
1075 * Flushes core chains, issues disk sync, flushes volume roots.
1077 * Primarily called from vfs_sync().
1080 hammer2_inode_xop_flush(hammer2_xop_t *arg, int clindex)
1082 hammer2_xop_flush_t *xop = &arg->xop_flush;
1083 hammer2_chain_t *chain;
1084 hammer2_chain_t *parent;
1087 int total_error = 0;
1093 chain = hammer2_inode_chain(xop->head.ip1, clindex,
1094 HAMMER2_RESOLVE_ALWAYS);
1097 if (chain->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1098 hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1099 parent = chain->parent;
1100 KKASSERT(chain->pmp != parent->pmp);
1101 hammer2_chain_setflush(parent);
1103 hammer2_chain_unlock(chain);
1104 hammer2_chain_drop(chain);
1111 * Flush volume roots. Avoid replication, we only want to
1112 * flush each hammer2_dev (hmp) once.
1114 for (j = clindex - 1; j >= 0; --j) {
1115 if ((chain = xop->head.ip1->cluster.array[j].chain) != NULL) {
1116 if (chain->hmp == hmp) {
1117 chain = NULL; /* safety */
1122 chain = NULL; /* safety */
1125 * spmp transaction. The super-root is never directly mounted so
1126 * there shouldn't be any vnodes, let alone any dirty vnodes
1127 * associated with it, so we shouldn't have to mess around with any
1128 * vnode flushes here.
1130 hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1133 * Media mounts have two 'roots', vchain for the topology
1134 * and fchain for the free block table. Flush both.
1136 * Note that the topology and free block table are handled
1137 * independently, so the free block table can wind up being
1138 * ahead of the topology. We depend on the bulk free scan
1139 * code to deal with any loose ends.
1141 hammer2_chain_ref(&hmp->vchain);
1142 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1143 hammer2_chain_ref(&hmp->fchain);
1144 hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1145 if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1147 * This will also modify vchain as a side effect,
1148 * mark vchain as modified now.
1150 hammer2_voldata_modify(hmp);
1151 chain = &hmp->fchain;
1152 hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1153 KKASSERT(chain == &hmp->fchain);
1155 hammer2_chain_unlock(&hmp->fchain);
1156 hammer2_chain_unlock(&hmp->vchain);
1157 hammer2_chain_drop(&hmp->fchain);
1158 /* vchain dropped down below */
1160 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1161 if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1162 chain = &hmp->vchain;
1163 hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1164 KKASSERT(chain == &hmp->vchain);
1166 hammer2_chain_unlock(&hmp->vchain);
1167 hammer2_chain_drop(&hmp->vchain);
1172 * We can't safely flush the volume header until we have
1173 * flushed any device buffers which have built up.
1175 * XXX this isn't being incremental
1177 vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
1178 error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
1179 vn_unlock(hmp->devvp);
1182 * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1183 * volume header needs synchronization via hmp->volsync.
1185 * XXX synchronize the flag & data with only this flush XXX
1188 (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1192 * Synchronize the disk before flushing the volume
1196 bp->b_bio1.bio_offset = 0;
1199 bp->b_cmd = BUF_CMD_FLUSH;
1200 bp->b_bio1.bio_done = biodone_sync;
1201 bp->b_bio1.bio_flags |= BIO_SYNC;
1202 vn_strategy(hmp->devvp, &bp->b_bio1);
1203 biowait(&bp->b_bio1, "h2vol");
1207 * Then we can safely flush the version of the
1208 * volume header synchronized by the flush code.
1210 j = hmp->volhdrno + 1;
1211 if (j >= HAMMER2_NUM_VOLHDRS)
1213 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1214 hmp->volsync.volu_size) {
1217 if (hammer2_debug & 0x8000) {
1218 /* debug only, avoid syslogd loop */
1219 kprintf("sync volhdr %d %jd\n",
1220 j, (intmax_t)hmp->volsync.volu_size);
1222 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1223 HAMMER2_PBUFSIZE, 0, 0);
1224 atomic_clear_int(&hmp->vchain.flags,
1225 HAMMER2_CHAIN_VOLUMESYNC);
1226 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1231 total_error = error;
1233 hammer2_trans_done(hmp->spmp); /* spmp trans */
1235 error = hammer2_xop_feed(&xop->head, NULL, clindex, total_error);