2 * Copyright (c) 2011-2015 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * TRANSACTION AND FLUSH HANDLING
38 * Deceptively simple but actually fairly difficult to implement properly is
39 * how I would describe it.
41 * Flushing generally occurs bottom-up but requires a top-down scan to
42 * locate chains with MODIFIED and/or UPDATE bits set. The ONFLUSH flag
43 * tells how to recurse downward to find these chains.
46 #include <sys/cdefs.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/types.h>
57 #define HAMMER2_FLUSH_DEPTH_LIMIT 10 /* stack recursion limit */
61 * Recursively flush the specified chain. The chain is locked and
62 * referenced by the caller and will remain so on return. The chain
63 * will remain referenced throughout but can temporarily lose its
64 * lock during the recursion to avoid unnecessarily stalling user
67 struct hammer2_flush_info {
68 hammer2_chain_t *parent;
73 struct h2_flush_list flushq;
74 hammer2_chain_t *debug;
77 typedef struct hammer2_flush_info hammer2_flush_info_t;
79 static void hammer2_flush_core(hammer2_flush_info_t *info,
80 hammer2_chain_t *chain, int flags);
81 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
84 * Any per-pfs transaction initialization goes here.
87 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
92 * Transaction support for any modifying operation. Transactions are used
93 * in the pmp layer by the frontend and in the spmp layer by the backend.
95 * 0 - Normal transaction, interlocked against flush
98 * TRANS_ISFLUSH - Flush transaction, interlocked against normal
101 * TRANS_BUFCACHE - Buffer cache transaction, no interlock.
103 * Initializing a new transaction allocates a transaction ID. Typically
104 * passed a pmp (hmp passed as NULL), indicating a cluster transaction. Can
105 * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
106 * media target. The latter mode is used by the recovery code.
108 * TWO TRANSACTION IDs can run concurrently, where one is a flush and the
109 * other is a set of any number of concurrent filesystem operations. We
110 * can either have <running_fs_ops> + <waiting_flush> + <blocked_fs_ops>
111 * or we can have <running_flush> + <concurrent_fs_ops>.
113 * During a flush, new fs_ops are only blocked until the fs_ops prior to
114 * the flush complete. The new fs_ops can then run concurrent with the flush.
116 * Buffer-cache transactions operate as fs_ops but never block. A
117 * buffer-cache flush will run either before or after the current pending
118 * flush depending on its state.
121 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
128 oflags = pmp->trans.flags;
132 if (flags & HAMMER2_TRANS_ISFLUSH) {
134 * Requesting flush transaction. Wait for all
135 * currently running transactions to finish.
136 * Afterwords, normal transactions will be
139 if (oflags & HAMMER2_TRANS_MASK) {
140 nflags = oflags | HAMMER2_TRANS_FPENDING |
141 HAMMER2_TRANS_WAITING;
144 nflags = (oflags | flags) + 1;
146 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
148 * Requesting strategy transaction from buffer-cache,
149 * or a VM getpages/putpages through the buffer cache.
150 * We must allow such transactions in all situations
151 * to avoid deadlocks.
153 nflags = (oflags | flags) + 1;
156 * (old) previous code interlocked against the main
159 if ((oflags & (HAMMER2_TRANS_ISFLUSH |
160 HAMMER2_TRANS_PREFLUSH)) ==
161 HAMMER2_TRANS_ISFLUSH) {
162 nflags = oflags | HAMMER2_TRANS_WAITING;
165 nflags = (oflags | flags) + 1;
170 * Requesting normal modifying transaction (read-only
171 * operations do not use transactions). Waits for
172 * any flush to finish before allowing. Multiple
173 * modifying transactions can run concurrently.
175 if (oflags & HAMMER2_TRANS_ISFLUSH) {
176 nflags = oflags | HAMMER2_TRANS_WAITING;
179 nflags = (oflags | flags) + 1;
183 tsleep_interlock(&pmp->trans.sync_wait, 0);
184 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
187 tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
197 * Start a sub-transaction, there is no 'subdone' function. This will
198 * issue a new modify_tid (mtid) for the current transaction, which is a
199 * CLC (cluster level change) id and not a per-node id.
201 * This function must be called for each XOP when multiple XOPs are run in
202 * sequence within a transaction.
204 * Callers typically update the inode with the transaction mtid manually
205 * to enforce sequencing.
208 hammer2_trans_sub(hammer2_pfs_t *pmp)
212 mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
218 hammer2_trans_done(hammer2_pfs_t *pmp)
224 oflags = pmp->trans.flags;
226 KKASSERT(oflags & HAMMER2_TRANS_MASK);
227 if ((oflags & HAMMER2_TRANS_MASK) == 1) {
229 * This was the last transaction
231 nflags = (oflags - 1) & ~(HAMMER2_TRANS_ISFLUSH |
232 HAMMER2_TRANS_BUFCACHE |
233 HAMMER2_TRANS_FPENDING |
234 HAMMER2_TRANS_WAITING);
237 * Still transactions pending
241 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
242 if ((nflags & HAMMER2_TRANS_MASK) == 0 &&
243 (oflags & HAMMER2_TRANS_WAITING)) {
244 wakeup(&pmp->trans.sync_wait);
255 * Obtain new, unique inode number (not serialized by caller).
258 hammer2_trans_newinum(hammer2_pfs_t *pmp)
262 tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
268 * Assert that a strategy call is ok here. Currently we allow strategy
269 * calls in all situations, including during flushes. Previously:
270 * (old) (1) In a normal transaction.
271 * (old) (2) In a flush transaction only if PREFLUSH is also set.
274 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
277 KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 ||
278 (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH));
284 * Chains undergoing destruction are removed from the in-memory topology.
285 * To avoid getting lost these chains are placed on the delayed flush
286 * queue which will properly dispose of them.
288 * We do this instead of issuing an immediate flush in order to give
289 * recursive deletions (rm -rf, etc) a chance to remove more of the
290 * hierarchy, potentially allowing an enormous amount of write I/O to
294 hammer2_delayed_flush(hammer2_chain_t *chain)
296 if ((chain->flags & HAMMER2_CHAIN_DELAYED) == 0) {
297 hammer2_spin_ex(&chain->hmp->list_spin);
298 if ((chain->flags & (HAMMER2_CHAIN_DELAYED |
299 HAMMER2_CHAIN_DEFERRED)) == 0) {
300 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELAYED |
301 HAMMER2_CHAIN_DEFERRED);
302 TAILQ_INSERT_TAIL(&chain->hmp->flushq,
304 hammer2_chain_ref(chain);
306 hammer2_spin_unex(&chain->hmp->list_spin);
307 hammer2_voldata_modify(chain->hmp);
312 * Flush the chain and all modified sub-chains through the specified
313 * synchronization point, propagating blockref updates back up. As
314 * part of this propagation, mirror_tid and inode/data usage statistics
315 * propagates back upward.
317 * modify_tid (clc - cluster level change) is not propagated.
319 * update_tid (clc) is used for validation and is not propagated by this
322 * This routine can be called from several places but the most important
323 * is from VFS_SYNC (frontend) via hammer2_inode_xop_flush (backend).
325 * chain is locked on call and will remain locked on return. The chain's
326 * UPDATE flag indicates that its parent's block table (which is not yet
327 * part of the flush) should be updated.
330 hammer2_flush(hammer2_chain_t *chain, int flags)
332 hammer2_chain_t *scan;
333 hammer2_flush_info_t info;
338 * Execute the recursive flush and handle deferrals.
340 * Chains can be ridiculously long (thousands deep), so to
341 * avoid blowing out the kernel stack the recursive flush has a
342 * depth limit. Elements at the limit are placed on a list
343 * for re-execution after the stack has been popped.
345 bzero(&info, sizeof(info));
346 TAILQ_INIT(&info.flushq);
347 info.cache_index = -1;
348 info.flags = flags & ~HAMMER2_FLUSH_TOP;
351 * Calculate parent (can be NULL), if not NULL the flush core
352 * expects the parent to be referenced so it can easily lock/unlock
353 * it without it getting ripped up.
355 if ((info.parent = chain->parent) != NULL)
356 hammer2_chain_ref(info.parent);
359 * Extra ref needed because flush_core expects it when replacing
362 hammer2_chain_ref(chain);
368 * Move hmp->flushq to info.flushq if non-empty so it can
371 if (TAILQ_FIRST(&hmp->flushq) != NULL) {
372 hammer2_spin_ex(&chain->hmp->list_spin);
373 TAILQ_CONCAT(&info.flushq, &hmp->flushq, flush_node);
374 hammer2_spin_unex(&chain->hmp->list_spin);
378 * Unwind deep recursions which had been deferred. This
379 * can leave the FLUSH_* bits set for these chains, which
380 * will be handled when we [re]flush chain after the unwind.
382 while ((scan = TAILQ_FIRST(&info.flushq)) != NULL) {
383 KKASSERT(scan->flags & HAMMER2_CHAIN_DEFERRED);
384 TAILQ_REMOVE(&info.flushq, scan, flush_node);
385 atomic_clear_int(&scan->flags, HAMMER2_CHAIN_DEFERRED |
386 HAMMER2_CHAIN_DELAYED);
389 * Now that we've popped back up we can do a secondary
390 * recursion on the deferred elements.
392 * NOTE: hammer2_flush() may replace scan.
394 if (hammer2_debug & 0x0040)
395 kprintf("deferred flush %p\n", scan);
396 hammer2_chain_lock(scan, HAMMER2_RESOLVE_MAYBE);
397 hammer2_flush(scan, flags & ~HAMMER2_FLUSH_TOP);
398 hammer2_chain_unlock(scan);
399 hammer2_chain_drop(scan); /* ref from deferral */
405 info.diddeferral = 0;
406 hammer2_flush_core(&info, chain, flags);
409 * Only loop if deep recursions have been deferred.
411 if (TAILQ_EMPTY(&info.flushq))
414 if (++loops % 1000 == 0) {
415 kprintf("hammer2_flush: excessive loops on %p\n",
417 if (hammer2_debug & 0x100000)
421 hammer2_chain_drop(chain);
423 hammer2_chain_drop(info.parent);
427 * This is the core of the chain flushing code. The chain is locked by the
428 * caller and must also have an extra ref on it by the caller, and remains
429 * locked and will have an extra ref on return. info.parent is referenced
432 * Upon return, the caller can test the UPDATE bit on the chain to determine
433 * if the parent needs updating.
435 * (1) Determine if this node is a candidate for the flush, return if it is
436 * not. fchain and vchain are always candidates for the flush.
438 * (2) If we recurse too deep the chain is entered onto the deferral list and
439 * the current flush stack is aborted until after the deferral list is
442 * (3) Recursively flush live children (rbtree). This can create deferrals.
443 * A successful flush clears the MODIFIED and UPDATE bits on the children
444 * and typically causes the parent to be marked MODIFIED as the children
445 * update the parent's block table. A parent might already be marked
446 * MODIFIED due to a deletion (whos blocktable update in the parent is
447 * handled by the frontend), or if the parent itself is modified by the
448 * frontend for other reasons.
450 * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
451 * Deleted-but-open inodes can still be individually flushed via the
454 * (5) Delete parents on the way back up if they are normal indirect blocks
455 * and have no children.
457 * (6) Note that an unmodified child may still need the block table in its
458 * parent updated (e.g. rename/move). The child will have UPDATE set
461 * WARNING ON BREF MODIFY_TID/MIRROR_TID
463 * blockref.modify_tid is consistent only within a PFS, and will not be
464 * consistent during synchronization. mirror_tid is consistent across the
465 * block device regardless of the PFS.
468 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
471 hammer2_chain_t *parent;
476 * (1) Optimize downward recursion to locate nodes needing action.
477 * Nothing to do if none of these flags are set.
479 if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) {
480 if (hammer2_debug & 0x200) {
481 if (info->debug == NULL)
489 diddeferral = info->diddeferral;
490 parent = info->parent; /* can be NULL */
491 KKASSERT(chain->parent == parent);
494 * Downward search recursion
496 if (chain->flags & (HAMMER2_CHAIN_DEFERRED | HAMMER2_CHAIN_DELAYED)) {
501 } else if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) &&
502 (flags & HAMMER2_FLUSH_ALL) == 0 &&
503 (flags & HAMMER2_FLUSH_TOP) == 0) {
505 * If FLUSH_ALL is not specified the caller does not want
506 * to recurse through PFS roots. The typical sequence is
507 * to flush dirty PFS's starting at their root downward,
508 * then flush the device root (vchain). It is this second
509 * flush that typically leaves out the ALL flag.
511 * However we must still process the PFSROOT chains for block
512 * table updates in their parent (which IS part of our flush).
514 * NOTE: The volume root, vchain, does not set PFSBOUNDARY.
516 * NOTE: This test must be done before the depth-limit test,
517 * else it might become the top on a flushq iteration.
519 * NOTE: We must re-set ONFLUSH in the parent to retain if
520 * this chain (that we are skipping) requires work.
522 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
523 HAMMER2_CHAIN_DESTROY |
524 HAMMER2_CHAIN_MODIFIED)) {
525 hammer2_chain_setflush(parent);
527 } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
529 * Recursion depth reached.
531 KKASSERT((chain->flags & HAMMER2_CHAIN_DELAYED) == 0);
532 hammer2_chain_ref(chain);
533 TAILQ_INSERT_TAIL(&info->flushq, chain, flush_node);
534 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEFERRED);
536 } else if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
537 HAMMER2_CHAIN_DESTROY)) {
539 * Downward recursion search (actual flush occurs bottom-up).
540 * pre-clear ONFLUSH. It can get set again due to races,
541 * which we want so the scan finds us again in the next flush.
543 * We must also recurse if DESTROY is set so we can finally
544 * get rid of the related children, otherwise the node will
545 * just get re-flushed on lastdrop.
547 * WARNING! The recursion will unlock/relock info->parent
548 * (which is 'chain'), potentially allowing it
551 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
552 info->parent = chain;
553 hammer2_spin_ex(&chain->core.spin);
554 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
555 NULL, hammer2_flush_recurse, info);
556 hammer2_spin_unex(&chain->core.spin);
557 info->parent = parent;
558 if (info->diddeferral)
559 hammer2_chain_setflush(chain);
562 * If we lost the parent->chain association we have to
563 * stop processing this chain because it is no longer
564 * in this recursion. If it moved, it will be handled
565 * by the ONFLUSH flag elsewhere.
567 if (chain->parent != parent) {
568 kprintf("LOST CHILD2 %p->%p (actual parent %p)\n",
569 parent, chain, chain->parent);
575 * Now we are in the bottom-up part of the recursion.
577 * Do not update chain if lower layers were deferred.
579 if (info->diddeferral)
583 * Both parent and chain must be locked in order to flush chain,
584 * in order to properly update the parent under certain conditions.
586 * In addition, we can't safely unlock/relock the chain once we
587 * start flushing the chain itself, which we would have to do later
588 * on in order to lock the parent if we didn't do that now.
590 hammer2_chain_unlock(chain);
592 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
593 hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
594 if (chain->parent != parent) {
595 kprintf("LOST CHILD3 %p->%p (actual parent %p)\n",
596 parent, chain, chain->parent);
597 KKASSERT(parent != NULL);
598 hammer2_chain_unlock(parent);
599 if ((chain->flags & HAMMER2_CHAIN_DELAYED) == 0) {
600 hammer2_chain_ref(chain);
601 TAILQ_INSERT_TAIL(&info->flushq, chain, flush_node);
602 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEFERRED);
609 * Propagate the DESTROY flag downwards. This dummies up the flush
610 * code and tries to invalidate related buffer cache buffers to
611 * avoid the disk write.
613 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
614 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
617 * Chain was already modified or has become modified, flush it out.
619 if ((hammer2_debug & 0x200) &&
621 (chain->flags & (HAMMER2_CHAIN_MODIFIED | HAMMER2_CHAIN_UPDATE))) {
622 hammer2_chain_t *scan = chain;
624 kprintf("DISCONNECTED FLUSH %p->%p\n", info->debug, chain);
626 kprintf(" chain %p [%08x] bref=%016jx:%02x\n",
628 scan->bref.key, scan->bref.type);
629 if (scan == info->debug)
635 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
637 * Dispose of the modified bit.
639 * If parent is present, the UPDATE bit should already be set.
640 * UPDATE should already be set.
641 * bref.mirror_tid should already be set.
643 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
644 chain->parent == NULL);
645 if (hammer2_debug & 0x800000) {
648 for (pp = chain; pp->parent; pp = pp->parent)
650 kprintf("FLUSH CHAIN %p (p=%p pp=%p/%d) TYPE %d FLAGS %08x (%s)\n",
651 chain, chain->parent, pp, pp->bref.type,
652 chain->bref.type, chain->flags,
653 (chain->bref.type == 1 ? (const char *)chain->data->ipdata.filename : "?")
658 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
659 atomic_add_long(&hammer2_count_modified_chains, -1);
662 * Manage threads waiting for excessive dirty memory to
666 hammer2_pfs_memory_wakeup(chain->pmp);
669 if ((chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
670 chain != &hmp->vchain &&
671 chain != &hmp->fchain) {
673 * Set UPDATE bit indicating that the parent block
674 * table requires updating.
676 atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
681 * Issue the flush. This is indirect via the DIO.
683 * NOTE: A DELETED node that reaches this point must be
684 * flushed for synchronization point consistency.
686 * NOTE: Even though MODIFIED was already set, the related DIO
687 * might not be dirty due to a system buffer cache
688 * flush and must be set dirty if we are going to make
689 * further modifications to the buffer. Chains with
690 * embedded data don't need this.
692 if (hammer2_debug & 0x1000) {
693 kprintf("Flush %p.%d %016jx/%d data=%016jx\n",
694 chain, chain->bref.type,
695 (uintmax_t)chain->bref.key,
697 (uintmax_t)chain->bref.data_off);
699 if (hammer2_debug & 0x2000) {
700 Debugger("Flush hell");
704 * Update chain CRCs for flush.
706 * NOTE: Volume headers are NOT flushed here as they require
707 * special processing.
709 switch(chain->bref.type) {
710 case HAMMER2_BREF_TYPE_FREEMAP:
712 * Update the volume header's freemap_tid to the
713 * freemap's flushing mirror_tid.
715 * (note: embedded data, do not call setdirty)
717 KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
718 KKASSERT(chain == &hmp->fchain);
719 hmp->voldata.freemap_tid = chain->bref.mirror_tid;
720 if (hammer2_debug & 0x8000) {
721 /* debug only, avoid syslogd loop */
722 kprintf("sync freemap mirror_tid %08jx\n",
723 (intmax_t)chain->bref.mirror_tid);
727 * The freemap can be flushed independently of the
728 * main topology, but for the case where it is
729 * flushed in the same transaction, and flushed
730 * before vchain (a case we want to allow for
731 * performance reasons), make sure modifications
732 * made during the flush under vchain use a new
735 * Otherwise the mount recovery code will get confused.
737 ++hmp->voldata.mirror_tid;
739 case HAMMER2_BREF_TYPE_VOLUME:
741 * The free block table is flushed by
742 * hammer2_vfs_sync() before it flushes vchain.
743 * We must still hold fchain locked while copying
744 * voldata to volsync, however.
746 * (note: embedded data, do not call setdirty)
748 hammer2_chain_lock(&hmp->fchain,
749 HAMMER2_RESOLVE_ALWAYS);
750 hammer2_voldata_lock(hmp);
751 if (hammer2_debug & 0x8000) {
752 /* debug only, avoid syslogd loop */
753 kprintf("sync volume mirror_tid %08jx\n",
754 (intmax_t)chain->bref.mirror_tid);
758 * Update the volume header's mirror_tid to the
759 * main topology's flushing mirror_tid. It is
760 * possible that voldata.mirror_tid is already
761 * beyond bref.mirror_tid due to the bump we made
762 * above in BREF_TYPE_FREEMAP.
764 if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
765 hmp->voldata.mirror_tid =
766 chain->bref.mirror_tid;
770 * The volume header is flushed manually by the
771 * syncer, not here. All we do here is adjust the
774 KKASSERT(chain->data != NULL);
775 KKASSERT(chain->dio == NULL);
777 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
779 (char *)&hmp->voldata +
780 HAMMER2_VOLUME_ICRC1_OFF,
781 HAMMER2_VOLUME_ICRC1_SIZE);
782 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
784 (char *)&hmp->voldata +
785 HAMMER2_VOLUME_ICRC0_OFF,
786 HAMMER2_VOLUME_ICRC0_SIZE);
787 hmp->voldata.icrc_volheader =
789 (char *)&hmp->voldata +
790 HAMMER2_VOLUME_ICRCVH_OFF,
791 HAMMER2_VOLUME_ICRCVH_SIZE);
793 if (hammer2_debug & 0x8000) {
794 /* debug only, avoid syslogd loop */
795 kprintf("syncvolhdr %016jx %016jx\n",
796 hmp->voldata.mirror_tid,
797 hmp->vchain.bref.mirror_tid);
799 hmp->volsync = hmp->voldata;
800 atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
801 hammer2_voldata_unlock(hmp);
802 hammer2_chain_unlock(&hmp->fchain);
804 case HAMMER2_BREF_TYPE_DATA:
806 * Data elements have already been flushed via the
807 * logical file buffer cache. Their hash was set in
808 * the bref by the vop_write code. Do not re-dirty.
810 * Make sure any device buffer(s) have been flushed
811 * out here (there aren't usually any to flush) XXX.
814 case HAMMER2_BREF_TYPE_INDIRECT:
815 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
816 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
818 * Buffer I/O will be cleaned up when the volume is
819 * flushed (but the kernel is free to flush it before
822 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
823 hammer2_chain_setcheck(chain, chain->data);
825 case HAMMER2_BREF_TYPE_DIRENT:
827 * A directory entry can use the check area to store
828 * the filename for filenames <= 64 bytes, don't blow
831 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
833 hammer2_chain_setcheck(chain, chain->data);
835 case HAMMER2_BREF_TYPE_INODE:
837 * NOTE: We must call io_setdirty() to make any late
838 * changes to the inode data, the system might
839 * have already flushed the buffer.
841 if (chain->data->ipdata.meta.op_flags &
842 HAMMER2_OPFLAG_PFSROOT) {
844 * non-NULL pmp if mounted as a PFS. We must
845 * sync fields cached in the pmp? XXX
847 hammer2_inode_data_t *ipdata;
849 hammer2_io_setdirty(chain->dio);
850 ipdata = &chain->data->ipdata;
852 ipdata->meta.pfs_inum =
853 chain->pmp->inode_tid;
856 /* can't be mounted as a PFS */
859 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
860 hammer2_chain_setcheck(chain, chain->data);
863 KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
864 panic("hammer2_flush_core: unsupported "
871 * If the chain was destroyed try to avoid unnecessary I/O.
872 * The DIO system buffer may silently disallow the
875 if (chain->flags & HAMMER2_CHAIN_DESTROY) {
879 hammer2_io_setinval(chain->dio,
880 chain->bref.data_off,
882 } else if ((dio = hammer2_io_getquick(hmp,
883 chain->bref.data_off,
884 chain->bytes)) != NULL) {
885 hammer2_io_setinval(dio,
886 chain->bref.data_off,
888 hammer2_io_putblk(&dio);
894 * If UPDATE is set the parent block table may need to be updated.
896 * NOTE: UPDATE may be set on vchain or fchain in which case
897 * parent could be NULL. It's easiest to allow the case
898 * and test for NULL. parent can also wind up being NULL
899 * due to a deletion so we need to handle the case anyway.
901 * If no parent exists we can just clear the UPDATE bit. If the
902 * chain gets reattached later on the bit will simply get set
905 if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL)
906 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
909 * The chain may need its blockrefs updated in the parent.
911 if (chain->flags & HAMMER2_CHAIN_UPDATE) {
912 hammer2_blockref_t *base;
916 * Clear UPDATE flag, mark parent modified, update its
917 * modify_tid if necessary, and adjust the parent blockmap.
919 if (chain->flags & HAMMER2_CHAIN_UPDATE)
920 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
925 * Avoid actually modifying and updating the parent if it
926 * was flagged for destruction. This can greatly reduce
927 * disk I/O in large tree removals because the
928 * hammer2_io_setinval() call in the upward recursion
929 * (see MODIFIED code above) can only handle a few cases.
931 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
932 if (parent->bref.modify_tid < chain->bref.modify_tid) {
933 parent->bref.modify_tid =
934 chain->bref.modify_tid;
936 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED |
937 HAMMER2_CHAIN_BMAPUPD);
942 * (semi-optional code)
944 * The flusher is responsible for deleting empty indirect
945 * blocks at this point. If we don't do this, no major harm
946 * will be done but the empty indirect blocks will stay in
947 * the topology and make it a bit messy.
949 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT &&
950 chain->core.live_count == 0 &&
951 (chain->flags & (HAMMER2_CHAIN_INITIAL |
952 HAMMER2_CHAIN_COUNTEDBREFS)) == 0) {
953 base = &chain->data->npdata[0];
954 count = chain->bytes / sizeof(hammer2_blockref_t);
955 hammer2_chain_countbrefs(chain, base, count);
957 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT &&
958 chain->core.live_count == 0) {
960 kprintf("DELETE CHAIN %016jx.%02x %016jx/%d refs=%d\n",
961 chain->bref.data_off, chain->bref.type,
962 chain->bref.key, chain->bref.keybits,
965 hammer2_chain_delete(parent, chain,
966 chain->bref.modify_tid,
967 HAMMER2_DELETE_PERMANENT);
972 * We are updating the parent's blockmap, the parent must
975 hammer2_chain_modify(parent, 0, 0, 0);
976 if (parent->bref.modify_tid < chain->bref.modify_tid)
977 parent->bref.modify_tid = chain->bref.modify_tid;
980 * Calculate blockmap pointer
982 switch(parent->bref.type) {
983 case HAMMER2_BREF_TYPE_INODE:
985 * Access the inode's block array. However, there is
986 * no block array if the inode is flagged DIRECTDATA.
989 (parent->data->ipdata.meta.op_flags &
990 HAMMER2_OPFLAG_DIRECTDATA) == 0) {
991 base = &parent->data->
992 ipdata.u.blockset.blockref[0];
996 count = HAMMER2_SET_COUNT;
998 case HAMMER2_BREF_TYPE_INDIRECT:
999 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1001 base = &parent->data->npdata[0];
1004 count = parent->bytes / sizeof(hammer2_blockref_t);
1006 case HAMMER2_BREF_TYPE_VOLUME:
1007 base = &chain->hmp->voldata.sroot_blockset.blockref[0];
1008 count = HAMMER2_SET_COUNT;
1010 case HAMMER2_BREF_TYPE_FREEMAP:
1011 base = &parent->data->npdata[0];
1012 count = HAMMER2_SET_COUNT;
1017 panic("hammer2_flush_core: "
1018 "unrecognized blockref type: %d",
1023 * Blocktable updates
1025 * We synchronize pending statistics at this time. Delta
1026 * adjustments designated for the current and upper level
1029 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) {
1030 if (chain->flags & HAMMER2_CHAIN_BMAPPED) {
1031 hammer2_spin_ex(&parent->core.spin);
1032 hammer2_base_delete(parent, base, count,
1033 &info->cache_index, chain);
1034 hammer2_spin_unex(&parent->core.spin);
1035 /* base_delete clears both bits */
1037 atomic_clear_int(&chain->flags,
1038 HAMMER2_CHAIN_BMAPUPD);
1041 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) {
1042 hammer2_spin_ex(&parent->core.spin);
1043 hammer2_base_insert(parent, base, count,
1044 &info->cache_index, chain);
1045 hammer2_spin_unex(&parent->core.spin);
1046 /* base_insert sets BMAPPED */
1051 hammer2_chain_unlock(parent);
1054 * Final cleanup after flush
1057 KKASSERT(chain->refs > 0);
1058 if (hammer2_debug & 0x200) {
1059 if (info->debug == chain)
1065 * Flush recursion helper, called from flush_core, calls flush_core.
1067 * Flushes the children of the caller's chain (info->parent), restricted
1068 * by sync_tid. Set info->domodify if the child's blockref must propagate
1069 * back up to the parent.
1071 * Ripouts can move child from rbtree to dbtree or dbq but the caller's
1072 * flush scan order prevents any chains from being lost. A child can be
1073 * executes more than once.
1075 * WARNING! If we do not call hammer2_flush_core() we must update
1076 * bref.mirror_tid ourselves to indicate that the flush has
1077 * processed the child.
1079 * WARNING! parent->core spinlock is held on entry and return.
1082 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
1084 hammer2_flush_info_t *info = data;
1085 hammer2_chain_t *parent = info->parent;
1088 * (child can never be fchain or vchain so a special check isn't
1091 * We must ref the child before unlocking the spinlock.
1093 * The caller has added a ref to the parent so we can temporarily
1094 * unlock it in order to lock the child. However, if it no longer
1095 * winds up being the child of the parent we must skip this child.
1097 hammer2_chain_ref(child);
1098 hammer2_spin_unex(&parent->core.spin);
1100 hammer2_chain_unlock(parent);
1101 hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
1102 if (child->parent != parent) {
1103 kprintf("LOST CHILD1 %p->%p (actual parent %p)\n",
1104 parent, child, child->parent);
1109 * Must propagate the DESTROY flag downwards, otherwise the
1110 * parent could end up never being removed because it will
1111 * be requeued to the flusher if it survives this run due to
1114 if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
1115 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROY);
1118 * Recurse and collect deferral data. We're in the media flush,
1119 * this can cross PFS boundaries.
1121 if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1123 hammer2_flush_core(info, child, info->flags);
1125 } else if (hammer2_debug & 0x200) {
1126 if (info->debug == NULL)
1127 info->debug = child;
1129 hammer2_flush_core(info, child, info->flags);
1131 if (info->debug == child)
1137 * Relock to continue the loop
1139 hammer2_chain_unlock(child);
1140 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1141 hammer2_chain_drop(child);
1142 KKASSERT(info->parent == parent);
1143 hammer2_spin_ex(&parent->core.spin);
1150 * Flush helper (direct)
1152 * Quickly flushes any dirty chains for a device and returns a temporary
1153 * out-of-band copy of hmp->vchain that the caller can use as a stable
1156 * This function does not flush the actual volume root and does not flush dirty
1157 * device buffers. We don't care about pending work, per-say. This function
1158 * is primarily used by the bulkfree code to create a stable snapshot of
1162 hammer2_flush_quick(hammer2_dev_t *hmp)
1164 hammer2_chain_t *chain;
1165 hammer2_chain_t *copy;
1167 hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1169 hammer2_chain_ref(&hmp->vchain);
1170 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1171 if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1172 chain = &hmp->vchain;
1173 hammer2_flush(chain, HAMMER2_FLUSH_TOP |
1175 KKASSERT(chain == &hmp->vchain);
1177 copy = hammer2_chain_bulksnap(&hmp->vchain);
1178 hammer2_chain_unlock(&hmp->vchain);
1179 hammer2_chain_drop(&hmp->vchain);
1181 hammer2_trans_done(hmp->spmp); /* spmp trans */
1188 * flush helper (backend threaded)
1190 * Flushes core chains, issues disk sync, flushes volume roots.
1192 * Primarily called from vfs_sync().
1195 hammer2_inode_xop_flush(hammer2_thread_t *thr, hammer2_xop_t *arg)
1197 hammer2_xop_flush_t *xop = &arg->xop_flush;
1198 hammer2_chain_t *chain;
1199 hammer2_chain_t *parent;
1202 int total_error = 0;
1208 chain = hammer2_inode_chain(xop->head.ip1, thr->clindex,
1209 HAMMER2_RESOLVE_ALWAYS);
1212 if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) ||
1213 TAILQ_FIRST(&hmp->flushq) != NULL) {
1214 hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1215 parent = chain->parent;
1216 KKASSERT(chain->pmp != parent->pmp);
1217 hammer2_chain_setflush(parent);
1219 hammer2_chain_unlock(chain);
1220 hammer2_chain_drop(chain);
1227 * Flush volume roots. Avoid replication, we only want to
1228 * flush each hammer2_dev (hmp) once.
1230 for (j = thr->clindex - 1; j >= 0; --j) {
1231 if ((chain = xop->head.ip1->cluster.array[j].chain) != NULL) {
1232 if (chain->hmp == hmp) {
1233 chain = NULL; /* safety */
1238 chain = NULL; /* safety */
1241 * spmp transaction. The super-root is never directly mounted so
1242 * there shouldn't be any vnodes, let alone any dirty vnodes
1243 * associated with it, so we shouldn't have to mess around with any
1244 * vnode flushes here.
1246 hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1249 * Media mounts have two 'roots', vchain for the topology
1250 * and fchain for the free block table. Flush both.
1252 * Note that the topology and free block table are handled
1253 * independently, so the free block table can wind up being
1254 * ahead of the topology. We depend on the bulk free scan
1255 * code to deal with any loose ends.
1257 hammer2_chain_ref(&hmp->vchain);
1258 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1259 hammer2_chain_ref(&hmp->fchain);
1260 hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1261 if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1263 * This will also modify vchain as a side effect,
1264 * mark vchain as modified now.
1266 hammer2_voldata_modify(hmp);
1267 chain = &hmp->fchain;
1268 hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1269 KKASSERT(chain == &hmp->fchain);
1271 hammer2_chain_unlock(&hmp->fchain);
1272 hammer2_chain_unlock(&hmp->vchain);
1273 hammer2_chain_drop(&hmp->fchain);
1274 /* vchain dropped down below */
1276 hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1277 if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1278 chain = &hmp->vchain;
1279 hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1280 KKASSERT(chain == &hmp->vchain);
1282 hammer2_chain_unlock(&hmp->vchain);
1283 hammer2_chain_drop(&hmp->vchain);
1288 * We can't safely flush the volume header until we have
1289 * flushed any device buffers which have built up.
1291 * XXX this isn't being incremental
1293 vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
1294 error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
1295 vn_unlock(hmp->devvp);
1298 * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1299 * volume header needs synchronization via hmp->volsync.
1301 * XXX synchronize the flag & data with only this flush XXX
1304 (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1308 * Synchronize the disk before flushing the volume
1312 bp->b_bio1.bio_offset = 0;
1315 bp->b_cmd = BUF_CMD_FLUSH;
1316 bp->b_bio1.bio_done = biodone_sync;
1317 bp->b_bio1.bio_flags |= BIO_SYNC;
1318 vn_strategy(hmp->devvp, &bp->b_bio1);
1319 biowait(&bp->b_bio1, "h2vol");
1323 * Then we can safely flush the version of the
1324 * volume header synchronized by the flush code.
1326 j = hmp->volhdrno + 1;
1327 if (j >= HAMMER2_NUM_VOLHDRS)
1329 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1330 hmp->volsync.volu_size) {
1333 if (hammer2_debug & 0x8000) {
1334 /* debug only, avoid syslogd loop */
1335 kprintf("sync volhdr %d %jd\n",
1336 j, (intmax_t)hmp->volsync.volu_size);
1338 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1339 HAMMER2_PBUFSIZE, 0, 0);
1340 atomic_clear_int(&hmp->vchain.flags,
1341 HAMMER2_CHAIN_VOLUMESYNC);
1342 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1347 total_error = error;
1349 hammer2_trans_done(hmp->spmp); /* spmp trans */
1351 error = hammer2_xop_feed(&xop->head, NULL, thr->clindex, total_error);