2 * Copyright (c) 2011-2013 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * 3. Neither the name of The DragonFly Project nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific, prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #include <sys/cdefs.h>
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/types.h>
46 * Recursively flush the specified chain. The chain is locked and
47 * referenced by the caller and will remain so on return. The chain
48 * will remain referenced throughout but can temporarily lose its
49 * lock during the recursion to avoid unnecessarily stalling user
52 struct hammer2_flush_info {
53 hammer2_chain_t *parent;
54 hammer2_trans_t *trans;
59 struct h2_flush_deferral_list flush_list;
60 hammer2_tid_t sync_tid; /* flush synchronization point */
61 hammer2_tid_t mirror_tid; /* collect mirror TID updates */
64 typedef struct hammer2_flush_info hammer2_flush_info_t;
66 static void hammer2_chain_flush_core(hammer2_flush_info_t *info,
67 hammer2_chain_t *chain);
68 static int hammer2_chain_flush_scan1(hammer2_chain_t *child, void *data);
69 static int hammer2_chain_flush_scan2(hammer2_chain_t *child, void *data);
70 static void hammer2_rollup_stats(hammer2_chain_t *parent,
71 hammer2_chain_t *child, int how);
76 hammer2_updatestats(hammer2_flush_info_t *info, hammer2_blockref_t *bref,
81 if (bref->type != 0) {
82 bytes = 1 << (bref->data_off & HAMMER2_OFF_MASK_RADIX);
83 if (bref->type == HAMMER2_BREF_TYPE_INODE)
84 info->inode_count += how;
86 info->data_count -= bytes;
88 info->data_count += bytes;
94 * Transaction support functions for writing to the filesystem.
96 * Initializing a new transaction allocates a transaction ID. We
97 * don't bother marking the volume header MODIFIED. Instead, the volume
98 * will be synchronized at a later time as part of a larger flush sequence.
100 * Non-flush transactions can typically run concurrently. However if
101 * there are non-flush transaction both before AND after a flush trans,
102 * the transactions after stall until the ones before finish.
104 * Non-flush transactions occuring after a flush pointer can run concurrently
105 * with that flush. They only have to wait for transactions prior to the
106 * flush trans to complete before they unstall.
108 * WARNING! Transaction ids are only allocated when the transaction becomes
109 * active, which allows other transactions to insert ahead of us
110 * if we are forced to block (only bioq transactions do that).
112 * WARNING! Modifications to the root volume cannot dup the root volume
113 * header to handle synchronization points, so alloc_tid can
114 * wind up (harmlessly) more advanced on flush.
116 * WARNING! Operations which might call inode_duplicate()/chain_duplicate()
117 * depend heavily on having a unique sync_tid to avoid duplication
118 * collisions (which key off of delete_tid).
121 hammer2_trans_init(hammer2_trans_t *trans, hammer2_pfsmount_t *pmp, int flags)
123 hammer2_mount_t *hmp;
124 hammer2_trans_t *scan;
126 bzero(trans, sizeof(*trans));
128 hmp = pmp->cluster.chains[0]->hmp; /* XXX */
130 hammer2_voldata_lock(hmp);
131 trans->flags = flags;
132 trans->td = curthread;
133 /*trans->delete_gen = 0;*/ /* multiple deletions within trans */
135 if (flags & HAMMER2_TRANS_ISFLUSH) {
137 * If multiple flushes are trying to run we have to
138 * wait until it is our turn, then set curflush to
139 * indicate that a flush is now pending (but not
140 * necessarily active yet).
142 * NOTE: Do not set trans->blocked here.
145 while (hmp->curflush != NULL) {
146 lksleep(&hmp->curflush, &hmp->voldatalk,
149 hmp->curflush = trans;
150 TAILQ_INSERT_TAIL(&hmp->transq, trans, entry);
153 * If we are a flush we have to wait for all transactions
154 * prior to our flush synchronization point to complete
155 * before we can start our flush.
157 * Most importantly, this includes bioq flushes.
159 * NOTE: Do not set trans->blocked here.
161 while (TAILQ_FIRST(&hmp->transq) != trans) {
162 lksleep(&trans->sync_tid, &hmp->voldatalk,
167 * don't assign sync_tid until we become the running
168 * flush. topo_flush_tid is used to control when
169 * chain modifications in concurrent transactions are
170 * required to delete-duplicate (so as not to disturb
171 * the state of what is being currently flushed).
173 trans->sync_tid = hmp->voldata.alloc_tid++;
174 hmp->topo_flush_tid = trans->sync_tid;
177 * Once we become the running flush we can wakeup anyone
178 * who blocked on us, up to the next flush. That is,
179 * our flush can run concurrent with frontend operations.
182 while ((scan = TAILQ_NEXT(scan, entry)) != NULL) {
183 if (scan->flags & HAMMER2_TRANS_ISFLUSH)
185 if (scan->blocked == 0)
188 wakeup(&scan->blocked);
190 } else if ((flags & HAMMER2_TRANS_BUFCACHE) && hmp->curflush) {
192 * We cannot block if we are the bioq thread. When a
193 * flush is not pending we can operate normally but
194 * if a flush IS pending the bioq thread's transaction
195 * must be placed either before or after curflush.
197 * If the current flush is waiting the bioq thread's
198 * transaction is placed before. If it is running the
199 * bioq thread's transaction is placed after.
201 scan = TAILQ_FIRST(&hmp->transq);
202 if (scan != hmp->curflush) {
203 TAILQ_INSERT_BEFORE(hmp->curflush, trans, entry);
205 TAILQ_INSERT_TAIL(&hmp->transq, trans, entry);
207 trans->sync_tid = hmp->voldata.alloc_tid++;
210 * If this is a normal transaction and not a flush, or
211 * if this is a bioq transaction and no flush is pending,
212 * we can queue normally.
214 * Normal transactions must block while a pending flush is
215 * waiting for prior transactions to complete. Once the
216 * pending flush becomes active we can run concurrently
219 TAILQ_INSERT_TAIL(&hmp->transq, trans, entry);
220 scan = TAILQ_FIRST(&hmp->transq);
221 if (hmp->curflush && hmp->curflush != scan) {
223 while (trans->blocked) {
224 lksleep(&trans->blocked, &hmp->voldatalk,
228 trans->sync_tid = hmp->voldata.alloc_tid++;
230 hammer2_voldata_unlock(hmp, 0);
234 hammer2_trans_done(hammer2_trans_t *trans)
236 hammer2_mount_t *hmp;
237 hammer2_trans_t *scan;
239 hmp = trans->pmp->cluster.chains[0]->hmp;
241 hammer2_voldata_lock(hmp);
242 TAILQ_REMOVE(&hmp->transq, trans, entry);
243 if (trans->flags & HAMMER2_TRANS_ISFLUSH) {
247 * If we were a flush then wakeup anyone waiting on
248 * curflush (i.e. other flushes that want to run).
249 * Leave topo_flush_id set (I think we could probably
250 * clear it to zero here).
252 hmp->curflush = NULL;
253 wakeup(&hmp->curflush);
256 * Theoretically we don't have to clear flush_tid
257 * here since the flush will have synchronized
258 * all operations <= flush_tid already. But for
261 hmp->curflush = NULL;
262 hmp->topo_flush_tid = 0;
266 * If we are not a flush but a flush is now at the head
267 * of the queue and we were previously blocking it,
268 * we can now unblock it.
271 (scan = TAILQ_FIRST(&hmp->transq)) != NULL &&
272 trans->sync_tid < scan->sync_tid &&
273 (scan->flags & HAMMER2_TRANS_ISFLUSH)) {
274 wakeup(&scan->sync_tid);
277 hammer2_voldata_unlock(hmp, 0);
281 * Flush the chain and all modified sub-chains through the specified
282 * synchronization point (sync_tid), propagating parent chain modifications
283 * and mirror_tid updates back up as needed. Since we are recursing downward
284 * we do not have to deal with the complexities of multi-homed chains (chains
285 * with multiple parents).
287 * Caller must have interlocked against any non-flush-related modifying
288 * operations in progress whos modify_tid values are less than or equal
289 * to the passed sync_tid.
291 * Caller must have already vetted synchronization points to ensure they
292 * are properly flushed. Only snapshots and cluster flushes can create
293 * these sorts of synchronization points.
295 * This routine can be called from several places but the most important
296 * is from the hammer2_vop_reclaim() function. We want to try to completely
297 * clean out the inode structure to prevent disconnected inodes from
298 * building up and blowing out the kmalloc pool. However, it is not actually
299 * necessary to flush reclaimed inodes to maintain HAMMER2's crash recovery
302 * chain is locked on call and will remain locked on return. If a flush
303 * occured, the chain's MOVED bit will be set indicating that its parent
304 * (which is not part of the flush) should be updated.
307 hammer2_chain_flush(hammer2_trans_t *trans, hammer2_chain_t *chain)
309 hammer2_chain_t *scan;
310 hammer2_chain_core_t *core;
311 hammer2_flush_info_t info;
314 * Execute the recursive flush and handle deferrals.
316 * Chains can be ridiculously long (thousands deep), so to
317 * avoid blowing out the kernel stack the recursive flush has a
318 * depth limit. Elements at the limit are placed on a list
319 * for re-execution after the stack has been popped.
321 bzero(&info, sizeof(info));
322 TAILQ_INIT(&info.flush_list);
324 info.sync_tid = trans->sync_tid;
326 info.cache_index = -1;
332 * Unwind deep recursions which had been deferred. This
333 * can leave MOVED set for these chains, which will be
334 * handled when we [re]flush chain after the unwind.
336 while ((scan = TAILQ_FIRST(&info.flush_list)) != NULL) {
337 KKASSERT(scan->flags & HAMMER2_CHAIN_DEFERRED);
338 TAILQ_REMOVE(&info.flush_list, scan, flush_node);
339 atomic_clear_int(&scan->flags, HAMMER2_CHAIN_DEFERRED);
342 * Now that we've popped back up we can do a secondary
343 * recursion on the deferred elements.
345 if (hammer2_debug & 0x0040)
346 kprintf("defered flush %p\n", scan);
347 hammer2_chain_lock(scan, HAMMER2_RESOLVE_MAYBE);
348 hammer2_chain_flush(trans, scan);
349 hammer2_chain_unlock(scan);
350 hammer2_chain_drop(scan); /* ref from deferral */
354 * Flush pass1 on root.
356 info.diddeferral = 0;
357 hammer2_chain_flush_core(&info, chain);
359 kprintf("flush_core_done parent=<base> chain=%p.%d %08x\n",
360 chain, chain->bref.type, chain->flags);
364 * Only loop if deep recursions have been deferred.
366 if (TAILQ_EMPTY(&info.flush_list))
372 * This is the core of the chain flushing code. The chain is locked by the
373 * caller and remains locked on return. This function is keyed off of
374 * the SUBMODIFIED bit but must make fine-grained choices based on the
375 * synchronization point we are flushing to.
377 * If the flush accomplished any work chain will be flagged MOVED
378 * indicating a copy-on-write propagation back up is required.
379 * Deep sub-nodes may also have been entered onto the deferral list.
380 * MOVED is never set on the volume root.
382 * NOTE: modify_tid is different from MODIFIED. modify_tid is updated
383 * only when a chain is specifically modified, and not updated
384 * for copy-on-write propagations. MODIFIED is set on any modification
385 * including copy-on-write propagations.
388 hammer2_chain_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain)
390 hammer2_mount_t *hmp;
391 hammer2_blockref_t *bref;
394 hammer2_tid_t saved_sync;
396 hammer2_trans_t *trans = info->trans;
398 hammer2_chain_core_t *core;
411 kprintf("flush_core %p->%p.%d %08x (%s)\n",
412 info->parent, chain, chain->bref.type,
414 ((chain->bref.type == HAMMER2_BREF_TYPE_INODE) ?
415 chain->data->ipdata.filename : "?"));
417 kprintf("flush_core NULL->%p.%d %08x (%s)\n",
418 chain, chain->bref.type,
420 ((chain->bref.type == HAMMER2_BREF_TYPE_INODE) ?
421 chain->data->ipdata.filename : "?"));
424 * Ignore chains modified beyond the current flush point. These
425 * will be treated as if they did not exist.
427 if (chain->modify_tid > info->sync_tid)
432 * Deleted chains which have not been destroyed must be retained,
433 * and we probably have to recurse to clean-up any sub-trees.
434 * However, restricted flushes can stop processing here because
435 * the chain cleanup will be handled by a later normal flush.
437 * The MODIFIED bit can likely be cleared in this situation and we
438 * will do so later on in this procedure.
440 if (chain->delete_tid <= info->sync_tid) {
441 if (trans->flags & HAMMER2_TRANS_RESTRICTED)
446 saved_sync = info->sync_tid;
450 * If SUBMODIFIED is set we recurse the flush and adjust the
451 * blockrefs accordingly.
453 * NOTE: Looping on SUBMODIFIED can prevent a flush from ever
454 * finishing in the face of filesystem activity.
456 if (chain->flags & HAMMER2_CHAIN_SUBMODIFIED) {
457 hammer2_chain_t *saved_parent;
458 hammer2_tid_t saved_mirror;
459 hammer2_chain_layer_t *layer;
462 * Clear SUBMODIFIED to catch races. Note that any child
463 * with MODIFIED, DELETED, or MOVED set during scan2, or
464 * which tries to lastdrop but can't free its structures,
465 * or which gets defered, will cause SUBMODIFIED to be set
468 * We don't want to set our chain to MODIFIED gratuitously.
470 * We need an extra ref on chain because we are going to
471 * release its lock temporarily in our child loop.
473 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_SUBMODIFIED);
474 hammer2_chain_ref(chain);
477 * Run two passes. The first pass handles MODIFIED and
478 * SUBMODIFIED chains and recurses while the second pass
479 * handles MOVED chains on the way back up.
481 * If the stack gets too deep we defer scan1, but must
482 * be sure to still run scan2 if on the next loop the
483 * deferred chain has been flushed and now needs MOVED
484 * handling on the way back up.
486 * Scan1 is recursive.
488 * NOTE: The act of handling a modified/submodified chain can
489 * cause the MOVED Flag to be set. It can also be set
490 * via hammer2_chain_delete() and in other situations.
492 * NOTE: RB_SCAN() must be used instead of RB_FOREACH()
493 * because children can be physically removed during
496 saved_parent = info->parent;
497 saved_mirror = info->mirror_tid;
498 info->parent = chain;
499 info->mirror_tid = chain->bref.mirror_tid;
501 if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
502 if ((chain->flags & HAMMER2_CHAIN_DEFERRED) == 0) {
503 hammer2_chain_ref(chain);
504 TAILQ_INSERT_TAIL(&info->flush_list,
506 atomic_set_int(&chain->flags,
507 HAMMER2_CHAIN_DEFERRED);
511 info->diddeferral = 0;
512 spin_lock(&core->cst.spin);
513 KKASSERT(core->good == 0x1234 && core->sharecnt > 0);
514 TAILQ_FOREACH_REVERSE(layer, &core->layerq,
515 h2_layer_list, entry) {
517 KKASSERT(layer->good == 0xABCD);
518 RB_SCAN(hammer2_chain_tree, &layer->rbtree,
519 NULL, hammer2_chain_flush_scan1, info);
521 diddeferral += info->diddeferral;
523 spin_unlock(&core->cst.spin);
527 * Handle successfully flushed children who are in the MOVED
528 * state on the way back up the recursion. This can have
529 * the side-effect of clearing MOVED.
531 * Scan2 is non-recursive.
534 atomic_set_int(&chain->flags,
535 HAMMER2_CHAIN_SUBMODIFIED);
536 spin_lock(&core->cst.spin);
538 spin_lock(&core->cst.spin);
539 KKASSERT(core->good == 0x1234 && core->sharecnt > 0);
540 TAILQ_FOREACH_REVERSE(layer, &core->layerq,
541 h2_layer_list, entry) {
544 KKASSERT(layer->good == 0xABCD);
545 RB_SCAN(hammer2_chain_tree, &layer->rbtree,
546 NULL, hammer2_chain_flush_scan2, info);
548 RB_SCAN(hammer2_chain_tree, &layer->rbtree,
549 NULL, hammer2_chain_flush_scan2, info);
550 /*diddeferral += info->diddeferral; n/a*/
554 hammer2_chain_layer_check_locked(chain->hmp, core);
555 spin_unlock(&core->cst.spin);
557 chain->bref.mirror_tid = info->mirror_tid;
558 info->mirror_tid = saved_mirror;
559 info->parent = saved_parent;
560 KKASSERT(chain->refs > 1);
561 hammer2_chain_drop(chain);
565 * Restore sync_tid in case it was restricted by a delete/duplicate.
567 info->sync_tid = saved_sync;
570 * Rollup diddeferral for caller. Note direct assignment, not +=.
572 info->diddeferral = diddeferral;
575 * Do not flush chain if there were any deferrals. It will be
576 * retried later after the deferrals are independently handled.
579 if (hammer2_debug & 0x0008) {
580 kprintf("%*.*s} %p/%d %04x (deferred)",
581 info->depth, info->depth, "",
582 chain, chain->refs, chain->flags);
588 * If we encounter a deleted chain within our flush we can clear
589 * the MODIFIED bit and avoid flushing it whether it has been
590 * destroyed or not. We must make sure that the chain is flagged
591 * MOVED in this situation so the parent picks up the deletion.
593 * Note that scan2 has already executed above so statistics have
594 * already been rolled up.
596 if (chain->delete_tid <= info->sync_tid) {
597 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
599 if (chain->bytes == chain->bp->b_bufsize)
600 chain->bp->b_flags |= B_INVAL|B_RELBUF;
602 if ((chain->flags & HAMMER2_CHAIN_MOVED) == 0) {
603 hammer2_chain_ref(chain);
604 atomic_set_int(&chain->flags,
605 HAMMER2_CHAIN_MOVED);
607 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
608 hammer2_chain_drop(chain);
613 if ((chain->flags & HAMMER2_CHAIN_DESTROYED) &&
614 (chain->flags & HAMMER2_CHAIN_DELETED) &&
615 (trans->flags & HAMMER2_TRANS_RESTRICTED) == 0) {
617 * Throw-away the MODIFIED flag
619 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
621 if (chain->bytes == chain->bp->b_bufsize)
622 chain->bp->b_flags |= B_INVAL|B_RELBUF;
624 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
625 hammer2_chain_drop(chain);
632 * A degenerate flush might not have flushed anything and thus not
633 * processed modified blocks on the way back up. Detect the case.
635 * Note that MOVED can be set without MODIFIED being set due to
636 * a deletion, in which case it is handled by Scan2 later on.
638 * Both bits can be set along with DELETED due to a deletion if
639 * modified data within the synchronization zone and the chain
640 * was then deleted beyond the zone, in which case we still have
641 * to flush for synchronization point consistency. Otherwise though
642 * DELETED and MODIFIED are treated as separate flags.
644 if ((chain->flags & HAMMER2_CHAIN_MODIFIED) == 0)
650 * A DESTROYED node that reaches this point must be flushed for
651 * synchronization point consistency.
655 * Update mirror_tid, clear MODIFIED, and set MOVED.
657 * The caller will update the parent's reference to this chain
658 * by testing MOVED as long as the modification was in-bounds.
660 * MOVED is never set on the volume root as there is no parent
663 if (chain->bref.mirror_tid < info->sync_tid)
664 chain->bref.mirror_tid = info->sync_tid;
665 wasmodified = (chain->flags & HAMMER2_CHAIN_MODIFIED) != 0;
666 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
667 if (chain == &hmp->vchain)
668 kprintf("(FLUSHED VOLUME HEADER)\n");
669 if (chain == &hmp->fchain)
670 kprintf("(FLUSHED FREEMAP HEADER)\n");
672 if ((chain->flags & HAMMER2_CHAIN_MOVED) ||
673 chain == &hmp->vchain ||
674 chain == &hmp->fchain) {
676 * Drop the ref from the MODIFIED bit we cleared.
677 * Net is -0 or -1 ref depending.
680 hammer2_chain_drop(chain);
683 * Drop the ref from the MODIFIED bit we cleared and
684 * set a ref for the MOVED bit we are setting. Net
685 * is +0 or +1 ref depending.
687 if (wasmodified == 0)
688 hammer2_chain_ref(chain);
689 atomic_set_int(&chain->flags, HAMMER2_CHAIN_MOVED);
693 * If this is part of a recursive flush we can go ahead and write
694 * out the buffer cache buffer and pass a new bref back up the chain
697 * Volume headers are NOT flushed here as they require special
700 switch(chain->bref.type) {
701 case HAMMER2_BREF_TYPE_FREEMAP:
702 hammer2_modify_volume(hmp);
704 case HAMMER2_BREF_TYPE_VOLUME:
706 * We should flush the free block table before we calculate
707 * CRCs and copy voldata -> volsync.
709 * To prevent SMP races, fchain must remain locked until
710 * voldata is copied to volsync.
712 hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
713 if (hmp->fchain.flags & (HAMMER2_CHAIN_MODIFIED |
714 HAMMER2_CHAIN_SUBMODIFIED)) {
715 /* this will modify vchain as a side effect */
716 hammer2_chain_flush(info->trans, &hmp->fchain);
720 * The volume header is flushed manually by the syncer, not
721 * here. All we do is adjust the crc's.
723 KKASSERT(chain->data != NULL);
724 KKASSERT(chain->bp == NULL);
725 kprintf("volume header mirror_tid %jd\n",
726 hmp->voldata.mirror_tid);
728 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
730 (char *)&hmp->voldata +
731 HAMMER2_VOLUME_ICRC1_OFF,
732 HAMMER2_VOLUME_ICRC1_SIZE);
733 hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
735 (char *)&hmp->voldata +
736 HAMMER2_VOLUME_ICRC0_OFF,
737 HAMMER2_VOLUME_ICRC0_SIZE);
738 hmp->voldata.icrc_volheader =
740 (char *)&hmp->voldata +
741 HAMMER2_VOLUME_ICRCVH_OFF,
742 HAMMER2_VOLUME_ICRCVH_SIZE);
743 hmp->volsync = hmp->voldata;
744 atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
745 hammer2_chain_unlock(&hmp->fchain);
747 case HAMMER2_BREF_TYPE_DATA:
749 * Data elements have already been flushed via the logical
750 * file buffer cache. Their hash was set in the bref by
751 * the vop_write code.
753 * Make sure any device buffer(s) have been flushed out here.
754 * (there aren't usually any to flush).
756 psize = hammer2_devblksize(chain->bytes);
757 pmask = (hammer2_off_t)psize - 1;
758 pbase = chain->bref.data_off & ~pmask;
759 boff = chain->bref.data_off & (HAMMER2_OFF_MASK & pmask);
761 bp = getblk(hmp->devvp, pbase, psize, GETBLK_NOWAIT, 0);
763 if ((bp->b_flags & (B_CACHE | B_DIRTY)) ==
764 (B_CACHE | B_DIRTY)) {
767 bp->b_flags |= B_RELBUF;
773 case HAMMER2_BREF_TYPE_INDIRECT:
775 * Indirect blocks may be in an INITIAL state. Use the
776 * chain_lock() call to ensure that the buffer has been
777 * instantiated (even though it is already locked the buffer
778 * might not have been instantiated).
780 * Only write the buffer out if it is dirty, it is possible
781 * the operating system had already written out the buffer.
783 hammer2_chain_lock(chain, HAMMER2_RESOLVE_ALWAYS);
784 KKASSERT(chain->bp != NULL);
787 if ((chain->flags & HAMMER2_CHAIN_DIRTYBP) ||
788 (bp->b_flags & B_DIRTY)) {
795 hammer2_chain_unlock(chain);
798 case HAMMER2_BREF_TYPE_INDIRECT:
799 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
801 * Device-backed. Buffer will be flushed by the sync
804 KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
806 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
809 * Embedded elements have to be flushed out.
810 * (Basically just BREF_TYPE_INODE).
812 KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
813 KKASSERT(chain->data != NULL);
814 KKASSERT(chain->bp == NULL);
817 KKASSERT((bref->data_off & HAMMER2_OFF_MASK) != 0);
818 KKASSERT(HAMMER2_DEC_CHECK(chain->bref.methods) ==
819 HAMMER2_CHECK_ISCSI32 ||
820 HAMMER2_DEC_CHECK(chain->bref.methods) ==
821 HAMMER2_CHECK_FREEMAP);
824 * The data is embedded, we have to acquire the
825 * buffer cache buffer and copy the data into it.
827 psize = hammer2_devblksize(chain->bytes);
828 pmask = (hammer2_off_t)psize - 1;
829 pbase = bref->data_off & ~pmask;
830 boff = bref->data_off & (HAMMER2_OFF_MASK & pmask);
833 * The getblk() optimization can only be used if the
834 * physical block size matches the request.
836 error = bread(hmp->devvp, pbase, psize, &bp);
837 KKASSERT(error == 0);
839 bdata = (char *)bp->b_data + boff;
842 * Copy the data to the buffer, mark the buffer
843 * dirty, and convert the chain to unmodified.
845 bcopy(chain->data, bdata, chain->bytes);
846 bp->b_flags |= B_CLUSTEROK;
850 switch(HAMMER2_DEC_CHECK(chain->bref.methods)) {
851 case HAMMER2_CHECK_FREEMAP:
852 chain->bref.check.freemap.icrc32 =
853 hammer2_icrc32(chain->data, chain->bytes);
855 case HAMMER2_CHECK_ISCSI32:
856 chain->bref.check.iscsi32.value =
857 hammer2_icrc32(chain->data, chain->bytes);
860 panic("hammer2_flush_core: bad crc type");
861 break; /* NOT REACHED */
863 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE)
864 ++hammer2_iod_meta_write;
866 ++hammer2_iod_indr_write;
871 * Flush helper scan1 (recursive)
873 * Flushes the children of the caller's chain (parent) and updates
874 * the blockref, restricted by sync_tid.
876 * Ripouts during the loop should not cause any problems. Because we are
877 * flushing to a synchronization point, modification races will occur after
878 * sync_tid and do not have to be flushed anyway.
880 * It is also ok if the parent is chain_duplicate()'d while unlocked because
881 * the delete/duplication will install a delete_tid that is still larger than
882 * our current sync_tid.
885 hammer2_chain_flush_scan1(hammer2_chain_t *child, void *data)
887 hammer2_flush_info_t *info = data;
888 hammer2_trans_t *trans = info->trans;
889 hammer2_chain_t *parent = info->parent;
893 * We should only need to recurse if SUBMODIFIED is set, but as
894 * a safety also recurse if MODIFIED is also set.
896 * Return early if neither bit is set. We must re-assert the
897 * SUBMODIFIED flag in the parent if any child covered by the
898 * parent (via delete_tid) is skipped.
900 if ((child->flags & (HAMMER2_CHAIN_MODIFIED |
901 HAMMER2_CHAIN_SUBMODIFIED)) == 0) {
904 if (child->modify_tid > trans->sync_tid) {
905 if (parent->delete_tid > trans->sync_tid) {
906 atomic_set_int(&parent->flags,
907 HAMMER2_CHAIN_SUBMODIFIED);
912 hammer2_chain_ref(child);
913 spin_unlock(&parent->core->cst.spin);
916 * The caller has added a ref to the parent so we can temporarily
917 * unlock it in order to lock the child. Re-check the flags before
920 hammer2_chain_unlock(parent);
921 hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
923 if ((child->flags & (HAMMER2_CHAIN_MODIFIED |
924 HAMMER2_CHAIN_SUBMODIFIED)) == 0) {
925 hammer2_chain_unlock(child);
926 hammer2_chain_drop(child);
927 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
928 spin_lock(&parent->core->cst.spin);
931 if (child->modify_tid > trans->sync_tid) {
932 hammer2_chain_unlock(child);
933 hammer2_chain_drop(child);
934 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
935 spin_lock(&parent->core->cst.spin);
936 if (parent->delete_tid > trans->sync_tid) {
937 atomic_set_int(&parent->flags,
938 HAMMER2_CHAIN_SUBMODIFIED);
944 * The DESTROYED flag can only be initially set on an unreferenced
945 * deleted inode and will propagate downward via the mechanic below.
946 * Such inode chains have been deleted for good and should no longer
947 * be subject to delete/duplication.
949 * This optimization allows the inode reclaim (destroy unlinked file
950 * on vnode reclamation after last close) to be flagged by just
951 * setting HAMMER2_CHAIN_DESTROYED at the top level and then will
952 * cause the chains to be terminated and related buffers to be
953 * invalidated and not flushed out.
955 * We have to be careful not to propagate the DESTROYED flag if
956 * the destruction occurred after our flush sync_tid.
958 if ((parent->flags & HAMMER2_CHAIN_DESTROYED) &&
959 (child->flags & HAMMER2_CHAIN_DELETED) &&
960 (child->flags & HAMMER2_CHAIN_DESTROYED) == 0) {
961 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROYED |
962 HAMMER2_CHAIN_SUBMODIFIED);
966 * Recurse and collect deferral data.
968 diddeferral = info->diddeferral;
970 hammer2_chain_flush_core(info, child);
972 kprintf("flush_core_done parent=%p flags=%08x child=%p.%d %08x\n",
973 parent, parent->flags, child, child->bref.type, child->flags);
976 info->diddeferral += diddeferral;
978 if (child->flags & HAMMER2_CHAIN_SUBMODIFIED)
979 atomic_set_int(&parent->flags, HAMMER2_CHAIN_SUBMODIFIED);
981 hammer2_chain_unlock(child);
982 hammer2_chain_drop(child);
984 hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
986 spin_lock(&parent->core->cst.spin);
992 * Flush helper scan2 (non-recursive)
994 * This pass on a chain's children propagates any MOVED or DELETED
995 * elements back up the chain towards the root after those elements have
996 * been fully flushed. Unlike scan1, this function is NOT recursive and
997 * the parent remains locked across the entire scan.
999 * SCAN2 is called twice, once with pass set to 1 and once with it set to 2.
1000 * We have to do this so base[] elements can be deleted in pass 1 to make
1001 * room for adding new elements in pass 2.
1003 * This function also rolls up storage statistics.
1005 * NOTE! We must re-set SUBMODIFIED on the parent(s) as appropriate, and
1006 * due to the above conditions it is possible to do this and still
1007 * have some children flagged MOVED depending on the synchronization.
1009 * NOTE! A deletion is a visbility issue, there can still be references to
1010 * deleted elements (for example, to an unlinked file which is still
1011 * open), and there can also be multiple chains pointing to the same
1012 * bref where some are deleted and some are not (for example due to
1013 * a rename). So a chain marked for deletion is basically considered
1014 * to be live until it is explicitly destroyed or until its ref-count
1015 * reaches zero (also implying that MOVED and MODIFIED are clear).
1018 hammer2_chain_flush_scan2(hammer2_chain_t *child, void *data)
1020 hammer2_flush_info_t *info = data;
1021 hammer2_chain_t *parent = info->parent;
1022 hammer2_chain_core_t *above = child->above;
1023 hammer2_mount_t *hmp = child->hmp;
1024 hammer2_trans_t *trans = info->trans;
1025 hammer2_blockref_t *base;
1030 * Inodes with stale children that have been converted to DIRECTDATA
1031 * mode (file extension or hardlink conversion typically) need to
1032 * skipped right now before we start messing with a non-existant
1036 if (parent->bref.type == HAMMER2_BREF_TYPE_INODE &&
1037 (parent->data->ipdata.op_flags & HAMMER2_OPFLAG_DIRECTDATA)) {
1043 * Ignore children created after our flush point, treating them as
1044 * if they did not exist). These children will not cause the parent
1047 * When we encounter such children and the parent chain has not been
1048 * deleted, delete/duplicated, or delete/duplicated-for-move, then
1049 * the parent may be used to funnel through several flush points.
1050 * We must re-set the SUBMODIFIED flag in the parent to ensure that
1051 * those flushes have visbility. A simple test of delete_tid suffices
1052 * to determine if the parent spans beyond our current flush.
1054 if (child->modify_tid > trans->sync_tid) {
1059 * Ignore children which have not changed. The parent's block table
1060 * is already correct.
1062 * XXX The MOVED bit is only cleared when all multi-homed parents
1063 * have flushed, creating a situation where a re-flush can occur
1064 * via a parent which has already flushed. The hammer2_base_*()
1065 * functions currently have a hack to deal with this case but
1066 * we need something better.
1068 if ((child->flags & HAMMER2_CHAIN_MOVED) == 0) {
1073 * Make sure child is referenced before we unlock.
1075 hammer2_chain_ref(child);
1076 spin_unlock(&above->cst.spin);
1079 * Parent reflushed after the child has passed them by should skip
1080 * due to the modify_tid test. XXX
1082 hammer2_chain_lock(child, HAMMER2_RESOLVE_NEVER);
1083 KKASSERT(child->above == above);
1084 KKASSERT(parent->core == above);
1087 * The parent's blockref to the child must be deleted or updated.
1089 * This point is not reached on successful DESTROYED optimizations
1090 * but can be reached on recursive deletions and restricted flushes.
1092 * Because flushes are ordered we do not have to make a
1093 * modify/duplicate of indirect blocks. That is, the flush
1094 * code does not have to kmalloc or duplicate anything. We
1095 * can adjust the indirect block table in-place and reuse the
1096 * chain. It IS possible that the chain has already been duplicated
1097 * or may wind up being duplicated on-the-fly by modifying code
1098 * on the frontend. We simply use the original and ignore such
1099 * chains. However, it does mean we can't clear the MOVED bit.
1101 * XXX recursive deletions not optimized.
1103 hammer2_chain_modify(trans, &parent,
1104 HAMMER2_MODIFY_NO_MODIFY_TID |
1105 HAMMER2_MODIFY_ASSERTNOCOPY);
1107 switch(parent->bref.type) {
1108 case HAMMER2_BREF_TYPE_INODE:
1110 * XXX Should assert that OPFLAG_DIRECTDATA is 0 once we
1111 * properly duplicate the inode headers and do proper flush
1112 * range checks (all the children should be beyond the flush
1113 * point). For now just don't sync the non-applicable
1116 * XXX Can also occur due to hardlink consolidation. We
1117 * set OPFLAG_DIRECTDATA to prevent the indirect and data
1118 * blocks from syncing ot the hardlink pointer.
1121 KKASSERT((parent->data->ipdata.op_flags &
1122 HAMMER2_OPFLAG_DIRECTDATA) == 0);
1125 if (parent->data->ipdata.op_flags & HAMMER2_OPFLAG_DIRECTDATA) {
1130 base = &parent->data->ipdata.u.blockset.blockref[0];
1131 count = HAMMER2_SET_COUNT;
1134 case HAMMER2_BREF_TYPE_INDIRECT:
1135 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1137 base = &parent->data->npdata[0];
1140 KKASSERT(child->flags & HAMMER2_CHAIN_DELETED);
1142 count = parent->bytes / sizeof(hammer2_blockref_t);
1144 case HAMMER2_BREF_TYPE_VOLUME:
1145 base = &hmp->voldata.sroot_blockset.blockref[0];
1146 count = HAMMER2_SET_COUNT;
1148 case HAMMER2_BREF_TYPE_FREEMAP:
1149 base = &parent->data->npdata[0];
1150 count = HAMMER2_SET_COUNT;
1155 panic("hammer2_chain_flush_scan2: "
1156 "unrecognized blockref type: %d",
1161 * Don't bother updating a deleted parent's blockrefs (caller will
1162 * optimize-out the disk write). Note that this is not optional,
1163 * a deleted parent's blockref array might not be synchronized at
1164 * all so calling hammer2_base*() functions could result in a panic.
1166 * Otherwise, we need to be COUNTEDBREFS synchronized for the
1167 * hammer2_base_*() functions.
1169 if (parent->delete_tid <= trans->sync_tid)
1171 else if ((parent->core->flags & HAMMER2_CORE_COUNTEDBREFS) == 0)
1172 hammer2_chain_countbrefs(parent, base, count);
1175 * Update the parent's blockref table and propagate mirror_tid.
1177 * NOTE! Children with modify_tid's beyond our flush point are
1178 * considered to not exist for the purposes of updating the
1179 * parent's blockref array.
1181 * NOTE! Updates to a parent's blockref table do not adjust the
1182 * parent's bref.modify_tid, only its bref.mirror_tid.
1184 if (info->pass == 1 && child->delete_tid <= trans->sync_tid) {
1186 * Deleting. Only adjust the block array if it contains
1187 * the child's entry (child's REPLACE flag is set). Clear
1188 * the child's REPLACE flag only once all possible parent's
1189 * have been updated.
1192 if (base && (child->flags & HAMMER2_CHAIN_REPLACE)) {
1193 hammer2_rollup_stats(parent, child, -1);
1194 spin_lock(&above->cst.spin);
1195 hammer2_base_delete(parent, base, count,
1196 &info->cache_index, &child->bref);
1197 if (TAILQ_NEXT(parent, core_entry) == NULL) {
1198 atomic_clear_int(&child->flags,
1199 HAMMER2_CHAIN_REPLACE);
1201 spin_unlock(&above->cst.spin);
1203 if (info->mirror_tid < child->delete_tid)
1204 info->mirror_tid = child->delete_tid;
1205 } else if (info->pass == 2 && child->delete_tid > trans->sync_tid) {
1207 * Inserting. Only set the child's REPLACE flag indicating
1208 * that the parent's blockref array entry is valid once all
1209 * possible parent's have been updated.
1213 if (child->flags & HAMMER2_CHAIN_REPLACE)
1214 hammer2_rollup_stats(parent, child, 0);
1216 hammer2_rollup_stats(parent, child, 1);
1217 spin_lock(&above->cst.spin);
1218 hammer2_base_insert(parent, base, count,
1219 &info->cache_index, &child->bref,
1221 if (TAILQ_NEXT(parent, core_entry) == NULL) {
1222 atomic_set_int(&child->flags,
1223 HAMMER2_CHAIN_REPLACE);
1225 spin_unlock(&above->cst.spin);
1227 if (info->mirror_tid < child->modify_tid)
1228 info->mirror_tid = child->modify_tid;
1233 if (info->mirror_tid < child->bref.mirror_tid) {
1234 info->mirror_tid = child->bref.mirror_tid;
1236 if ((parent->bref.type == HAMMER2_BREF_TYPE_VOLUME ||
1237 parent->bref.type == HAMMER2_BREF_TYPE_FREEMAP) &&
1238 hmp->voldata.mirror_tid < child->bref.mirror_tid) {
1239 hmp->voldata.mirror_tid = child->bref.mirror_tid;
1243 * Only clear MOVED once all possible parents have been flushed.
1245 * When can we safely clear the MOVED flag? Flushes down duplicate
1246 * paths can occur out of order, for example if an inode is moved
1247 * as part of a hardlink consolidation or if an inode is moved into
1248 * an indirect block indexed before the inode.
1250 if (ok && (child->flags & HAMMER2_CHAIN_MOVED)) {
1251 hammer2_chain_t *scan;
1254 spin_lock(&above->cst.spin);
1255 TAILQ_FOREACH(scan, &above->ownerq, core_entry) {
1257 * XXX weird code also checked at the top of scan2,
1258 * I would like to fix this by detaching the core
1259 * on initial hardlink consolidation (1->2 nlinks).
1262 if (scan->bref.type == HAMMER2_BREF_TYPE_INODE &&
1263 (scan->data->ipdata.op_flags &
1264 HAMMER2_OPFLAG_DIRECTDATA)) {
1268 if (scan->flags & HAMMER2_CHAIN_SUBMODIFIED) {
1273 spin_unlock(&above->cst.spin);
1275 atomic_clear_int(&child->flags, HAMMER2_CHAIN_MOVED);
1276 hammer2_chain_drop(child); /* flag */
1281 * Unlock the child. This can wind up dropping the child's
1282 * last ref, removing it from the parent's RB tree, and deallocating
1283 * the structure. The RB_SCAN() our caller is doing handles the
1286 hammer2_chain_unlock(child);
1287 hammer2_chain_drop(child);
1288 spin_lock(&above->cst.spin);
1291 * The parent cleared SUBMODIFIED prior to the scan. If the child
1292 * still requires a flush (possibly due to being outside the current
1293 * synchronization zone), we must re-set SUBMODIFIED on the way back
1302 hammer2_rollup_stats(hammer2_chain_t *parent, hammer2_chain_t *child, int how)
1305 hammer2_chain_t *grandp;
1308 parent->data_count += child->data_count;
1309 parent->inode_count += child->inode_count;
1310 child->data_count = 0;
1311 child->inode_count = 0;
1313 parent->data_count -= child->bytes;
1314 if (child->bref.type == HAMMER2_BREF_TYPE_INODE) {
1315 parent->inode_count -= 1;
1317 /* XXX child->data may be NULL atm */
1318 parent->data_count -= child->data->ipdata.data_count;
1319 parent->inode_count -= child->data->ipdata.inode_count;
1322 } else if (how > 0) {
1323 parent->data_count += child->bytes;
1324 if (child->bref.type == HAMMER2_BREF_TYPE_INODE) {
1325 parent->inode_count += 1;
1327 /* XXX child->data may be NULL atm */
1328 parent->data_count += child->data->ipdata.data_count;
1329 parent->inode_count += child->data->ipdata.inode_count;
1333 if (parent->bref.type == HAMMER2_BREF_TYPE_INODE) {
1334 parent->data->ipdata.data_count += parent->data_count;
1335 parent->data->ipdata.inode_count += parent->inode_count;
1337 for (grandp = parent->above->first_parent;
1339 grandp = grandp->next_parent) {
1340 grandp->data_count += parent->data_count;
1341 grandp->inode_count += parent->inode_count;
1344 parent->data_count = 0;
1345 parent->inode_count = 0;