hammer - Fix bugs, fix serious snapshot bug, flush adjustments
[dragonfly.git] / sys / vfs / hammer2 / hammer2_flush.c
1 /*
2  * Copyright (c) 2011-2018 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@dragonflybsd.org>
6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in
16  *    the documentation and/or other materials provided with the
17  *    distribution.
18  * 3. Neither the name of The DragonFly Project nor the names of its
19  *    contributors may be used to endorse or promote products derived
20  *    from this software without specific, prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33  * SUCH DAMAGE.
34  */
35 /*
36  *                      TRANSACTION AND FLUSH HANDLING
37  *
38  * Deceptively simple but actually fairly difficult to implement properly is
39  * how I would describe it.
40  *
41  * Flushing generally occurs bottom-up but requires a top-down scan to
42  * locate chains with MODIFIED and/or UPDATE bits set.  The ONFLUSH flag
43  * tells how to recurse downward to find these chains.
44  */
45
46 #include <sys/cdefs.h>
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/types.h>
50 #include <sys/lock.h>
51 #include <sys/uuid.h>
52
53 #include "hammer2.h"
54
55 #define FLUSH_DEBUG 0
56
57 #define HAMMER2_FLUSH_DEPTH_LIMIT       60      /* stack recursion limit */
58
59
60 /*
61  * Recursively flush the specified chain.  The chain is locked and
62  * referenced by the caller and will remain so on return.  The chain
63  * will remain referenced throughout but can temporarily lose its
64  * lock during the recursion to avoid unnecessarily stalling user
65  * processes.
66  */
67 struct hammer2_flush_info {
68         hammer2_chain_t *parent;
69         int             depth;
70         long            diddeferral;
71         int             error;                  /* cumulative error */
72         int             flags;
73 #ifdef HAMMER2_SCAN_DEBUG
74         long            scan_count;
75         long            scan_mod_count;
76         long            scan_upd_count;
77         long            scan_onf_count;
78         long            scan_del_count;
79         long            scan_btype[7];
80         long            flushq_count;
81 #endif
82         struct h2_flush_list flushq;
83         hammer2_chain_t *debug;
84 };
85
86 typedef struct hammer2_flush_info hammer2_flush_info_t;
87
88 static void hammer2_flush_core(hammer2_flush_info_t *info,
89                                 hammer2_chain_t *chain, int flags);
90 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
91
92 /*
93  * Any per-pfs transaction initialization goes here.
94  */
95 void
96 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
97 {
98 }
99
100 /*
101  * Transaction support for any modifying operation.  Transactions are used
102  * in the pmp layer by the frontend and in the spmp layer by the backend.
103  *
104  * 0                    - Normal transaction, interlocked against flush
105  *                        transaction.
106  *
107  * TRANS_ISFLUSH        - Flush transaction, interlocked against normal
108  *                        transaction.
109  *
110  * TRANS_BUFCACHE       - Buffer cache transaction, no interlock.
111  *
112  * Initializing a new transaction allocates a transaction ID.  Typically
113  * passed a pmp (hmp passed as NULL), indicating a cluster transaction.  Can
114  * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
115  * media target.  The latter mode is used by the recovery code.
116  *
117  * TWO TRANSACTION IDs can run concurrently, where one is a flush and the
118  * other is a set of any number of concurrent filesystem operations.  We
119  * can either have <running_fs_ops> + <waiting_flush> + <blocked_fs_ops>
120  * or we can have <running_flush> + <concurrent_fs_ops>.
121  *
122  * During a flush, new fs_ops are only blocked until the fs_ops prior to
123  * the flush complete.  The new fs_ops can then run concurrent with the flush.
124  *
125  * Buffer-cache transactions operate as fs_ops but never block.  A
126  * buffer-cache flush will run either before or after the current pending
127  * flush depending on its state.
128  */
129 void
130 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
131 {
132         uint32_t oflags;
133         uint32_t nflags;
134         int dowait;
135
136         for (;;) {
137                 oflags = pmp->trans.flags;
138                 cpu_ccfence();
139                 dowait = 0;
140
141                 if (flags & HAMMER2_TRANS_ISFLUSH) {
142                         /*
143                          * Requesting flush transaction.  Wait for all
144                          * currently running transactions to finish.
145                          * Afterwords, normal transactions will be
146                          * interlocked.
147                          */
148                         if (oflags & HAMMER2_TRANS_MASK) {
149                                 nflags = oflags | HAMMER2_TRANS_FPENDING |
150                                                   HAMMER2_TRANS_WAITING;
151                                 dowait = 1;
152                         } else {
153                                 nflags = (oflags | flags) + 1;
154                         }
155                 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
156                         /*
157                          * Requesting strategy transaction from buffer-cache,
158                          * or a VM getpages/putpages through the buffer cache.
159                          * We must allow such transactions in all situations
160                          * to avoid deadlocks.
161                          */
162                         nflags = (oflags | flags) + 1;
163 #if 0
164                         /*
165                          * (old) previous code interlocked against the main
166                          *       flush pass.
167                          */
168                         if ((oflags & (HAMMER2_TRANS_ISFLUSH |
169                                        HAMMER2_TRANS_PREFLUSH)) ==
170                             HAMMER2_TRANS_ISFLUSH) {
171                                 nflags = oflags | HAMMER2_TRANS_WAITING;
172                                 dowait = 1;
173                         } else {
174                                 nflags = (oflags | flags) + 1;
175                         }
176 #endif
177                 } else {
178                         /*
179                          * Requesting a normal modifying transaction.
180                          * Waits for any flush to finish before allowing.
181                          * Multiple modifying transactions can run
182                          * concurrently.
183                          *
184                          * If a flush is pending for more than one second
185                          * but can't run because many modifying transactions
186                          * are active, we wait for the flush to be granted.
187                          *
188                          * NOTE: Remember that non-modifying operations
189                          *       such as read, stat, readdir, etc, do
190                          *       not use transactions.
191                          */
192                         if ((oflags & HAMMER2_TRANS_FPENDING) &&
193                             (u_int)(ticks - pmp->trans.fticks) >= (u_int)hz) {
194                                 nflags = oflags | HAMMER2_TRANS_WAITING;
195                                 dowait = 1;
196                         } else if (oflags & HAMMER2_TRANS_ISFLUSH) {
197                                 nflags = oflags | HAMMER2_TRANS_WAITING;
198                                 dowait = 1;
199                         } else {
200                                 nflags = (oflags | flags) + 1;
201                         }
202                 }
203                 if (dowait)
204                         tsleep_interlock(&pmp->trans.sync_wait, 0);
205                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
206                         if ((oflags & HAMMER2_TRANS_FPENDING) == 0 &&
207                             (nflags & HAMMER2_TRANS_FPENDING)) {
208                                 pmp->trans.fticks = ticks;
209                         }
210                         if (dowait == 0)
211                                 break;
212                         tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
213                                "h2trans", hz);
214                 } else {
215                         cpu_pause();
216                 }
217                 /* retry */
218         }
219 }
220
221 /*
222  * Start a sub-transaction, there is no 'subdone' function.  This will
223  * issue a new modify_tid (mtid) for the current transaction, which is a
224  * CLC (cluster level change) id and not a per-node id.
225  *
226  * This function must be called for each XOP when multiple XOPs are run in
227  * sequence within a transaction.
228  *
229  * Callers typically update the inode with the transaction mtid manually
230  * to enforce sequencing.
231  */
232 hammer2_tid_t
233 hammer2_trans_sub(hammer2_pfs_t *pmp)
234 {
235         hammer2_tid_t mtid;
236
237         mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
238
239         return (mtid);
240 }
241
242 void
243 hammer2_trans_done(hammer2_pfs_t *pmp)
244 {
245         uint32_t oflags;
246         uint32_t nflags;
247
248         for (;;) {
249                 oflags = pmp->trans.flags;
250                 cpu_ccfence();
251                 KKASSERT(oflags & HAMMER2_TRANS_MASK);
252                 if ((oflags & HAMMER2_TRANS_MASK) == 1) {
253                         /*
254                          * This was the last transaction
255                          */
256                         nflags = (oflags - 1) & ~(HAMMER2_TRANS_ISFLUSH |
257                                                   HAMMER2_TRANS_BUFCACHE |
258                                                   HAMMER2_TRANS_FPENDING |
259                                                   HAMMER2_TRANS_WAITING);
260                 } else {
261                         /*
262                          * Still transactions pending
263                          */
264                         nflags = oflags - 1;
265                 }
266                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
267                         if ((nflags & HAMMER2_TRANS_MASK) == 0 &&
268                             (oflags & HAMMER2_TRANS_WAITING)) {
269                                 wakeup(&pmp->trans.sync_wait);
270                         }
271                         break;
272                 } else {
273                         cpu_pause();
274                 }
275                 /* retry */
276         }
277 }
278
279 /*
280  * Obtain new, unique inode number (not serialized by caller).
281  */
282 hammer2_tid_t
283 hammer2_trans_newinum(hammer2_pfs_t *pmp)
284 {
285         hammer2_tid_t tid;
286
287         tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
288
289         return tid;
290 }
291
292 /*
293  * Assert that a strategy call is ok here.  Currently we allow strategy
294  * calls in all situations, including during flushes.  Previously:
295  *      (old) (1) In a normal transaction.
296  *      (old) (2) In a flush transaction only if PREFLUSH is also set.
297  */
298 void
299 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
300 {
301 #if 0
302         KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 ||
303                  (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH));
304 #endif
305 }
306
307
308 /*
309  * Chains undergoing destruction are removed from the in-memory topology.
310  * To avoid getting lost these chains are placed on the delayed flush
311  * queue which will properly dispose of them.
312  *
313  * We do this instead of issuing an immediate flush in order to give
314  * recursive deletions (rm -rf, etc) a chance to remove more of the
315  * hierarchy, potentially allowing an enormous amount of write I/O to
316  * be avoided.
317  *
318  * NOTE: The flush code tests HAMMER2_CHAIN_DESTROY to differentiate
319  *       between these chains and the deep-recursion requeue.
320  */
321 void
322 hammer2_delayed_flush(hammer2_chain_t *chain)
323 {
324         if ((chain->flags & HAMMER2_CHAIN_DELAYED) == 0) {
325                 hammer2_spin_ex(&chain->hmp->list_spin);
326                 if ((chain->flags & (HAMMER2_CHAIN_DELAYED |
327                                      HAMMER2_CHAIN_DEFERRED)) == 0) {
328                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELAYED |
329                                                       HAMMER2_CHAIN_DEFERRED);
330                         TAILQ_INSERT_TAIL(&chain->hmp->flushq,
331                                           chain, flush_node);
332                         hammer2_chain_ref(chain);
333                 }
334                 hammer2_spin_unex(&chain->hmp->list_spin);
335                 hammer2_voldata_modify(chain->hmp);
336         }
337 }
338
339 /*
340  * Flush the chain and all modified sub-chains through the specified
341  * synchronization point, propagating blockref updates back up.  As
342  * part of this propagation, mirror_tid and inode/data usage statistics
343  * propagates back upward.
344  *
345  * Returns a HAMMER2 error code, 0 if no error.  Note that I/O errors from
346  * buffers dirtied during the flush operation can occur later.
347  *
348  * modify_tid (clc - cluster level change) is not propagated.
349  *
350  * update_tid (clc) is used for validation and is not propagated by this
351  * function.
352  *
353  * This routine can be called from several places but the most important
354  * is from VFS_SYNC (frontend) via hammer2_inode_xop_flush (backend).
355  *
356  * chain is locked on call and will remain locked on return.  The chain's
357  * UPDATE flag indicates that its parent's block table (which is not yet
358  * part of the flush) should be updated.
359  *
360  * flags:
361  *      HAMMER2_FLUSH_TOP       Indicates that this is the top of the flush.
362  *                              Is cleared for the recursion.
363  *
364  *      HAMMER2_FLUSH_ALL       Recurse everything
365  *
366  *      HAMMER2_FLUSH_INODE_STOP
367  *                              Stop at PFS inode or normal inode boundary
368  */
369 int
370 hammer2_flush(hammer2_chain_t *chain, int flags)
371 {
372         hammer2_chain_t *scan;
373         hammer2_flush_info_t info;
374         hammer2_dev_t *hmp;
375         int loops;
376
377         /*
378          * Execute the recursive flush and handle deferrals.
379          *
380          * Chains can be ridiculously long (thousands deep), so to
381          * avoid blowing out the kernel stack the recursive flush has a
382          * depth limit.  Elements at the limit are placed on a list
383          * for re-execution after the stack has been popped.
384          */
385         bzero(&info, sizeof(info));
386         TAILQ_INIT(&info.flushq);
387         info.flags = flags & ~HAMMER2_FLUSH_TOP;
388
389         /*
390          * Calculate parent (can be NULL), if not NULL the flush core
391          * expects the parent to be referenced so it can easily lock/unlock
392          * it without it getting ripped up.
393          */
394         if ((info.parent = chain->parent) != NULL)
395                 hammer2_chain_ref(info.parent);
396
397         /*
398          * Extra ref needed because flush_core expects it when replacing
399          * chain.
400          */
401         hammer2_chain_ref(chain);
402         hmp = chain->hmp;
403         loops = 0;
404
405         for (;;) {
406                 /*
407                  * Move hmp->flushq to info.flushq if non-empty so it can
408                  * be processed.
409                  */
410                 if (TAILQ_FIRST(&hmp->flushq) != NULL) {
411                         hammer2_spin_ex(&chain->hmp->list_spin);
412                         TAILQ_CONCAT(&info.flushq, &hmp->flushq, flush_node);
413                         hammer2_spin_unex(&chain->hmp->list_spin);
414                 }
415
416                 /*
417                  * Unwind deep recursions which had been deferred.  This
418                  * can leave the FLUSH_* bits set for these chains, which
419                  * will be handled when we [re]flush chain after the unwind.
420                  */
421                 while ((scan = TAILQ_FIRST(&info.flushq)) != NULL) {
422                         KKASSERT(scan->flags & HAMMER2_CHAIN_DEFERRED);
423                         TAILQ_REMOVE(&info.flushq, scan, flush_node);
424 #ifdef HAMMER2_SCAN_DEBUG
425                         ++info.flushq_count;
426 #endif
427                         atomic_clear_int(&scan->flags, HAMMER2_CHAIN_DEFERRED |
428                                                        HAMMER2_CHAIN_DELAYED);
429
430                         /*
431                          * Now that we've popped back up we can do a secondary
432                          * recursion on the deferred elements.
433                          *
434                          * NOTE: hmp->flushq chains (marked DESTROY) must be
435                          *       handled unconditionally so they can be cleaned
436                          *       out.
437                          *
438                          * NOTE: hammer2_flush() may replace scan.
439                          */
440                         if (hammer2_debug & 0x0040)
441                                 kprintf("deferred flush %p\n", scan);
442                         hammer2_chain_lock(scan, HAMMER2_RESOLVE_MAYBE);
443                         if (scan->error == 0) {
444                                 if (scan->flags & HAMMER2_CHAIN_DESTROY) {
445                                         hammer2_flush(scan,
446                                                     flags |
447                                                     HAMMER2_FLUSH_TOP |
448                                                     HAMMER2_FLUSH_ALL);
449                                 } else {
450                                         hammer2_flush(scan,
451                                                     flags & ~HAMMER2_FLUSH_TOP);
452                                 }
453                         } else {
454                                 info.error |= scan->error;
455                         }
456                         hammer2_chain_unlock(scan);
457                         hammer2_chain_drop(scan);/* ref from defer */
458                 }
459
460                 /*
461                  * [re]flush chain as the deep recursion may have generated
462                  * additional modifications.
463                  */
464                 info.diddeferral = 0;
465                 if (info.parent != chain->parent) {
466                         if (hammer2_debug & 0x0040) {
467                                 kprintf("LOST CHILD4 %p->%p "
468                                         "(actual parent %p)\n",
469                                         info.parent, chain, chain->parent);
470                         }
471                         hammer2_chain_drop(info.parent);
472                         info.parent = chain->parent;
473                         hammer2_chain_ref(info.parent);
474                 }
475                 hammer2_flush_core(&info, chain, flags);
476
477                 /*
478                  * Only loop if deep recursions have been deferred.
479                  */
480                 if (TAILQ_EMPTY(&info.flushq))
481                         break;
482
483                 if (++loops % 1000 == 0) {
484                         kprintf("hammer2_flush: excessive loops on %p\n",
485                                 chain);
486                         if (hammer2_debug & 0x100000)
487                                 Debugger("hell4");
488                 }
489         }
490 #ifdef HAMMER2_SCAN_DEBUG
491         if (info.scan_count >= 10)
492         kprintf("hammer2_flush: scan_count %ld (%ld,%ld,%ld,%ld) "
493                 "bt(%ld,%ld,%ld,%ld,%ld,%ld) flushq %ld\n",
494                 info.scan_count,
495                 info.scan_mod_count,
496                 info.scan_upd_count,
497                 info.scan_onf_count,
498                 info.scan_del_count,
499                 info.scan_btype[1],
500                 info.scan_btype[2],
501                 info.scan_btype[3],
502                 info.scan_btype[4],
503                 info.scan_btype[5],
504                 info.scan_btype[6],
505                 info.flushq_count);
506 #endif
507         hammer2_chain_drop(chain);
508         if (info.parent)
509                 hammer2_chain_drop(info.parent);
510         return (info.error);
511 }
512
513 /*
514  * This is the core of the chain flushing code.  The chain is locked by the
515  * caller and must also have an extra ref on it by the caller, and remains
516  * locked and will have an extra ref on return.  info.parent is referenced
517  * but not locked.
518  *
519  * Upon return, the caller can test the UPDATE bit on the chain to determine
520  * if the parent needs updating.
521  *
522  * (1) Determine if this node is a candidate for the flush, return if it is
523  *     not.  fchain and vchain are always candidates for the flush.
524  *
525  * (2) If we recurse too deep the chain is entered onto the deferral list and
526  *     the current flush stack is aborted until after the deferral list is
527  *     run.
528  *
529  * (3) Recursively flush live children (rbtree).  This can create deferrals.
530  *     A successful flush clears the MODIFIED and UPDATE bits on the children
531  *     and typically causes the parent to be marked MODIFIED as the children
532  *     update the parent's block table.  A parent might already be marked
533  *     MODIFIED due to a deletion (whos blocktable update in the parent is
534  *     handled by the frontend), or if the parent itself is modified by the
535  *     frontend for other reasons.
536  *
537  * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
538  *     Deleted-but-open inodes can still be individually flushed via the
539  *     filesystem syncer.
540  *
541  * (5) Delete parents on the way back up if they are normal indirect blocks
542  *     and have no children.
543  *
544  * (6) Note that an unmodified child may still need the block table in its
545  *     parent updated (e.g. rename/move).  The child will have UPDATE set
546  *     in this case.
547  *
548  *                      WARNING ON BREF MODIFY_TID/MIRROR_TID
549  *
550  * blockref.modify_tid is consistent only within a PFS, and will not be
551  * consistent during synchronization.  mirror_tid is consistent across the
552  * block device regardless of the PFS.
553  */
554 static void
555 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
556                    int flags)
557 {
558         hammer2_chain_t *parent;
559         hammer2_dev_t *hmp;
560         int save_error;
561
562         /*
563          * (1) Optimize downward recursion to locate nodes needing action.
564          *     Nothing to do if none of these flags are set.
565          */
566         if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) {
567                 if (hammer2_debug & 0x200) {
568                         if (info->debug == NULL)
569                                 info->debug = chain;
570                 } else {
571                         return;
572                 }
573         }
574
575         hmp = chain->hmp;
576
577         /*
578          * NOTE: parent can be NULL, usually due to destroy races.
579          */
580         parent = info->parent;
581         KKASSERT(chain->parent == parent);
582
583         /*
584          * Downward search recursion
585          *
586          * We must be careful on cold stops.  If CHAIN_UPDATE is set and
587          * we stop cold (verses a deferral which will re-run the chain later),
588          * the update can wind up never being applied.  This situation most
589          * typically occurs on inode boundaries due to the way
590          * hammer2_vfs_sync() breaks-up the flush.  As a safety, we
591          * flush-through such situations.
592          */
593         if (chain->flags & (HAMMER2_CHAIN_DEFERRED | HAMMER2_CHAIN_DELAYED)) {
594                 /*
595                  * Already deferred.
596                  */
597                 ++info->diddeferral;
598         } else if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) &&
599                    (chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
600                    (flags & HAMMER2_FLUSH_ALL) == 0 &&
601                    (flags & HAMMER2_FLUSH_TOP) == 0 &&
602                    chain->pmp && chain->pmp->mp) {
603                 /*
604                  * If FLUSH_ALL is not specified the caller does not want
605                  * to recurse through PFS roots that have been mounted.
606                  *
607                  * (If the PFS has not been mounted there may not be
608                  *  anything monitoring its chains and its up to us
609                  *  to flush it).
610                  *
611                  * The typical sequence is to flush dirty PFS's starting at
612                  * their root downward, then flush the device root (vchain).
613                  * It is this second flush that typically leaves out the
614                  * ALL flag.
615                  *
616                  * However we must still process the PFSROOT chains for block
617                  * table updates in their parent (which IS part of our flush).
618                  *
619                  * NOTE: The volume root, vchain, does not set PFSBOUNDARY.
620                  *
621                  * NOTE: This test must be done before the depth-limit test,
622                  *       else it might become the top on a flushq iteration.
623                  *
624                  * NOTE: We must re-set ONFLUSH in the parent to retain if
625                  *       this chain (that we are skipping) requires work.
626                  */
627                 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
628                                     HAMMER2_CHAIN_DESTROY |
629                                     HAMMER2_CHAIN_MODIFIED)) {
630                         hammer2_chain_setflush(parent);
631                 }
632         } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
633                    (chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
634                    (flags & HAMMER2_FLUSH_INODE_STOP) &&
635                    (flags & HAMMER2_FLUSH_ALL) == 0 &&
636                    (flags & HAMMER2_FLUSH_TOP) == 0 &&
637                    chain->pmp && chain->pmp->mp) {
638                 /*
639                  * If FLUSH_INODE_STOP is specified and both ALL and TOP
640                  * are clear, we must not flush the chain.  The chain should
641                  * have already been flushed and any further ONFLUSH/UPDATE
642                  * setting will be related to the next flush.
643                  *
644                  * This features allows us to flush inodes independently of
645                  * each other and meta-data above the inodes separately.
646                  */
647                 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
648                                     HAMMER2_CHAIN_DESTROY |
649                                     HAMMER2_CHAIN_MODIFIED)) {
650                         if (parent)
651                                 hammer2_chain_setflush(parent);
652                 }
653         } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
654                 /*
655                  * Recursion depth reached.
656                  */
657                 KKASSERT((chain->flags & HAMMER2_CHAIN_DELAYED) == 0);
658                 hammer2_chain_ref(chain);
659                 TAILQ_INSERT_TAIL(&info->flushq, chain, flush_node);
660                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEFERRED);
661                 ++info->diddeferral;
662         } else if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
663                                    HAMMER2_CHAIN_DESTROY)) {
664                 /*
665                  * Downward recursion search (actual flush occurs bottom-up).
666                  * pre-clear ONFLUSH.  It can get set again due to races or
667                  * flush errors, which we want so the scan finds us again in
668                  * the next flush.
669                  *
670                  * We must also recurse if DESTROY is set so we can finally
671                  * get rid of the related children, otherwise the node will
672                  * just get re-flushed on lastdrop.
673                  *
674                  * WARNING!  The recursion will unlock/relock info->parent
675                  *           (which is 'chain'), potentially allowing it
676                  *           to be ripped up.
677                  */
678                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
679                 save_error = info->error;
680                 info->error = 0;
681                 info->parent = chain;
682
683                 /*
684                  * We may have to do this twice to catch any indirect
685                  * block maintenance that occurs.  Other conditions which
686                  * can keep setting ONFLUSH (such as deferrals) ought to
687                  * be handled by the flushq code.  XXX needs more help
688                  */
689                 hammer2_spin_ex(&chain->core.spin);
690                 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
691                         NULL, hammer2_flush_recurse, info);
692                 if (chain->flags & HAMMER2_CHAIN_ONFLUSH) {
693                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
694                         RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
695                                 NULL, hammer2_flush_recurse, info);
696                 }
697                 hammer2_spin_unex(&chain->core.spin);
698                 info->parent = parent;
699
700                 /*
701                  * Re-set the flush bits if the flush was incomplete or
702                  * an error occurred.  If an error occurs it is typically
703                  * an allocation error.  Errors do not cause deferrals.
704                  */
705                 if (info->error)
706                         hammer2_chain_setflush(chain);
707                 info->error |= save_error;
708                 if (info->diddeferral)
709                         hammer2_chain_setflush(chain);
710
711                 /*
712                  * If we lost the parent->chain association we have to
713                  * stop processing this chain because it is no longer
714                  * in this recursion.  If it moved, it will be handled
715                  * by the ONFLUSH flag elsewhere.
716                  */
717                 if (chain->parent != parent) {
718                         kprintf("LOST CHILD2 %p->%p (actual parent %p)\n",
719                                 parent, chain, chain->parent);
720                         goto done;
721                 }
722         }
723
724         /*
725          * Now we are in the bottom-up part of the recursion.
726          *
727          * Do not update chain if lower layers were deferred.  We continue
728          * to try to update the chain on lower-level errors, but the flush
729          * code may decide not to flush the volume root.
730          *
731          * XXX should we continue to try to update the chain if an error
732          *     occurred?
733          */
734         if (info->diddeferral)
735                 goto done;
736
737         /*
738          * Both parent and chain must be locked in order to flush chain,
739          * in order to properly update the parent under certain conditions.
740          *
741          * In addition, we can't safely unlock/relock the chain once we
742          * start flushing the chain itself, which we would have to do later
743          * on in order to lock the parent if we didn't do that now.
744          */
745         hammer2_chain_ref_hold(chain);
746         hammer2_chain_unlock(chain);
747         if (parent)
748                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
749         hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
750         hammer2_chain_drop_unhold(chain);
751
752         /*
753          * Can't process if we can't access their content.
754          */
755         if ((parent && parent->error) || chain->error) {
756                 kprintf("hammer2: chain error during flush\n");
757                 info->error |= chain->error;
758                 if (parent) {
759                         info->error |= parent->error;
760                         hammer2_chain_unlock(parent);
761                 }
762                 goto done;
763         }
764
765         if (chain->parent != parent) {
766                 if (hammer2_debug & 0x0040) {
767                         kprintf("LOST CHILD3 %p->%p (actual parent %p)\n",
768                                 parent, chain, chain->parent);
769                 }
770                 KKASSERT(parent != NULL);
771                 hammer2_chain_unlock(parent);
772                 if ((chain->flags & HAMMER2_CHAIN_DELAYED) == 0) {
773                         hammer2_chain_ref(chain);
774                         TAILQ_INSERT_TAIL(&info->flushq, chain, flush_node);
775                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEFERRED);
776                         ++info->diddeferral;
777                 }
778                 goto done;
779         }
780
781         /*
782          * Propagate the DESTROY flag downwards.  This dummies up the flush
783          * code and tries to invalidate related buffer cache buffers to
784          * avoid the disk write.
785          */
786         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
787                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
788
789         /*
790          * Dispose of the modified bit.
791          *
792          * If parent is present, the UPDATE bit should already be set.
793          * UPDATE should already be set.
794          * bref.mirror_tid should already be set.
795          */
796         if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
797                 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
798                          chain->parent == NULL);
799                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
800                 atomic_add_long(&hammer2_count_modified_chains, -1);
801
802                 /*
803                  * Manage threads waiting for excessive dirty memory to
804                  * be retired.
805                  */
806                 if (chain->pmp)
807                         hammer2_pfs_memory_wakeup(chain->pmp);
808
809 #if 0
810                 if ((chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
811                     chain != &hmp->vchain &&
812                     chain != &hmp->fchain) {
813                         /*
814                          * Set UPDATE bit indicating that the parent block
815                          * table requires updating.
816                          */
817                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
818                 }
819 #endif
820
821                 /*
822                  * Issue the flush.  This is indirect via the DIO.
823                  *
824                  * NOTE: A DELETED node that reaches this point must be
825                  *       flushed for synchronization point consistency.
826                  *
827                  * NOTE: Even though MODIFIED was already set, the related DIO
828                  *       might not be dirty due to a system buffer cache
829                  *       flush and must be set dirty if we are going to make
830                  *       further modifications to the buffer.  Chains with
831                  *       embedded data don't need this.
832                  */
833                 if (hammer2_debug & 0x1000) {
834                         kprintf("Flush %p.%d %016jx/%d data=%016jx\n",
835                                 chain, chain->bref.type,
836                                 (uintmax_t)chain->bref.key,
837                                 chain->bref.keybits,
838                                 (uintmax_t)chain->bref.data_off);
839                 }
840                 if (hammer2_debug & 0x2000) {
841                         Debugger("Flush hell");
842                 }
843
844                 /*
845                  * Update chain CRCs for flush.
846                  *
847                  * NOTE: Volume headers are NOT flushed here as they require
848                  *       special processing.
849                  */
850                 switch(chain->bref.type) {
851                 case HAMMER2_BREF_TYPE_FREEMAP:
852                         /*
853                          * Update the volume header's freemap_tid to the
854                          * freemap's flushing mirror_tid.
855                          *
856                          * (note: embedded data, do not call setdirty)
857                          */
858                         KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
859                         KKASSERT(chain == &hmp->fchain);
860                         hmp->voldata.freemap_tid = chain->bref.mirror_tid;
861                         if (hammer2_debug & 0x8000) {
862                                 /* debug only, avoid syslogd loop */
863                                 kprintf("sync freemap mirror_tid %08jx\n",
864                                         (intmax_t)chain->bref.mirror_tid);
865                         }
866
867                         /*
868                          * The freemap can be flushed independently of the
869                          * main topology, but for the case where it is
870                          * flushed in the same transaction, and flushed
871                          * before vchain (a case we want to allow for
872                          * performance reasons), make sure modifications
873                          * made during the flush under vchain use a new
874                          * transaction id.
875                          *
876                          * Otherwise the mount recovery code will get confused.
877                          */
878                         ++hmp->voldata.mirror_tid;
879                         break;
880                 case HAMMER2_BREF_TYPE_VOLUME:
881                         /*
882                          * The free block table is flushed by
883                          * hammer2_vfs_sync() before it flushes vchain.
884                          * We must still hold fchain locked while copying
885                          * voldata to volsync, however.
886                          *
887                          * These do not error per-say since their data does
888                          * not need to be re-read from media on lock.
889                          *
890                          * (note: embedded data, do not call setdirty)
891                          */
892                         hammer2_chain_lock(&hmp->fchain,
893                                            HAMMER2_RESOLVE_ALWAYS);
894                         hammer2_voldata_lock(hmp);
895                         if (hammer2_debug & 0x8000) {
896                                 /* debug only, avoid syslogd loop */
897                                 kprintf("sync volume  mirror_tid %08jx\n",
898                                         (intmax_t)chain->bref.mirror_tid);
899                         }
900
901                         /*
902                          * Update the volume header's mirror_tid to the
903                          * main topology's flushing mirror_tid.  It is
904                          * possible that voldata.mirror_tid is already
905                          * beyond bref.mirror_tid due to the bump we made
906                          * above in BREF_TYPE_FREEMAP.
907                          */
908                         if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
909                                 hmp->voldata.mirror_tid =
910                                         chain->bref.mirror_tid;
911                         }
912
913                         /*
914                          * The volume header is flushed manually by the
915                          * syncer, not here.  All we do here is adjust the
916                          * crc's.
917                          */
918                         KKASSERT(chain->data != NULL);
919                         KKASSERT(chain->dio == NULL);
920
921                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
922                                 hammer2_icrc32(
923                                         (char *)&hmp->voldata +
924                                          HAMMER2_VOLUME_ICRC1_OFF,
925                                         HAMMER2_VOLUME_ICRC1_SIZE);
926                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
927                                 hammer2_icrc32(
928                                         (char *)&hmp->voldata +
929                                          HAMMER2_VOLUME_ICRC0_OFF,
930                                         HAMMER2_VOLUME_ICRC0_SIZE);
931                         hmp->voldata.icrc_volheader =
932                                 hammer2_icrc32(
933                                         (char *)&hmp->voldata +
934                                          HAMMER2_VOLUME_ICRCVH_OFF,
935                                         HAMMER2_VOLUME_ICRCVH_SIZE);
936
937                         if (hammer2_debug & 0x8000) {
938                                 /* debug only, avoid syslogd loop */
939                                 kprintf("syncvolhdr %016jx %016jx\n",
940                                         hmp->voldata.mirror_tid,
941                                         hmp->vchain.bref.mirror_tid);
942                         }
943                         hmp->volsync = hmp->voldata;
944                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
945                         hammer2_voldata_unlock(hmp);
946                         hammer2_chain_unlock(&hmp->fchain);
947                         break;
948                 case HAMMER2_BREF_TYPE_DATA:
949                         /*
950                          * Data elements have already been flushed via the
951                          * logical file buffer cache.  Their hash was set in
952                          * the bref by the vop_write code.  Do not re-dirty.
953                          *
954                          * Make sure any device buffer(s) have been flushed
955                          * out here (there aren't usually any to flush) XXX.
956                          */
957                         break;
958                 case HAMMER2_BREF_TYPE_INDIRECT:
959                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
960                 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
961                         /*
962                          * Buffer I/O will be cleaned up when the volume is
963                          * flushed (but the kernel is free to flush it before
964                          * then, as well).
965                          */
966                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
967                         hammer2_chain_setcheck(chain, chain->data);
968                         break;
969                 case HAMMER2_BREF_TYPE_DIRENT:
970                         /*
971                          * A directory entry can use the check area to store
972                          * the filename for filenames <= 64 bytes, don't blow
973                          * it up!
974                          */
975                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
976                         if (chain->bytes)
977                                 hammer2_chain_setcheck(chain, chain->data);
978                         break;
979                 case HAMMER2_BREF_TYPE_INODE:
980                         /*
981                          * NOTE: We must call io_setdirty() to make any late
982                          *       changes to the inode data, the system might
983                          *       have already flushed the buffer.
984                          */
985                         if (chain->data->ipdata.meta.op_flags &
986                             HAMMER2_OPFLAG_PFSROOT) {
987                                 /*
988                                  * non-NULL pmp if mounted as a PFS.  We must
989                                  * sync fields cached in the pmp? XXX
990                                  */
991                                 hammer2_inode_data_t *ipdata;
992
993                                 hammer2_io_setdirty(chain->dio);
994                                 ipdata = &chain->data->ipdata;
995                                 if (chain->pmp) {
996                                         ipdata->meta.pfs_inum =
997                                                 chain->pmp->inode_tid;
998                                 }
999                         } else {
1000                                 /* can't be mounted as a PFS */
1001                         }
1002
1003                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
1004                         hammer2_chain_setcheck(chain, chain->data);
1005
1006                                 hammer2_inode_data_t *ipdata;
1007                         ipdata = &chain->data->ipdata;
1008                         break;
1009                 default:
1010                         KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
1011                         panic("hammer2_flush_core: unsupported "
1012                               "embedded bref %d",
1013                               chain->bref.type);
1014                         /* NOT REACHED */
1015                 }
1016
1017                 /*
1018                  * If the chain was destroyed try to avoid unnecessary I/O
1019                  * that might not have yet occurred.  Remove the data range
1020                  * from dedup candidacy and attempt to invalidation that
1021                  * potentially dirty portion of the I/O buffer.
1022                  */
1023                 if (chain->flags & HAMMER2_CHAIN_DESTROY) {
1024                         hammer2_io_dedup_delete(hmp,
1025                                                 chain->bref.type,
1026                                                 chain->bref.data_off,
1027                                                 chain->bytes);
1028 #if 0
1029                         hammer2_io_t *dio;
1030                         if (chain->dio) {
1031                                 hammer2_io_inval(chain->dio,
1032                                                  chain->bref.data_off,
1033                                                  chain->bytes);
1034                         } else if ((dio = hammer2_io_getquick(hmp,
1035                                                   chain->bref.data_off,
1036                                                   chain->bytes,
1037                                                   1)) != NULL) {
1038                                 hammer2_io_inval(dio,
1039                                                  chain->bref.data_off,
1040                                                  chain->bytes);
1041                                 hammer2_io_putblk(&dio);
1042                         }
1043 #endif
1044                 }
1045         }
1046
1047         /*
1048          * If UPDATE is set the parent block table may need to be updated.
1049          * This can fail if the hammer2_chain_modify() fails.
1050          *
1051          * NOTE: UPDATE may be set on vchain or fchain in which case
1052          *       parent could be NULL.  It's easiest to allow the case
1053          *       and test for NULL.  parent can also wind up being NULL
1054          *       due to a deletion so we need to handle the case anyway.
1055          *
1056          * If no parent exists we can just clear the UPDATE bit.  If the
1057          * chain gets reattached later on the bit will simply get set
1058          * again.
1059          */
1060         if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL)
1061                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1062
1063         /*
1064          * The chain may need its blockrefs updated in the parent.
1065          */
1066         if (chain->flags & HAMMER2_CHAIN_UPDATE) {
1067                 hammer2_blockref_t *base;
1068                 int count;
1069
1070                 /*
1071                  * Clear UPDATE flag, mark parent modified, update its
1072                  * modify_tid if necessary, and adjust the parent blockmap.
1073                  */
1074                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1075
1076                 /*
1077                  * (optional code)
1078                  *
1079                  * Avoid actually modifying and updating the parent if it
1080                  * was flagged for destruction.  This can greatly reduce
1081                  * disk I/O in large tree removals because the
1082                  * hammer2_io_setinval() call in the upward recursion
1083                  * (see MODIFIED code above) can only handle a few cases.
1084                  */
1085                 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
1086                         if (parent->bref.modify_tid < chain->bref.modify_tid) {
1087                                 parent->bref.modify_tid =
1088                                         chain->bref.modify_tid;
1089                         }
1090                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED |
1091                                                         HAMMER2_CHAIN_BMAPUPD);
1092                         goto skipupdate;
1093                 }
1094
1095                 /*
1096                  * The flusher is responsible for deleting empty indirect
1097                  * blocks at this point.  If we don't do this, no major harm
1098                  * will be done but the empty indirect blocks will stay in
1099                  * the topology and make it a messy and inefficient.
1100                  *
1101                  * The flusher is also responsible for collapsing the
1102                  * content of an indirect block into its parent whenever
1103                  * possible (with some hysteresis).  Not doing this will also
1104                  * not harm the topology, but would make it messy and
1105                  * inefficient.
1106                  */
1107                 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT) {
1108                         if (hammer2_chain_indirect_maintenance(parent, chain))
1109                                 goto skipupdate;
1110                 }
1111
1112                 /*
1113                  * We are updating the parent's blockmap, the parent must
1114                  * be set modified.  If this fails we re-set the UPDATE flag
1115                  * in the child.
1116                  *
1117                  * NOTE! A modification error can be ENOSPC.  We still want
1118                  *       to flush modified chains recursively, not break out,
1119                  *       so we just skip the update in this situation and
1120                  *       continue.  That is, we still need to try to clean
1121                  *       out dirty chains and buffers.
1122                  *
1123                  *       This may not help bulkfree though. XXX
1124                  */
1125                 save_error = hammer2_chain_modify(parent, 0, 0, 0);
1126                 if (save_error) {
1127                         info->error |= save_error;
1128                         kprintf("hammer2_flush: %016jx.%02x error=%08x\n",
1129                                 parent->bref.data_off, parent->bref.type,
1130                                 save_error);
1131                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1132                         goto skipupdate;
1133                 }
1134                 if (parent->bref.modify_tid < chain->bref.modify_tid)
1135                         parent->bref.modify_tid = chain->bref.modify_tid;
1136
1137                 /*
1138                  * Calculate blockmap pointer
1139                  */
1140                 switch(parent->bref.type) {
1141                 case HAMMER2_BREF_TYPE_INODE:
1142                         /*
1143                          * Access the inode's block array.  However, there is
1144                          * no block array if the inode is flagged DIRECTDATA.
1145                          */
1146                         if (parent->data &&
1147                             (parent->data->ipdata.meta.op_flags &
1148                              HAMMER2_OPFLAG_DIRECTDATA) == 0) {
1149                                 base = &parent->data->
1150                                         ipdata.u.blockset.blockref[0];
1151                         } else {
1152                                 base = NULL;
1153                         }
1154                         count = HAMMER2_SET_COUNT;
1155                         break;
1156                 case HAMMER2_BREF_TYPE_INDIRECT:
1157                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1158                         if (parent->data)
1159                                 base = &parent->data->npdata[0];
1160                         else
1161                                 base = NULL;
1162                         count = parent->bytes / sizeof(hammer2_blockref_t);
1163                         break;
1164                 case HAMMER2_BREF_TYPE_VOLUME:
1165                         base = &chain->hmp->voldata.sroot_blockset.blockref[0];
1166                         count = HAMMER2_SET_COUNT;
1167                         break;
1168                 case HAMMER2_BREF_TYPE_FREEMAP:
1169                         base = &parent->data->npdata[0];
1170                         count = HAMMER2_SET_COUNT;
1171                         break;
1172                 default:
1173                         base = NULL;
1174                         count = 0;
1175                         panic("hammer2_flush_core: "
1176                               "unrecognized blockref type: %d",
1177                               parent->bref.type);
1178                 }
1179
1180                 /*
1181                  * Blocktable updates
1182                  *
1183                  * We synchronize pending statistics at this time.  Delta
1184                  * adjustments designated for the current and upper level
1185                  * are synchronized.
1186                  */
1187                 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) {
1188                         if (chain->flags & HAMMER2_CHAIN_BMAPPED) {
1189                                 hammer2_spin_ex(&parent->core.spin);
1190                                 hammer2_base_delete(parent, base, count, chain);
1191                                 hammer2_spin_unex(&parent->core.spin);
1192                                 /* base_delete clears both bits */
1193                         } else {
1194                                 atomic_clear_int(&chain->flags,
1195                                                  HAMMER2_CHAIN_BMAPUPD);
1196                         }
1197                 }
1198                 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) {
1199                         hammer2_spin_ex(&parent->core.spin);
1200                         hammer2_base_insert(parent, base, count,
1201                                             chain, &chain->bref);
1202                         hammer2_spin_unex(&parent->core.spin);
1203                         /* base_insert sets BMAPPED */
1204                 }
1205         }
1206 skipupdate:
1207         if (parent)
1208                 hammer2_chain_unlock(parent);
1209
1210         /*
1211          * Final cleanup after flush
1212          */
1213 done:
1214         KKASSERT(chain->refs > 0);
1215         if (hammer2_debug & 0x200) {
1216                 if (info->debug == chain)
1217                         info->debug = NULL;
1218         }
1219 }
1220
1221 /*
1222  * Flush recursion helper, called from flush_core, calls flush_core.
1223  *
1224  * Flushes the children of the caller's chain (info->parent), restricted
1225  * by sync_tid.  Set info->domodify if the child's blockref must propagate
1226  * back up to the parent.
1227  *
1228  * This function may set info->error as a side effect.
1229  *
1230  * Ripouts can move child from rbtree to dbtree or dbq but the caller's
1231  * flush scan order prevents any chains from being lost.  A child can be
1232  * executes more than once.
1233  *
1234  * WARNING! If we do not call hammer2_flush_core() we must update
1235  *          bref.mirror_tid ourselves to indicate that the flush has
1236  *          processed the child.
1237  *
1238  * WARNING! parent->core spinlock is held on entry and return.
1239  */
1240 static int
1241 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
1242 {
1243         hammer2_flush_info_t *info = data;
1244         hammer2_chain_t *parent = info->parent;
1245
1246 #ifdef HAMMER2_SCAN_DEBUG
1247         ++info->scan_count;
1248         if (child->flags & HAMMER2_CHAIN_MODIFIED)
1249                 ++info->scan_mod_count;
1250         if (child->flags & HAMMER2_CHAIN_UPDATE)
1251                 ++info->scan_upd_count;
1252         if (child->flags & HAMMER2_CHAIN_ONFLUSH)
1253                 ++info->scan_onf_count;
1254 #endif
1255
1256         /*
1257          * (child can never be fchain or vchain so a special check isn't
1258          *  needed).
1259          *
1260          * We must ref the child before unlocking the spinlock.
1261          *
1262          * The caller has added a ref to the parent so we can temporarily
1263          * unlock it in order to lock the child.  However, if it no longer
1264          * winds up being the child of the parent we must skip this child.
1265          *
1266          * NOTE! chain locking errors are fatal.  They are never out-of-space
1267          *       errors.
1268          */
1269         hammer2_chain_ref(child);
1270         hammer2_spin_unex(&parent->core.spin);
1271
1272         hammer2_chain_ref_hold(parent);
1273         hammer2_chain_unlock(parent);
1274         hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
1275         if (child->parent != parent) {
1276                 kprintf("LOST CHILD1 %p->%p (actual parent %p)\n",
1277                         parent, child, child->parent);
1278                 goto done;
1279         }
1280         if (child->error) {
1281                 kprintf("CHILD ERROR DURING FLUSH LOCK %p->%p\n",
1282                         parent, child);
1283                 info->error |= child->error;
1284                 goto done;
1285         }
1286
1287         /*
1288          * Must propagate the DESTROY flag downwards, otherwise the
1289          * parent could end up never being removed because it will
1290          * be requeued to the flusher if it survives this run due to
1291          * the flag.
1292          */
1293         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
1294                 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROY);
1295 #ifdef HAMMER2_SCAN_DEBUG
1296         if (child->flags & HAMMER2_CHAIN_DESTROY)
1297                 ++info->scan_del_count;
1298 #endif
1299
1300         /*
1301          * Recurse and collect deferral data.  We're in the media flush,
1302          * this can cross PFS boundaries.
1303          */
1304         if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1305 #ifdef HAMMER2_SCAN_DEBUG
1306                 if (child->bref.type < 7)
1307                         ++info->scan_btype[child->bref.type];
1308 #endif
1309                 ++info->depth;
1310                 hammer2_flush_core(info, child, info->flags);
1311                 --info->depth;
1312         } else if (hammer2_debug & 0x200) {
1313                 if (info->debug == NULL)
1314                         info->debug = child;
1315                 ++info->depth;
1316                 hammer2_flush_core(info, child, info->flags);
1317                 --info->depth;
1318                 if (info->debug == child)
1319                         info->debug = NULL;
1320         }
1321
1322 done:
1323         /*
1324          * Relock to continue the loop.
1325          */
1326         hammer2_chain_unlock(child);
1327         hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1328         hammer2_chain_drop_unhold(parent);
1329         if (parent->error) {
1330                 kprintf("PARENT ERROR DURING FLUSH LOCK %p->%p\n",
1331                         parent, child);
1332                 info->error |= parent->error;
1333         }
1334         hammer2_chain_drop(child);
1335         KKASSERT(info->parent == parent);
1336         hammer2_spin_ex(&parent->core.spin);
1337
1338         return (0);
1339 }
1340
1341 /*
1342  * flush helper (backend threaded)
1343  *
1344  * Flushes chain topology for the specified inode.
1345  *
1346  * If HAMMER2_XOP_FLUSH is set we flush all chains from the current inode
1347  * through but stop at sub-inodes (we flush the inode chains for sub-inodes,
1348  * but do not go further as deeper modifications do not belong to the current
1349  * flush cycle).
1350  *
1351  * If HAMMER2_XOP_FLUSH is not set we flush the current inode's chains only
1352  * and do not recurse through sub-inodes, including not including those
1353  * sub-inodes.
1354  *
1355  * Remember that HAMMER2 is currently using a flat inode model, so directory
1356  * hierarchies do not translate to inode hierarchies.  PFS ROOTs, however,
1357  * do.
1358  *
1359  * chain->parent can be NULL, usually due to destroy races.
1360  *
1361  * Primarily called from vfs_sync().
1362  */
1363 void
1364 hammer2_inode_xop_flush(hammer2_thread_t *thr, hammer2_xop_t *arg)
1365 {
1366         hammer2_xop_flush_t *xop = &arg->xop_flush;
1367         hammer2_chain_t *chain;
1368         hammer2_chain_t *parent;
1369         hammer2_dev_t *hmp;
1370         int flush_error = 0;
1371         int fsync_error = 0;
1372         int total_error = 0;
1373         int j;
1374         int xflags;
1375         int ispfsroot = 0;
1376
1377         xflags = HAMMER2_FLUSH_TOP;
1378         if (xop->head.flags & HAMMER2_XOP_INODE_STOP)
1379                 xflags |= HAMMER2_FLUSH_INODE_STOP;
1380
1381         /*
1382          * Flush core chains
1383          */
1384         chain = hammer2_inode_chain(xop->head.ip1, thr->clindex,
1385                                     HAMMER2_RESOLVE_ALWAYS);
1386         if (chain) {
1387                 hmp = chain->hmp;
1388                 if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) ||
1389                     TAILQ_FIRST(&hmp->flushq) != NULL) {
1390                         hammer2_flush(chain, xflags);
1391                         parent = chain->parent;
1392                         if (parent)
1393                                 hammer2_chain_setflush(parent);
1394                 }
1395                 if (chain->flags & HAMMER2_CHAIN_PFSBOUNDARY)
1396                         ispfsroot = 1;
1397                 hammer2_chain_unlock(chain);
1398                 hammer2_chain_drop(chain);
1399                 chain = NULL;
1400         } else {
1401                 hmp = NULL;
1402         }
1403
1404         /*
1405          * Only flush the volume header if asked to, plus the inode must also
1406          * be the PFS root.
1407          */
1408         if ((xop->head.flags & HAMMER2_XOP_VOLHDR) == 0)
1409                 goto skip;
1410         if (ispfsroot == 0)
1411                 goto skip;
1412
1413         /*
1414          * Flush volume roots.  Avoid replication, we only want to
1415          * flush each hammer2_dev (hmp) once.
1416          */
1417         for (j = thr->clindex - 1; j >= 0; --j) {
1418                 if ((chain = xop->head.ip1->cluster.array[j].chain) != NULL) {
1419                         if (chain->hmp == hmp) {
1420                                 chain = NULL;   /* safety */
1421                                 goto skip;
1422                         }
1423                 }
1424         }
1425         chain = NULL;   /* safety */
1426
1427         /*
1428          * spmp transaction.  The super-root is never directly mounted so
1429          * there shouldn't be any vnodes, let alone any dirty vnodes
1430          * associated with it, so we shouldn't have to mess around with any
1431          * vnode flushes here.
1432          */
1433         hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1434
1435         /*
1436          * Media mounts have two 'roots', vchain for the topology
1437          * and fchain for the free block table.  Flush both.
1438          *
1439          * Note that the topology and free block table are handled
1440          * independently, so the free block table can wind up being
1441          * ahead of the topology.  We depend on the bulk free scan
1442          * code to deal with any loose ends.
1443          *
1444          * vchain and fchain do not error on-lock since their data does
1445          * not have to be re-read from media.
1446          */
1447         hammer2_chain_ref(&hmp->vchain);
1448         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1449         hammer2_chain_ref(&hmp->fchain);
1450         hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1451         if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1452                 /*
1453                  * This will also modify vchain as a side effect,
1454                  * mark vchain as modified now.
1455                  */
1456                 hammer2_voldata_modify(hmp);
1457                 chain = &hmp->fchain;
1458                 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1459                 KKASSERT(chain == &hmp->fchain);
1460         }
1461         hammer2_chain_unlock(&hmp->fchain);
1462         hammer2_chain_unlock(&hmp->vchain);
1463         hammer2_chain_drop(&hmp->fchain);
1464         /* vchain dropped down below */
1465
1466         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1467         if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1468                 chain = &hmp->vchain;
1469                 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1470                 KKASSERT(chain == &hmp->vchain);
1471         }
1472         hammer2_chain_unlock(&hmp->vchain);
1473         hammer2_chain_drop(&hmp->vchain);
1474
1475         /*
1476          * We can't safely flush the volume header until we have
1477          * flushed any device buffers which have built up.
1478          *
1479          * XXX this isn't being incremental
1480          */
1481         vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
1482         fsync_error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
1483         vn_unlock(hmp->devvp);
1484         if (fsync_error || flush_error) {
1485                 kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n",
1486                         fsync_error, flush_error, hmp->devrepname);
1487         }
1488
1489         /*
1490          * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1491          * volume header needs synchronization via hmp->volsync.
1492          *
1493          * XXX synchronize the flag & data with only this flush XXX
1494          */
1495         if (fsync_error == 0 && flush_error == 0 &&
1496             (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1497                 struct buf *bp;
1498                 int vol_error = 0;
1499
1500                 /*
1501                  * Synchronize the disk before flushing the volume
1502                  * header.
1503                  */
1504                 bp = getpbuf(NULL);
1505                 bp->b_bio1.bio_offset = 0;
1506                 bp->b_bufsize = 0;
1507                 bp->b_bcount = 0;
1508                 bp->b_cmd = BUF_CMD_FLUSH;
1509                 bp->b_bio1.bio_done = biodone_sync;
1510                 bp->b_bio1.bio_flags |= BIO_SYNC;
1511                 vn_strategy(hmp->devvp, &bp->b_bio1);
1512                 fsync_error = biowait(&bp->b_bio1, "h2vol");
1513                 relpbuf(bp, NULL);
1514
1515                 /*
1516                  * Then we can safely flush the version of the
1517                  * volume header synchronized by the flush code.
1518                  */
1519                 j = hmp->volhdrno + 1;
1520                 if (j < 0)
1521                         j = 0;
1522                 if (j >= HAMMER2_NUM_VOLHDRS)
1523                         j = 0;
1524                 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1525                     hmp->volsync.volu_size) {
1526                         j = 0;
1527                 }
1528                 if (hammer2_debug & 0x8000) {
1529                         /* debug only, avoid syslogd loop */
1530                         kprintf("sync volhdr %d %jd\n",
1531                                 j, (intmax_t)hmp->volsync.volu_size);
1532                 }
1533                 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1534                             HAMMER2_PBUFSIZE, GETBLK_KVABIO, 0);
1535                 atomic_clear_int(&hmp->vchain.flags,
1536                                  HAMMER2_CHAIN_VOLUMESYNC);
1537                 bkvasync(bp);
1538                 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1539                 vol_error = bwrite(bp);
1540                 hmp->volhdrno = j;
1541                 if (vol_error)
1542                         fsync_error = vol_error;
1543         }
1544         if (flush_error)
1545                 total_error = flush_error;
1546         if (fsync_error)
1547                 total_error = hammer2_errno_to_error(fsync_error);
1548
1549         hammer2_trans_done(hmp->spmp);  /* spmp trans */
1550 skip:
1551         hammer2_xop_feed(&xop->head, NULL, thr->clindex, total_error);
1552 }