sys/vfs/hammer2/hammer2_flush.c

   1 /*
   2  * Copyright (c) 2011-2018 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@dragonflybsd.org>
   6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 /*
  36  *                      TRANSACTION AND FLUSH HANDLING
  37  *
  38  * Deceptively simple but actually fairly difficult to implement properly is
  39  * how I would describe it.
  40  *
  41  * Flushing generally occurs bottom-up but requires a top-down scan to
  42  * locate chains with MODIFIED and/or UPDATE bits set.  The ONFLUSH flag
  43  * tells how to recurse downward to find these chains.
  44  */
  45
  46 #include <sys/cdefs.h>
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/types.h>
  50 #include <sys/lock.h>
  51 #include <sys/uuid.h>
  52
  53 #include "hammer2.h"
  54
  55 #define FLUSH_DEBUG 0
  56
  57 #define HAMMER2_FLUSH_DEPTH_LIMIT       60      /* stack recursion limit */
  58
  59
  60 /*
  61  * Recursively flush the specified chain.  The chain is locked and
  62  * referenced by the caller and will remain so on return.  The chain
  63  * will remain referenced throughout but can temporarily lose its
  64  * lock during the recursion to avoid unnecessarily stalling user
  65  * processes.
  66  */
  67 struct hammer2_flush_info {
  68         hammer2_chain_t *parent;
  69         int             depth;
  70         int             error;                  /* cumulative error */
  71         int             flags;
  72 #ifdef HAMMER2_SCAN_DEBUG
  73         long            scan_count;
  74         long            scan_mod_count;
  75         long            scan_upd_count;
  76         long            scan_onf_count;
  77         long            scan_del_count;
  78         long            scan_btype[7];
  79 #endif
  80         hammer2_chain_t *debug;
  81 };
  82
  83 typedef struct hammer2_flush_info hammer2_flush_info_t;
  84
  85 static int hammer2_flush_core(hammer2_flush_info_t *info,
  86                                 hammer2_chain_t *chain, int flags);
  87 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
  88
  89 /*
  90  * Any per-pfs transaction initialization goes here.
  91  */
  92 void
  93 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
  94 {
  95 }
  96
  97 /*
  98  * Transaction support for any modifying operation.  Transactions are used
  99  * in the pmp layer by the frontend and in the spmp layer by the backend.
 100  *
 101  * 0                    - Normal transaction.  Interlocks against just the
 102  *                        COPYQ portion of an ISFLUSH transaction.
 103  *
 104  * TRANS_ISFLUSH        - Flush transaction.  Interlocks against other flush
 105  *                        transactions.
 106  *
 107  *                        When COPYQ is also specified, waits for the count
 108  *                        to drop to 1.
 109  *
 110  * TRANS_BUFCACHE       - Buffer cache transaction.  No interlock.
 111  *
 112  * TRANS_SIDEQ          - Run the sideq (only tested in trans_done())
 113  *
 114  * Initializing a new transaction allocates a transaction ID.  Typically
 115  * passed a pmp (hmp passed as NULL), indicating a cluster transaction.  Can
 116  * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
 117  * media target.  The latter mode is used by the recovery code.
 118  */
 119 void
 120 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
 121 {
 122         uint32_t oflags;
 123         uint32_t nflags;
 124         int dowait;
 125
 126         for (;;) {
 127                 oflags = pmp->trans.flags;
 128                 cpu_ccfence();
 129                 dowait = 0;
 130
 131                 if (flags & HAMMER2_TRANS_ISFLUSH) {
 132                         /*
 133                          * Interlock against other flush transactions.
 134                          */
 135                         if (oflags & HAMMER2_TRANS_ISFLUSH) {
 136                                 nflags = oflags | HAMMER2_TRANS_WAITING;
 137                                 dowait = 1;
 138                         } else {
 139                                 nflags = (oflags | flags) + 1;
 140                         }
 141                 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
 142                         /*
 143                          * Requesting strategy transaction from buffer-cache,
 144                          * or a VM getpages/putpages through the buffer cache.
 145                          * We must allow such transactions in all situations
 146                          * to avoid deadlocks.
 147                          */
 148                         nflags = (oflags | flags) + 1;
 149                 } else {
 150                         /*
 151                          * Normal transaction.  We do not interlock against
 152                          * BUFCACHE or ISFLUSH.
 153                          *
 154                          * Note that vnode locks may be held going into
 155                          * this call.
 156                          *
 157                          * NOTE: Remember that non-modifying operations
 158                          *       such as read, stat, readdir, etc, do
 159                          *       not use transactions.
 160                          */
 161                         nflags = (oflags | flags) + 1;
 162                 }
 163                 if (dowait)
 164                         tsleep_interlock(&pmp->trans.sync_wait, 0);
 165                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 166                         if (dowait == 0)
 167                                 break;
 168                         tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
 169                                "h2trans", hz);
 170                         /* retry */
 171                 } else {
 172                         cpu_pause();
 173                         /* retry */
 174                 }
 175                 /* retry */
 176         }
 177
 178 #if 0
 179         /*
 180          * When entering a FLUSH transaction with COPYQ set, wait for the
 181          * transaction count to drop to 1 (our flush transaction only)
 182          * before proceeding.
 183          *
 184          * This waits for all non-flush transactions to complete and blocks
 185          * new non-flush transactions from starting until COPYQ is cleared.
 186          * (the flush will then proceed after clearing COPYQ).  This should
 187          * be a very short stall on modifying operations.
 188          */
 189         while ((flags & HAMMER2_TRANS_ISFLUSH) &&
 190                (flags & HAMMER2_TRANS_COPYQ)) {
 191                 oflags = pmp->trans.flags;
 192                 cpu_ccfence();
 193                 if ((oflags & HAMMER2_TRANS_MASK) == 1)
 194                         break;
 195                 nflags = oflags | HAMMER2_TRANS_WAITING;
 196                 tsleep_interlock(&pmp->trans.sync_wait, 0);
 197                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 198                         tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
 199                                "h2trans2", hz);
 200                 }
 201         }
 202 #endif
 203 }
 204
 205 /*
 206  * Start a sub-transaction, there is no 'subdone' function.  This will
 207  * issue a new modify_tid (mtid) for the current transaction, which is a
 208  * CLC (cluster level change) id and not a per-node id.
 209  *
 210  * This function must be called for each XOP when multiple XOPs are run in
 211  * sequence within a transaction.
 212  *
 213  * Callers typically update the inode with the transaction mtid manually
 214  * to enforce sequencing.
 215  */
 216 hammer2_tid_t
 217 hammer2_trans_sub(hammer2_pfs_t *pmp)
 218 {
 219         hammer2_tid_t mtid;
 220
 221         mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
 222
 223         return (mtid);
 224 }
 225
 226 void
 227 hammer2_trans_setflags(hammer2_pfs_t *pmp, uint32_t flags)
 228 {
 229         atomic_set_int(&pmp->trans.flags, flags);
 230 }
 231
 232 /*
 233  * Typically used to clear trans flags asynchronously.  If TRANS_WAITING
 234  * is in the mask, and was previously set, this function will wake up
 235  * any waiters.
 236  */
 237 void
 238 hammer2_trans_clearflags(hammer2_pfs_t *pmp, uint32_t flags)
 239 {
 240         uint32_t oflags;
 241         uint32_t nflags;
 242
 243         for (;;) {
 244                 oflags = pmp->trans.flags;
 245                 cpu_ccfence();
 246                 nflags = oflags & ~flags;
 247                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 248                         if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING)
 249                                 wakeup(&pmp->trans.sync_wait);
 250                         break;
 251                 }
 252                 cpu_pause();
 253                 /* retry */
 254         }
 255 }
 256
 257 void
 258 hammer2_trans_done(hammer2_pfs_t *pmp, uint32_t flags)
 259 {
 260         uint32_t oflags;
 261         uint32_t nflags;
 262
 263 #if 0
 264         /*
 265          * Modifying ops on the front-end can cause dirty inodes to
 266          * build up in the sideq.  We don't flush these on inactive/reclaim
 267          * due to potential deadlocks, so we have to deal with them from
 268          * inside other nominal modifying front-end transactions.
 269          */
 270         if ((flags & HAMMER2_TRANS_SIDEQ) &&
 271             pmp->sideq_count > hammer2_limit_dirty_inodes / 2 &&
 272             pmp->sideq_count > (pmp->inum_count >> 3) &&
 273             pmp->mp) {
 274                 speedup_syncer(pmp->mp);
 275         }
 276 #endif
 277
 278         /*
 279          * Clean-up the transaction.  Wakeup any waiters when finishing
 280          * a flush transaction or transitioning the non-flush transaction
 281          * count from 2->1 while a flush transaction is pending.
 282          */
 283         for (;;) {
 284                 oflags = pmp->trans.flags;
 285                 cpu_ccfence();
 286                 KKASSERT(oflags & HAMMER2_TRANS_MASK);
 287
 288                 nflags = (oflags - 1) & ~flags;
 289                 if (flags & HAMMER2_TRANS_ISFLUSH) {
 290                         nflags &= ~HAMMER2_TRANS_WAITING;
 291                 }
 292                 if ((oflags & (HAMMER2_TRANS_ISFLUSH|HAMMER2_TRANS_MASK)) ==
 293                     (HAMMER2_TRANS_ISFLUSH|2)) {
 294                         nflags &= ~HAMMER2_TRANS_WAITING;
 295                 }
 296                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 297                         if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING)
 298                                 wakeup(&pmp->trans.sync_wait);
 299                         break;
 300                 }
 301                 cpu_pause();
 302                 /* retry */
 303         }
 304 }
 305
 306 /*
 307  * Obtain new, unique inode number (not serialized by caller).
 308  */
 309 hammer2_tid_t
 310 hammer2_trans_newinum(hammer2_pfs_t *pmp)
 311 {
 312         hammer2_tid_t tid;
 313
 314         tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
 315
 316         return tid;
 317 }
 318
 319 /*
 320  * Assert that a strategy call is ok here.  Currently we allow strategy
 321  * calls in all situations, including during flushes.  Previously:
 322  *      (old) (1) In a normal transaction.
 323  *      (old) (2) In a flush transaction only if PREFLUSH is also set.
 324  */
 325 void
 326 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
 327 {
 328 #if 0
 329         KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 ||
 330                  (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH));
 331 #endif
 332 }
 333
 334 /*
 335  * Flush the chain and all modified sub-chains through the specified
 336  * synchronization point, propagating blockref updates back up.  As
 337  * part of this propagation, mirror_tid and inode/data usage statistics
 338  * propagates back upward.
 339  *
 340  * Returns a HAMMER2 error code, 0 if no error.  Note that I/O errors from
 341  * buffers dirtied during the flush operation can occur later.
 342  *
 343  * modify_tid (clc - cluster level change) is not propagated.
 344  *
 345  * update_tid (clc) is used for validation and is not propagated by this
 346  * function.
 347  *
 348  * This routine can be called from several places but the most important
 349  * is from VFS_SYNC (frontend) via hammer2_xop_inode_flush (backend).
 350  *
 351  * chain is locked on call and will remain locked on return.  The chain's
 352  * UPDATE flag indicates that its parent's block table (which is not yet
 353  * part of the flush) should be updated.
 354  *
 355  * flags:
 356  *      HAMMER2_FLUSH_TOP       Indicates that this is the top of the flush.
 357  *                              Is cleared for the recursion.
 358  *
 359  *      HAMMER2_FLUSH_ALL       Recurse everything
 360  *
 361  *      HAMMER2_FLUSH_INODE_STOP
 362  *                              Stop at PFS inode or normal inode boundary
 363  */
 364 int
 365 hammer2_flush(hammer2_chain_t *chain, int flags)
 366 {
 367         hammer2_flush_info_t info;
 368         hammer2_dev_t *hmp;
 369         int loops;
 370
 371         /*
 372          * Execute the recursive flush and handle deferrals.
 373          *
 374          * Chains can be ridiculously long (thousands deep), so to
 375          * avoid blowing out the kernel stack the recursive flush has a
 376          * depth limit.  Elements at the limit are placed on a list
 377          * for re-execution after the stack has been popped.
 378          */
 379         bzero(&info, sizeof(info));
 380         info.flags = flags & ~HAMMER2_FLUSH_TOP;
 381
 382         /*
 383          * Calculate parent (can be NULL), if not NULL the flush core
 384          * expects the parent to be referenced so it can easily lock/unlock
 385          * it without it getting ripped up.
 386          */
 387         if ((info.parent = chain->parent) != NULL)
 388                 hammer2_chain_ref(info.parent);
 389
 390         /*
 391          * Extra ref needed because flush_core expects it when replacing
 392          * chain.
 393          */
 394         hammer2_chain_ref(chain);
 395         hmp = chain->hmp;
 396         loops = 0;
 397
 398         for (;;) {
 399                 /*
 400                  * [re]flush chain as the deep recursion may have generated
 401                  * additional modifications.
 402                  */
 403                 if (info.parent != chain->parent) {
 404                         if (hammer2_debug & 0x0040) {
 405                                 kprintf("LOST CHILD4 %p->%p "
 406                                         "(actual parent %p)\n",
 407                                         info.parent, chain, chain->parent);
 408                         }
 409                         hammer2_chain_drop(info.parent);
 410                         info.parent = chain->parent;
 411                         hammer2_chain_ref(info.parent);
 412                 }
 413                 if (hammer2_flush_core(&info, chain, flags) == 0)
 414                         break;
 415
 416                 if (++loops % 1000 == 0) {
 417                         kprintf("hammer2_flush: excessive loops on %p\n",
 418                                 chain);
 419                         if (hammer2_debug & 0x100000)
 420                                 Debugger("hell4");
 421                 }
 422         }
 423 #ifdef HAMMER2_SCAN_DEBUG
 424         if (info.scan_count >= 10)
 425         kprintf("hammer2_flush: scan_count %ld (%ld,%ld,%ld,%ld) "
 426                 "bt(%ld,%ld,%ld,%ld,%ld,%ld)\n",
 427                 info.scan_count,
 428                 info.scan_mod_count,
 429                 info.scan_upd_count,
 430                 info.scan_onf_count,
 431                 info.scan_del_count,
 432                 info.scan_btype[1],
 433                 info.scan_btype[2],
 434                 info.scan_btype[3],
 435                 info.scan_btype[4],
 436                 info.scan_btype[5],
 437                 info.scan_btype[6]);
 438 #endif
 439         hammer2_chain_drop(chain);
 440         if (info.parent)
 441                 hammer2_chain_drop(info.parent);
 442         return (info.error);
 443 }
 444
 445 /*
 446  * This is the core of the chain flushing code.  The chain is locked by the
 447  * caller and must also have an extra ref on it by the caller, and remains
 448  * locked and will have an extra ref on return.  info.parent is referenced
 449  * but not locked.
 450  *
 451  * Upon return, the caller can test the UPDATE bit on the chain to determine
 452  * if the parent needs updating.
 453  *
 454  * If non-zero is returned, the chain's parent changed during the flush and
 455  * the caller must retry the operation.
 456  *
 457  * (1) Determine if this node is a candidate for the flush, return if it is
 458  *     not.  fchain and vchain are always candidates for the flush.
 459  *
 460  * (2) If we recurse too deep the chain is entered onto the deferral list and
 461  *     the current flush stack is aborted until after the deferral list is
 462  *     run.
 463  *
 464  * (3) Recursively flush live children (rbtree).  This can create deferrals.
 465  *     A successful flush clears the MODIFIED and UPDATE bits on the children
 466  *     and typically causes the parent to be marked MODIFIED as the children
 467  *     update the parent's block table.  A parent might already be marked
 468  *     MODIFIED due to a deletion (whos blocktable update in the parent is
 469  *     handled by the frontend), or if the parent itself is modified by the
 470  *     frontend for other reasons.
 471  *
 472  * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
 473  *     Deleted-but-open inodes can still be individually flushed via the
 474  *     filesystem syncer.
 475  *
 476  * (5) Delete parents on the way back up if they are normal indirect blocks
 477  *     and have no children.
 478  *
 479  * (6) Note that an unmodified child may still need the block table in its
 480  *     parent updated (e.g. rename/move).  The child will have UPDATE set
 481  *     in this case.
 482  *
 483  *                      WARNING ON BREF MODIFY_TID/MIRROR_TID
 484  *
 485  * blockref.modify_tid is consistent only within a PFS, and will not be
 486  * consistent during synchronization.  mirror_tid is consistent across the
 487  * block device regardless of the PFS.
 488  */
 489 static int
 490 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
 491                    int flags)
 492 {
 493         hammer2_chain_t *parent;
 494         hammer2_dev_t *hmp;
 495         int save_error;
 496         int retry;
 497
 498         retry = 0;
 499
 500         /*
 501          * (1) Optimize downward recursion to locate nodes needing action.
 502          *     Nothing to do if none of these flags are set.
 503          */
 504         if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) {
 505                 if (hammer2_debug & 0x200) {
 506                         if (info->debug == NULL)
 507                                 info->debug = chain;
 508                 } else {
 509                         return 0;
 510                 }
 511         }
 512
 513         hmp = chain->hmp;
 514
 515         /*
 516          * NOTE: parent can be NULL, usually due to destroy races.
 517          */
 518         parent = info->parent;
 519         KKASSERT(chain->parent == parent);
 520
 521         /*
 522          * Downward search recursion
 523          *
 524          * We must be careful on cold stops, which often occur on inode
 525          * boundaries due to the way hammer2_vfs_sync() sequences the flush.
 526          * Be sure to issue an appropriate chain_setflush()
 527          */
 528         if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) &&
 529             (flags & HAMMER2_FLUSH_ALL) == 0 &&
 530             (flags & HAMMER2_FLUSH_TOP) == 0 &&
 531             chain->pmp && chain->pmp->mp) {
 532                 /*
 533                  * If FLUSH_ALL is not specified the caller does not want
 534                  * to recurse through PFS roots that have been mounted.
 535                  *
 536                  * (If the PFS has not been mounted there may not be
 537                  *  anything monitoring its chains and its up to us
 538                  *  to flush it).
 539                  *
 540                  * The typical sequence is to flush dirty PFS's starting at
 541                  * their root downward, then flush the device root (vchain).
 542                  * It is this second flush that typically leaves out the
 543                  * ALL flag.
 544                  *
 545                  * However we must still process the PFSROOT chains for block
 546                  * table updates in their parent (which IS part of our flush).
 547                  *
 548                  * NOTE: The volume root, vchain, does not set PFSBOUNDARY.
 549                  *
 550                  * NOTE: We must re-set ONFLUSH in the parent to retain if
 551                  *       this chain (that we are skipping) requires work.
 552                  */
 553                 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
 554                                     HAMMER2_CHAIN_DESTROY |
 555                                     HAMMER2_CHAIN_MODIFIED)) {
 556                         hammer2_chain_setflush(parent);
 557                 }
 558                 goto done;
 559         } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
 560                    (flags & HAMMER2_FLUSH_INODE_STOP) &&
 561                    (flags & HAMMER2_FLUSH_ALL) == 0 &&
 562                    (flags & HAMMER2_FLUSH_TOP) == 0 &&
 563                    chain->pmp && chain->pmp->mp) {
 564                 /*
 565                  * When FLUSH_INODE_STOP is specified we are being asked not
 566                  * to include any inode changes for inodes we encounter,
 567                  * with the exception of the inode that the flush began with.
 568                  * So: INODE, INODE_STOP, and TOP==0 basically.
 569                  *
 570                  * Dirty inodes are flushed based on the hammer2_inode
 571                  * in-memory structure, issuing a chain_setflush() here
 572                  * will only cause unnecessary traversals of the topology.
 573                  */
 574                 goto done;
 575 #if 0
 576                 /*
 577                  * If FLUSH_INODE_STOP is specified and both ALL and TOP
 578                  * are clear, we must not flush the chain.  The chain should
 579                  * have already been flushed and any further ONFLUSH/UPDATE
 580                  * setting will be related to the next flush.
 581                  *
 582                  * This features allows us to flush inodes independently of
 583                  * each other and meta-data above the inodes separately.
 584                  */
 585                 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
 586                                     HAMMER2_CHAIN_DESTROY |
 587                                     HAMMER2_CHAIN_MODIFIED)) {
 588                         if (parent)
 589                                 hammer2_chain_setflush(parent);
 590                 }
 591 #endif
 592         } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
 593                 /*
 594                  * Recursion depth reached.
 595                  */
 596                 panic("hammer2: flush depth limit");
 597         } else if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
 598                                    HAMMER2_CHAIN_DESTROY)) {
 599                 /*
 600                  * Downward recursion search (actual flush occurs bottom-up).
 601                  * pre-clear ONFLUSH.  It can get set again due to races or
 602                  * flush errors, which we want so the scan finds us again in
 603                  * the next flush.
 604                  *
 605                  * We must also recurse if DESTROY is set so we can finally
 606                  * get rid of the related children, otherwise the node will
 607                  * just get re-flushed on lastdrop.
 608                  *
 609                  * WARNING!  The recursion will unlock/relock info->parent
 610                  *           (which is 'chain'), potentially allowing it
 611                  *           to be ripped up.
 612                  */
 613                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
 614                 save_error = info->error;
 615                 info->error = 0;
 616                 info->parent = chain;
 617
 618                 /*
 619                  * We may have to do this twice to catch any indirect
 620                  * block maintenance that occurs.
 621                  */
 622                 hammer2_spin_ex(&chain->core.spin);
 623                 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
 624                         NULL, hammer2_flush_recurse, info);
 625                 if (chain->flags & HAMMER2_CHAIN_ONFLUSH) {
 626                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
 627                         RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
 628                                 NULL, hammer2_flush_recurse, info);
 629                 }
 630                 hammer2_spin_unex(&chain->core.spin);
 631                 info->parent = parent;
 632
 633                 /*
 634                  * Re-set the flush bits if the flush was incomplete or
 635                  * an error occurred.  If an error occurs it is typically
 636                  * an allocation error.  Errors do not cause deferrals.
 637                  */
 638                 if (info->error)
 639                         hammer2_chain_setflush(chain);
 640                 info->error |= save_error;
 641
 642                 /*
 643                  * If we lost the parent->chain association we have to
 644                  * stop processing this chain because it is no longer
 645                  * in this recursion.  If it moved, it will be handled
 646                  * by the ONFLUSH flag elsewhere.
 647                  */
 648                 if (chain->parent != parent) {
 649                         kprintf("LOST CHILD2 %p->%p (actual parent %p)\n",
 650                                 parent, chain, chain->parent);
 651                         goto done;
 652                 }
 653         }
 654
 655         /*
 656          * Now we are in the bottom-up part of the recursion.
 657          *
 658          * We continue to try to update the chain on lower-level errors, but
 659          * the flush code may decide not to flush the volume root.
 660          *
 661          * XXX should we continue to try to update the chain if an error
 662          *     occurred?
 663          */
 664
 665         /*
 666          * Both parent and chain must be locked in order to flush chain,
 667          * in order to properly update the parent under certain conditions.
 668          *
 669          * In addition, we can't safely unlock/relock the chain once we
 670          * start flushing the chain itself, which we would have to do later
 671          * on in order to lock the parent if we didn't do that now.
 672          */
 673         hammer2_chain_ref_hold(chain);
 674         hammer2_chain_unlock(chain);
 675         if (parent)
 676                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
 677         hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
 678         hammer2_chain_drop_unhold(chain);
 679
 680         /*
 681          * Can't process if we can't access their content.
 682          */
 683         if ((parent && parent->error) || chain->error) {
 684                 kprintf("hammer2: chain error during flush\n");
 685                 info->error |= chain->error;
 686                 if (parent) {
 687                         info->error |= parent->error;
 688                         hammer2_chain_unlock(parent);
 689                 }
 690                 goto done;
 691         }
 692
 693         if (chain->parent != parent) {
 694                 if (hammer2_debug & 0x0040) {
 695                         kprintf("LOST CHILD3 %p->%p (actual parent %p)\n",
 696                                 parent, chain, chain->parent);
 697                 }
 698                 KKASSERT(parent != NULL);
 699                 hammer2_chain_unlock(parent);
 700                 retry = 1;
 701                 goto done;
 702         }
 703
 704         /*
 705          * Propagate the DESTROY flag downwards.  This dummies up the flush
 706          * code and tries to invalidate related buffer cache buffers to
 707          * avoid the disk write.
 708          */
 709         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
 710                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
 711
 712         /*
 713          * Dispose of the modified bit.
 714          *
 715          * If parent is present, the UPDATE bit should already be set.
 716          * UPDATE should already be set.
 717          * bref.mirror_tid should already be set.
 718          */
 719         if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
 720                 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
 721                          chain->parent == NULL);
 722                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
 723                 atomic_add_long(&hammer2_count_modified_chains, -1);
 724
 725                 /*
 726                  * Manage threads waiting for excessive dirty memory to
 727                  * be retired.
 728                  */
 729                 if (chain->pmp)
 730                         hammer2_pfs_memory_wakeup(chain->pmp);
 731
 732 #if 0
 733                 if ((chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
 734                     chain != &hmp->vchain &&
 735                     chain != &hmp->fchain) {
 736                         /*
 737                          * Set UPDATE bit indicating that the parent block
 738                          * table requires updating.
 739                          */
 740                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 741                 }
 742 #endif
 743
 744                 /*
 745                  * Issue the flush.  This is indirect via the DIO.
 746                  *
 747                  * NOTE: A DELETED node that reaches this point must be
 748                  *       flushed for synchronization point consistency.
 749                  *
 750                  * NOTE: Even though MODIFIED was already set, the related DIO
 751                  *       might not be dirty due to a system buffer cache
 752                  *       flush and must be set dirty if we are going to make
 753                  *       further modifications to the buffer.  Chains with
 754                  *       embedded data don't need this.
 755                  */
 756                 if (hammer2_debug & 0x1000) {
 757                         kprintf("Flush %p.%d %016jx/%d data=%016jx\n",
 758                                 chain, chain->bref.type,
 759                                 (uintmax_t)chain->bref.key,
 760                                 chain->bref.keybits,
 761                                 (uintmax_t)chain->bref.data_off);
 762                 }
 763                 if (hammer2_debug & 0x2000) {
 764                         Debugger("Flush hell");
 765                 }
 766
 767                 /*
 768                  * Update chain CRCs for flush.
 769                  *
 770                  * NOTE: Volume headers are NOT flushed here as they require
 771                  *       special processing.
 772                  */
 773                 switch(chain->bref.type) {
 774                 case HAMMER2_BREF_TYPE_FREEMAP:
 775                         /*
 776                          * Update the volume header's freemap_tid to the
 777                          * freemap's flushing mirror_tid.
 778                          *
 779                          * (note: embedded data, do not call setdirty)
 780                          */
 781                         KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
 782                         KKASSERT(chain == &hmp->fchain);
 783                         hmp->voldata.freemap_tid = chain->bref.mirror_tid;
 784                         if (hammer2_debug & 0x8000) {
 785                                 /* debug only, avoid syslogd loop */
 786                                 kprintf("sync freemap mirror_tid %08jx\n",
 787                                         (intmax_t)chain->bref.mirror_tid);
 788                         }
 789
 790                         /*
 791                          * The freemap can be flushed independently of the
 792                          * main topology, but for the case where it is
 793                          * flushed in the same transaction, and flushed
 794                          * before vchain (a case we want to allow for
 795                          * performance reasons), make sure modifications
 796                          * made during the flush under vchain use a new
 797                          * transaction id.
 798                          *
 799                          * Otherwise the mount recovery code will get confused.
 800                          */
 801                         ++hmp->voldata.mirror_tid;
 802                         break;
 803                 case HAMMER2_BREF_TYPE_VOLUME:
 804                         /*
 805                          * The free block table is flushed by
 806                          * hammer2_vfs_sync() before it flushes vchain.
 807                          * We must still hold fchain locked while copying
 808                          * voldata to volsync, however.
 809                          *
 810                          * These do not error per-say since their data does
 811                          * not need to be re-read from media on lock.
 812                          *
 813                          * (note: embedded data, do not call setdirty)
 814                          */
 815                         hammer2_chain_lock(&hmp->fchain,
 816                                            HAMMER2_RESOLVE_ALWAYS);
 817                         hammer2_voldata_lock(hmp);
 818                         if (hammer2_debug & 0x8000) {
 819                                 /* debug only, avoid syslogd loop */
 820                                 kprintf("sync volume  mirror_tid %08jx\n",
 821                                         (intmax_t)chain->bref.mirror_tid);
 822                         }
 823
 824                         /*
 825                          * Update the volume header's mirror_tid to the
 826                          * main topology's flushing mirror_tid.  It is
 827                          * possible that voldata.mirror_tid is already
 828                          * beyond bref.mirror_tid due to the bump we made
 829                          * above in BREF_TYPE_FREEMAP.
 830                          */
 831                         if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
 832                                 hmp->voldata.mirror_tid =
 833                                         chain->bref.mirror_tid;
 834                         }
 835
 836                         /*
 837                          * The volume header is flushed manually by the
 838                          * syncer, not here.  All we do here is adjust the
 839                          * crc's.
 840                          */
 841                         KKASSERT(chain->data != NULL);
 842                         KKASSERT(chain->dio == NULL);
 843
 844                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
 845                                 hammer2_icrc32(
 846                                         (char *)&hmp->voldata +
 847                                          HAMMER2_VOLUME_ICRC1_OFF,
 848                                         HAMMER2_VOLUME_ICRC1_SIZE);
 849                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
 850                                 hammer2_icrc32(
 851                                         (char *)&hmp->voldata +
 852                                          HAMMER2_VOLUME_ICRC0_OFF,
 853                                         HAMMER2_VOLUME_ICRC0_SIZE);
 854                         hmp->voldata.icrc_volheader =
 855                                 hammer2_icrc32(
 856                                         (char *)&hmp->voldata +
 857                                          HAMMER2_VOLUME_ICRCVH_OFF,
 858                                         HAMMER2_VOLUME_ICRCVH_SIZE);
 859
 860                         if (hammer2_debug & 0x8000) {
 861                                 /* debug only, avoid syslogd loop */
 862                                 kprintf("syncvolhdr %016jx %016jx\n",
 863                                         hmp->voldata.mirror_tid,
 864                                         hmp->vchain.bref.mirror_tid);
 865                         }
 866                         hmp->volsync = hmp->voldata;
 867                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
 868                         hammer2_voldata_unlock(hmp);
 869                         hammer2_chain_unlock(&hmp->fchain);
 870                         break;
 871                 case HAMMER2_BREF_TYPE_DATA:
 872                         /*
 873                          * Data elements have already been flushed via the
 874                          * logical file buffer cache.  Their hash was set in
 875                          * the bref by the vop_write code.  Do not re-dirty.
 876                          *
 877                          * Make sure any device buffer(s) have been flushed
 878                          * out here (there aren't usually any to flush) XXX.
 879                          */
 880                         break;
 881                 case HAMMER2_BREF_TYPE_INDIRECT:
 882                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
 883                 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
 884                         /*
 885                          * Buffer I/O will be cleaned up when the volume is
 886                          * flushed (but the kernel is free to flush it before
 887                          * then, as well).
 888                          */
 889                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
 890                         hammer2_chain_setcheck(chain, chain->data);
 891                         break;
 892                 case HAMMER2_BREF_TYPE_DIRENT:
 893                         /*
 894                          * A directory entry can use the check area to store
 895                          * the filename for filenames <= 64 bytes, don't blow
 896                          * it up!
 897                          */
 898                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
 899                         if (chain->bytes)
 900                                 hammer2_chain_setcheck(chain, chain->data);
 901                         break;
 902                 case HAMMER2_BREF_TYPE_INODE:
 903                         /*
 904                          * NOTE: We must call io_setdirty() to make any late
 905                          *       changes to the inode data, the system might
 906                          *       have already flushed the buffer.
 907                          */
 908                         if (chain->data->ipdata.meta.op_flags &
 909                             HAMMER2_OPFLAG_PFSROOT) {
 910                                 /*
 911                                  * non-NULL pmp if mounted as a PFS.  We must
 912                                  * sync fields cached in the pmp? XXX
 913                                  */
 914                                 hammer2_inode_data_t *ipdata;
 915
 916                                 hammer2_io_setdirty(chain->dio);
 917                                 ipdata = &chain->data->ipdata;
 918                                 if (chain->pmp) {
 919                                         ipdata->meta.pfs_inum =
 920                                                 chain->pmp->inode_tid;
 921                                 }
 922                         } else {
 923                                 /* can't be mounted as a PFS */
 924                         }
 925
 926                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
 927                         hammer2_chain_setcheck(chain, chain->data);
 928                         break;
 929                 default:
 930                         KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
 931                         panic("hammer2_flush_core: unsupported "
 932                               "embedded bref %d",
 933                               chain->bref.type);
 934                         /* NOT REACHED */
 935                 }
 936
 937                 /*
 938                  * If the chain was destroyed try to avoid unnecessary I/O
 939                  * that might not have yet occurred.  Remove the data range
 940                  * from dedup candidacy and attempt to invalidation that
 941                  * potentially dirty portion of the I/O buffer.
 942                  */
 943                 if (chain->flags & HAMMER2_CHAIN_DESTROY) {
 944                         hammer2_io_dedup_delete(hmp,
 945                                                 chain->bref.type,
 946                                                 chain->bref.data_off,
 947                                                 chain->bytes);
 948 #if 0
 949                         hammer2_io_t *dio;
 950                         if (chain->dio) {
 951                                 hammer2_io_inval(chain->dio,
 952                                                  chain->bref.data_off,
 953                                                  chain->bytes);
 954                         } else if ((dio = hammer2_io_getquick(hmp,
 955                                                   chain->bref.data_off,
 956                                                   chain->bytes,
 957                                                   1)) != NULL) {
 958                                 hammer2_io_inval(dio,
 959                                                  chain->bref.data_off,
 960                                                  chain->bytes);
 961                                 hammer2_io_putblk(&dio);
 962                         }
 963 #endif
 964                 }
 965         }
 966
 967         /*
 968          * If UPDATE is set the parent block table may need to be updated.
 969          * This can fail if the hammer2_chain_modify() fails.
 970          *
 971          * NOTE: UPDATE may be set on vchain or fchain in which case
 972          *       parent could be NULL, or on an inode that has not yet
 973          *       been inserted into the radix tree.  It's easiest to allow
 974          *       the case and test for NULL.  parent can also wind up being
 975          *       NULL due to a deletion so we need to handle the case anyway.
 976          *
 977          * NOTE: UPDATE can be set when chains are renamed into or out of
 978          *       an indirect block, without the chain itself being flagged
 979          *       MODIFIED.
 980          *
 981          * If no parent exists we can just clear the UPDATE bit.  If the
 982          * chain gets reattached later on the bit will simply get set
 983          * again.
 984          */
 985         if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL)
 986                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 987
 988         /*
 989          * When flushing an inode outside of a FLUSH_FSSYNC we must NOT
 990          * update the parent block table to point at the flushed inode.
 991          * The block table should only ever be updated by the filesystem
 992          * sync code.  If we do, inode<->inode dependencies (such as
 993          * directory entries vs inode nlink count) can wind up not being
 994          * flushed together and result in a broken topology if a crash/reboot
 995          * occurs at the wrong time.
 996          */
 997         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
 998             (flags & HAMMER2_FLUSH_INODE_STOP) &&
 999             (flags & HAMMER2_FLUSH_FSSYNC) == 0 &&
1000             (flags & HAMMER2_FLUSH_ALL) == 0 &&
1001             chain->pmp && chain->pmp->mp) {
1002 #ifdef HAMMER2_DEBUG_SYNC
1003                 kprintf("inum %ld do not update parent, non-fssync\n",
1004                         (long)chain->bref.key);
1005 #endif
1006                 goto skipupdate;
1007         }
1008 #ifdef HAMMER2_DEBUG_SYNC
1009         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE)
1010                 kprintf("inum %ld update parent\n", (long)chain->bref.key);
1011 #endif
1012
1013         /*
1014          * The chain may need its blockrefs updated in the parent, normal
1015          * path.
1016          */
1017         if (chain->flags & HAMMER2_CHAIN_UPDATE) {
1018                 hammer2_blockref_t *base;
1019                 int count;
1020
1021                 /*
1022                  * Clear UPDATE flag, mark parent modified, update its
1023                  * modify_tid if necessary, and adjust the parent blockmap.
1024                  */
1025                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1026
1027                 /*
1028                  * (optional code)
1029                  *
1030                  * Avoid actually modifying and updating the parent if it
1031                  * was flagged for destruction.  This can greatly reduce
1032                  * disk I/O in large tree removals because the
1033                  * hammer2_io_setinval() call in the upward recursion
1034                  * (see MODIFIED code above) can only handle a few cases.
1035                  */
1036                 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
1037                         if (parent->bref.modify_tid < chain->bref.modify_tid) {
1038                                 parent->bref.modify_tid =
1039                                         chain->bref.modify_tid;
1040                         }
1041                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED |
1042                                                         HAMMER2_CHAIN_BMAPUPD);
1043                         goto skipupdate;
1044                 }
1045
1046                 /*
1047                  * The flusher is responsible for deleting empty indirect
1048                  * blocks at this point.  If we don't do this, no major harm
1049                  * will be done but the empty indirect blocks will stay in
1050                  * the topology and make it a messy and inefficient.
1051                  *
1052                  * The flusher is also responsible for collapsing the
1053                  * content of an indirect block into its parent whenever
1054                  * possible (with some hysteresis).  Not doing this will also
1055                  * not harm the topology, but would make it messy and
1056                  * inefficient.
1057                  */
1058                 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT) {
1059                         if (hammer2_chain_indirect_maintenance(parent, chain))
1060                                 goto skipupdate;
1061                 }
1062
1063                 /*
1064                  * We are updating the parent's blockmap, the parent must
1065                  * be set modified.  If this fails we re-set the UPDATE flag
1066                  * in the child.
1067                  *
1068                  * NOTE! A modification error can be ENOSPC.  We still want
1069                  *       to flush modified chains recursively, not break out,
1070                  *       so we just skip the update in this situation and
1071                  *       continue.  That is, we still need to try to clean
1072                  *       out dirty chains and buffers.
1073                  *
1074                  *       This may not help bulkfree though. XXX
1075                  */
1076                 save_error = hammer2_chain_modify(parent, 0, 0, 0);
1077                 if (save_error) {
1078                         info->error |= save_error;
1079                         kprintf("hammer2_flush: %016jx.%02x error=%08x\n",
1080                                 parent->bref.data_off, parent->bref.type,
1081                                 save_error);
1082                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1083                         goto skipupdate;
1084                 }
1085                 if (parent->bref.modify_tid < chain->bref.modify_tid)
1086                         parent->bref.modify_tid = chain->bref.modify_tid;
1087
1088                 /*
1089                  * Calculate blockmap pointer
1090                  */
1091                 switch(parent->bref.type) {
1092                 case HAMMER2_BREF_TYPE_INODE:
1093                         /*
1094                          * Access the inode's block array.  However, there is
1095                          * no block array if the inode is flagged DIRECTDATA.
1096                          */
1097                         if (parent->data &&
1098                             (parent->data->ipdata.meta.op_flags &
1099                              HAMMER2_OPFLAG_DIRECTDATA) == 0) {
1100                                 base = &parent->data->
1101                                         ipdata.u.blockset.blockref[0];
1102                         } else {
1103                                 base = NULL;
1104                         }
1105                         count = HAMMER2_SET_COUNT;
1106                         break;
1107                 case HAMMER2_BREF_TYPE_INDIRECT:
1108                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1109                         if (parent->data)
1110                                 base = &parent->data->npdata[0];
1111                         else
1112                                 base = NULL;
1113                         count = parent->bytes / sizeof(hammer2_blockref_t);
1114                         break;
1115                 case HAMMER2_BREF_TYPE_VOLUME:
1116                         base = &chain->hmp->voldata.sroot_blockset.blockref[0];
1117                         count = HAMMER2_SET_COUNT;
1118                         break;
1119                 case HAMMER2_BREF_TYPE_FREEMAP:
1120                         base = &parent->data->npdata[0];
1121                         count = HAMMER2_SET_COUNT;
1122                         break;
1123                 default:
1124                         base = NULL;
1125                         count = 0;
1126                         panic("hammer2_flush_core: "
1127                               "unrecognized blockref type: %d",
1128                               parent->bref.type);
1129                 }
1130
1131                 /*
1132                  * Blocktable updates
1133                  *
1134                  * We synchronize pending statistics at this time.  Delta
1135                  * adjustments designated for the current and upper level
1136                  * are synchronized.
1137                  */
1138                 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) {
1139                         if (chain->flags & HAMMER2_CHAIN_BMAPPED) {
1140                                 hammer2_spin_ex(&parent->core.spin);
1141                                 hammer2_base_delete(parent, base, count, chain,
1142                                                     NULL);
1143                                 hammer2_spin_unex(&parent->core.spin);
1144                                 /* base_delete clears both bits */
1145                         } else {
1146                                 atomic_clear_int(&chain->flags,
1147                                                  HAMMER2_CHAIN_BMAPUPD);
1148                         }
1149                 }
1150                 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) {
1151                         hammer2_spin_ex(&parent->core.spin);
1152                         hammer2_base_insert(parent, base, count,
1153                                             chain, &chain->bref);
1154                         hammer2_spin_unex(&parent->core.spin);
1155                         /* base_insert sets BMAPPED */
1156                 }
1157         }
1158 skipupdate:
1159         if (parent)
1160                 hammer2_chain_unlock(parent);
1161
1162         /*
1163          * Final cleanup after flush
1164          */
1165 done:
1166         KKASSERT(chain->refs > 0);
1167         if (hammer2_debug & 0x200) {
1168                 if (info->debug == chain)
1169                         info->debug = NULL;
1170         }
1171         return retry;
1172 }
1173
1174 /*
1175  * Flush recursion helper, called from flush_core, calls flush_core.
1176  *
1177  * Flushes the children of the caller's chain (info->parent), restricted
1178  * by sync_tid.
1179  *
1180  * This function may set info->error as a side effect.
1181  *
1182  * WARNING! If we do not call hammer2_flush_core() we must update
1183  *          bref.mirror_tid ourselves to indicate that the flush has
1184  *          processed the child.
1185  *
1186  * WARNING! parent->core spinlock is held on entry and return.
1187  */
1188 static int
1189 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
1190 {
1191         hammer2_flush_info_t *info = data;
1192         hammer2_chain_t *parent = info->parent;
1193
1194 #ifdef HAMMER2_SCAN_DEBUG
1195         ++info->scan_count;
1196         if (child->flags & HAMMER2_CHAIN_MODIFIED)
1197                 ++info->scan_mod_count;
1198         if (child->flags & HAMMER2_CHAIN_UPDATE)
1199                 ++info->scan_upd_count;
1200         if (child->flags & HAMMER2_CHAIN_ONFLUSH)
1201                 ++info->scan_onf_count;
1202 #endif
1203
1204         /*
1205          * (child can never be fchain or vchain so a special check isn't
1206          *  needed).
1207          *
1208          * We must ref the child before unlocking the spinlock.
1209          *
1210          * The caller has added a ref to the parent so we can temporarily
1211          * unlock it in order to lock the child.  However, if it no longer
1212          * winds up being the child of the parent we must skip this child.
1213          *
1214          * NOTE! chain locking errors are fatal.  They are never out-of-space
1215          *       errors.
1216          */
1217         hammer2_chain_ref(child);
1218         hammer2_spin_unex(&parent->core.spin);
1219
1220         hammer2_chain_ref_hold(parent);
1221         hammer2_chain_unlock(parent);
1222         hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
1223         if (child->parent != parent) {
1224                 kprintf("LOST CHILD1 %p->%p (actual parent %p)\n",
1225                         parent, child, child->parent);
1226                 goto done;
1227         }
1228         if (child->error) {
1229                 kprintf("CHILD ERROR DURING FLUSH LOCK %p->%p\n",
1230                         parent, child);
1231                 info->error |= child->error;
1232                 goto done;
1233         }
1234
1235         /*
1236          * Must propagate the DESTROY flag downwards, otherwise the
1237          * parent could end up never being removed because it will
1238          * be requeued to the flusher if it survives this run due to
1239          * the flag.
1240          */
1241         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
1242                 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROY);
1243 #ifdef HAMMER2_SCAN_DEBUG
1244         if (child->flags & HAMMER2_CHAIN_DESTROY)
1245                 ++info->scan_del_count;
1246 #endif
1247         /*
1248          * Special handling of the root inode.  Because the root inode
1249          * contains an index of all the inodes in the PFS in addition to
1250          * its normal directory entries, any flush that is not part of a
1251          * filesystem sync must only flush the directory entries, and not
1252          * anything else.
1253          *
1254          * The child might be an indirect block, but H2 guarantees that
1255          * the key-range will fully partition the inode index from the
1256          * directory entries so the case just works naturally.
1257          */
1258         if ((parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
1259             (child->flags & HAMMER2_CHAIN_DESTROY) == 0 &&
1260             parent->bref.type == HAMMER2_BREF_TYPE_INODE &&
1261             (info->flags & HAMMER2_FLUSH_FSSYNC) == 0) {
1262                 if ((child->bref.key & HAMMER2_DIRHASH_VISIBLE) == 0) {
1263                         if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1264                                 hammer2_chain_setflush(parent);
1265                         }
1266                         goto done;
1267                 }
1268         }
1269
1270         /*
1271          * Recurse and collect deferral data.  We're in the media flush,
1272          * this can cross PFS boundaries.
1273          */
1274         if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1275 #ifdef HAMMER2_SCAN_DEBUG
1276                 if (child->bref.type < 7)
1277                         ++info->scan_btype[child->bref.type];
1278 #endif
1279                 ++info->depth;
1280                 hammer2_flush_core(info, child, info->flags);
1281                 --info->depth;
1282         } else if (hammer2_debug & 0x200) {
1283                 if (info->debug == NULL)
1284                         info->debug = child;
1285                 ++info->depth;
1286                 hammer2_flush_core(info, child, info->flags);
1287                 --info->depth;
1288                 if (info->debug == child)
1289                         info->debug = NULL;
1290         }
1291
1292 done:
1293         /*
1294          * Relock to continue the loop.
1295          */
1296         hammer2_chain_unlock(child);
1297         hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1298         hammer2_chain_drop_unhold(parent);
1299         if (parent->error) {
1300                 kprintf("PARENT ERROR DURING FLUSH LOCK %p->%p\n",
1301                         parent, child);
1302                 info->error |= parent->error;
1303         }
1304         hammer2_chain_drop(child);
1305         KKASSERT(info->parent == parent);
1306         hammer2_spin_ex(&parent->core.spin);
1307
1308         return (0);
1309 }
1310
1311 /*
1312  * flush helper (backend threaded)
1313  *
1314  * Flushes chain topology for the specified inode.
1315  *
1316  * HAMMER2_XOP_INODE_STOP       The flush recursion stops at inode boundaries.
1317  *                              Inodes belonging to the same flush are flushed
1318  *                              separately.
1319  *
1320  * chain->parent can be NULL, usually due to destroy races or detached inodes.
1321  *
1322  * Primarily called from vfs_sync().
1323  */
1324 void
1325 hammer2_xop_inode_flush(hammer2_xop_t *arg, void *scratch __unused, int clindex)
1326 {
1327         hammer2_xop_flush_t *xop = &arg->xop_flush;
1328         hammer2_chain_t *chain;
1329         hammer2_inode_t *ip;
1330         hammer2_dev_t *hmp;
1331         hammer2_pfs_t *pmp;
1332         int flush_error = 0;
1333         int fsync_error = 0;
1334         int total_error = 0;
1335         int j;
1336         int xflags;
1337         int ispfsroot = 0;
1338
1339         xflags = HAMMER2_FLUSH_TOP;
1340         if (xop->head.flags & HAMMER2_XOP_INODE_STOP)
1341                 xflags |= HAMMER2_FLUSH_INODE_STOP;
1342         if (xop->head.flags & HAMMER2_XOP_FSSYNC)
1343                 xflags |= HAMMER2_FLUSH_FSSYNC;
1344
1345         /*
1346          * Flush core chains
1347          */
1348         ip = xop->head.ip1;
1349         pmp = ip->pmp;
1350         chain = hammer2_inode_chain(ip, clindex, HAMMER2_RESOLVE_ALWAYS);
1351         if (chain) {
1352                 hmp = chain->hmp;
1353                 if (chain->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1354                         /*
1355                          * Due to flush partitioning the chain topology
1356                          * above the inode's chain may no longer be flagged.
1357                          * When asked to flush an inode, remark the topology
1358                          * leading to that inode.
1359                          */
1360                         if (chain->parent)
1361                                 hammer2_chain_setflush(chain->parent);
1362                         hammer2_flush(chain, xflags);
1363
1364                         /* XXX cluster */
1365                         if (ip == pmp->iroot && pmp != hmp->spmp) {
1366                                 hammer2_spin_ex(&pmp->inum_spin);
1367                                 pmp->pfs_iroot_blocksets[clindex] =
1368                                         chain->data->ipdata.u.blockset;
1369                                 hammer2_spin_unex(&pmp->inum_spin);
1370                         }
1371
1372 #if 0
1373                         /*
1374                          * Propogate upwards but only cross an inode boundary
1375                          * for inodes associated with the current filesystem
1376                          * sync.
1377                          */
1378                         if ((xop->head.flags & HAMMER2_XOP_PARENTONFLUSH) ||
1379                             chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
1380                                 parent = chain->parent;
1381                                 if (parent)
1382                                         hammer2_chain_setflush(parent);
1383                         }
1384 #endif
1385                 }
1386                 if (chain->flags & HAMMER2_CHAIN_PFSBOUNDARY)
1387                         ispfsroot = 1;
1388                 hammer2_chain_unlock(chain);
1389                 hammer2_chain_drop(chain);
1390                 chain = NULL;
1391         } else {
1392                 hmp = NULL;
1393         }
1394
1395         /*
1396          * Only flush the volume header if asked to, plus the inode must also
1397          * be the PFS root.
1398          */
1399         if ((xop->head.flags & HAMMER2_XOP_VOLHDR) == 0)
1400                 goto skip;
1401         if (ispfsroot == 0)
1402                 goto skip;
1403
1404         /*
1405          * Flush volume roots.  Avoid replication, we only want to
1406          * flush each hammer2_dev (hmp) once.
1407          */
1408         for (j = clindex - 1; j >= 0; --j) {
1409                 if ((chain = ip->cluster.array[j].chain) != NULL) {
1410                         if (chain->hmp == hmp) {
1411                                 chain = NULL;   /* safety */
1412                                 goto skip;
1413                         }
1414                 }
1415         }
1416         chain = NULL;   /* safety */
1417
1418         /*
1419          * spmp transaction.  The super-root is never directly mounted so
1420          * there shouldn't be any vnodes, let alone any dirty vnodes
1421          * associated with it, so we shouldn't have to mess around with any
1422          * vnode flushes here.
1423          */
1424         hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1425
1426         /*
1427          * We must flush the superroot down to the PFS iroot.  Remember
1428          * that hammer2_chain_setflush() stops at inode boundaries, so
1429          * the pmp->iroot has been flushed and flagged down to the superroot,
1430          * but the volume root (vchain) probably has not yet been flagged.
1431          */
1432         if (hmp->spmp->iroot) {
1433                 chain = hmp->spmp->iroot->cluster.array[0].chain;
1434                 if (chain) {
1435                         hammer2_chain_ref(chain);
1436                         hammer2_chain_lock(chain, HAMMER2_RESOLVE_ALWAYS);
1437                         flush_error |=
1438                                 hammer2_flush(chain,
1439                                               HAMMER2_FLUSH_TOP |
1440                                               HAMMER2_FLUSH_INODE_STOP |
1441                                               HAMMER2_FLUSH_FSSYNC);
1442                         hammer2_chain_unlock(chain);
1443                         hammer2_chain_drop(chain);
1444                 }
1445         }
1446
1447         /*
1448          * Media mounts have two 'roots', vchain for the topology
1449          * and fchain for the free block table.  Flush both.
1450          *
1451          * Note that the topology and free block table are handled
1452          * independently, so the free block table can wind up being
1453          * ahead of the topology.  We depend on the bulk free scan
1454          * code to deal with any loose ends.
1455          *
1456          * vchain and fchain do not error on-lock since their data does
1457          * not have to be re-read from media.
1458          */
1459         hammer2_chain_ref(&hmp->vchain);
1460         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1461         hammer2_chain_ref(&hmp->fchain);
1462         hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1463         if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1464                 /*
1465                  * This will also modify vchain as a side effect,
1466                  * mark vchain as modified now.
1467                  */
1468                 hammer2_voldata_modify(hmp);
1469                 chain = &hmp->fchain;
1470                 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1471                 KKASSERT(chain == &hmp->fchain);
1472         }
1473         hammer2_chain_unlock(&hmp->fchain);
1474         hammer2_chain_unlock(&hmp->vchain);
1475         hammer2_chain_drop(&hmp->fchain);
1476         /* vchain dropped down below */
1477
1478         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1479         if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1480                 chain = &hmp->vchain;
1481                 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1482                 KKASSERT(chain == &hmp->vchain);
1483         }
1484         hammer2_chain_unlock(&hmp->vchain);
1485         hammer2_chain_drop(&hmp->vchain);
1486
1487         /*
1488          * We can't safely flush the volume header until we have
1489          * flushed any device buffers which have built up.
1490          *
1491          * XXX this isn't being incremental
1492          */
1493         vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
1494         fsync_error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
1495         vn_unlock(hmp->devvp);
1496         if (fsync_error || flush_error) {
1497                 kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n",
1498                         fsync_error, flush_error, hmp->devrepname);
1499         }
1500
1501         /*
1502          * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1503          * volume header needs synchronization via hmp->volsync.
1504          *
1505          * XXX synchronize the flag & data with only this flush XXX
1506          */
1507         if (fsync_error == 0 && flush_error == 0 &&
1508             (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1509                 struct buf *bp;
1510                 int vol_error = 0;
1511
1512                 /*
1513                  * Synchronize the disk before flushing the volume
1514                  * header.
1515                  */
1516                 bp = getpbuf(NULL);
1517                 bp->b_bio1.bio_offset = 0;
1518                 bp->b_bufsize = 0;
1519                 bp->b_bcount = 0;
1520                 bp->b_cmd = BUF_CMD_FLUSH;
1521                 bp->b_bio1.bio_done = biodone_sync;
1522                 bp->b_bio1.bio_flags |= BIO_SYNC;
1523                 vn_strategy(hmp->devvp, &bp->b_bio1);
1524                 fsync_error = biowait(&bp->b_bio1, "h2vol");
1525                 relpbuf(bp, NULL);
1526
1527                 /*
1528                  * Then we can safely flush the version of the
1529                  * volume header synchronized by the flush code.
1530                  */
1531                 j = hmp->volhdrno + 1;
1532                 if (j < 0)
1533                         j = 0;
1534                 if (j >= HAMMER2_NUM_VOLHDRS)
1535                         j = 0;
1536                 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1537                     hmp->volsync.volu_size) {
1538                         j = 0;
1539                 }
1540                 if (hammer2_debug & 0x8000) {
1541                         /* debug only, avoid syslogd loop */
1542                         kprintf("sync volhdr %d %jd\n",
1543                                 j, (intmax_t)hmp->volsync.volu_size);
1544                 }
1545                 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1546                             HAMMER2_PBUFSIZE, GETBLK_KVABIO, 0);
1547                 atomic_clear_int(&hmp->vchain.flags,
1548                                  HAMMER2_CHAIN_VOLUMESYNC);
1549                 bkvasync(bp);
1550                 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1551                 vol_error = bwrite(bp);
1552                 hmp->volhdrno = j;
1553                 if (vol_error)
1554                         fsync_error = vol_error;
1555         }
1556         if (flush_error)
1557                 total_error = flush_error;
1558         if (fsync_error)
1559                 total_error = hammer2_errno_to_error(fsync_error);
1560
1561         /* spmp trans */
1562         hammer2_trans_done(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1563 skip:
1564         hammer2_xop_feed(&xop->head, NULL, clindex, total_error);
1565 }