sys/vfs/hammer2/hammer2_flush.c

   1 /*
   2  * Copyright (c) 2011-2018 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@dragonflybsd.org>
   6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 /*
  36  *                      TRANSACTION AND FLUSH HANDLING
  37  *
  38  * Deceptively simple but actually fairly difficult to implement properly is
  39  * how I would describe it.
  40  *
  41  * Flushing generally occurs bottom-up but requires a top-down scan to
  42  * locate chains with MODIFIED and/or UPDATE bits set.  The ONFLUSH flag
  43  * tells how to recurse downward to find these chains.
  44  */
  45
  46 #include <sys/cdefs.h>
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/types.h>
  50 #include <sys/lock.h>
  51 #include <sys/vnode.h>
  52 #include <sys/buf.h>
  53
  54 #include "hammer2.h"
  55
  56 #define HAMMER2_FLUSH_DEPTH_LIMIT       60      /* stack recursion limit */
  57
  58
  59 /*
  60  * Recursively flush the specified chain.  The chain is locked and
  61  * referenced by the caller and will remain so on return.  The chain
  62  * will remain referenced throughout but can temporarily lose its
  63  * lock during the recursion to avoid unnecessarily stalling user
  64  * processes.
  65  */
  66 struct hammer2_flush_info {
  67         hammer2_chain_t *parent;
  68         int             depth;
  69         int             error;                  /* cumulative error */
  70         int             flags;
  71 #ifdef HAMMER2_SCAN_DEBUG
  72         long            scan_count;
  73         long            scan_mod_count;
  74         long            scan_upd_count;
  75         long            scan_onf_count;
  76         long            scan_del_count;
  77         long            scan_btype[7];
  78 #endif
  79 };
  80
  81 typedef struct hammer2_flush_info hammer2_flush_info_t;
  82
  83 static int hammer2_flush_core(hammer2_flush_info_t *info,
  84                                 hammer2_chain_t *chain, int flags);
  85 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
  86
  87 /*
  88  * Any per-pfs transaction initialization goes here.
  89  */
  90 void
  91 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
  92 {
  93 }
  94
  95 /*
  96  * Transaction support for any modifying operation.  Transactions are used
  97  * in the pmp layer by the frontend and in the spmp layer by the backend.
  98  *
  99  * 0                    - Normal transaction.  Interlocks against just the
 100  *                        COPYQ portion of an ISFLUSH transaction.
 101  *
 102  * TRANS_ISFLUSH        - Flush transaction.  Interlocks against other flush
 103  *                        transactions.
 104  *
 105  *                        When COPYQ is also specified, waits for the count
 106  *                        to drop to 1.
 107  *
 108  * TRANS_BUFCACHE       - Buffer cache transaction.  No interlock.
 109  *
 110  * TRANS_SIDEQ          - Run the sideq (only tested in trans_done())
 111  *
 112  * Initializing a new transaction allocates a transaction ID.  Typically
 113  * passed a pmp (hmp passed as NULL), indicating a cluster transaction.  Can
 114  * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
 115  * media target.  The latter mode is used by the recovery code.
 116  */
 117 void
 118 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
 119 {
 120         uint32_t oflags;
 121         uint32_t nflags;
 122         int dowait;
 123
 124         for (;;) {
 125                 oflags = pmp->trans.flags;
 126                 cpu_ccfence();
 127                 dowait = 0;
 128
 129                 if (flags & HAMMER2_TRANS_ISFLUSH) {
 130                         /*
 131                          * Interlock against other flush transactions.
 132                          */
 133                         if (oflags & HAMMER2_TRANS_ISFLUSH) {
 134                                 nflags = oflags | HAMMER2_TRANS_WAITING;
 135                                 dowait = 1;
 136                         } else {
 137                                 nflags = (oflags | flags) + 1;
 138                         }
 139                 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
 140                         /*
 141                          * Requesting strategy transaction from buffer-cache,
 142                          * or a VM getpages/putpages through the buffer cache.
 143                          * We must allow such transactions in all situations
 144                          * to avoid deadlocks.
 145                          */
 146                         nflags = (oflags | flags) + 1;
 147                 } else {
 148                         /*
 149                          * Normal transaction.  We do not interlock against
 150                          * BUFCACHE or ISFLUSH.
 151                          *
 152                          * Note that vnode locks may be held going into
 153                          * this call.
 154                          *
 155                          * NOTE: Remember that non-modifying operations
 156                          *       such as read, stat, readdir, etc, do
 157                          *       not use transactions.
 158                          */
 159                         nflags = (oflags | flags) + 1;
 160                 }
 161                 if (dowait)
 162                         tsleep_interlock(&pmp->trans.sync_wait, 0);
 163                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 164                         if (dowait == 0)
 165                                 break;
 166                         tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
 167                                "h2trans", hz);
 168                         /* retry */
 169                 } else {
 170                         cpu_pause();
 171                         /* retry */
 172                 }
 173                 /* retry */
 174         }
 175
 176 #if 0
 177         /*
 178          * When entering a FLUSH transaction with COPYQ set, wait for the
 179          * transaction count to drop to 1 (our flush transaction only)
 180          * before proceeding.
 181          *
 182          * This waits for all non-flush transactions to complete and blocks
 183          * new non-flush transactions from starting until COPYQ is cleared.
 184          * (the flush will then proceed after clearing COPYQ).  This should
 185          * be a very short stall on modifying operations.
 186          */
 187         while ((flags & HAMMER2_TRANS_ISFLUSH) &&
 188                (flags & HAMMER2_TRANS_COPYQ)) {
 189                 oflags = pmp->trans.flags;
 190                 cpu_ccfence();
 191                 if ((oflags & HAMMER2_TRANS_MASK) == 1)
 192                         break;
 193                 nflags = oflags | HAMMER2_TRANS_WAITING;
 194                 tsleep_interlock(&pmp->trans.sync_wait, 0);
 195                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 196                         tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
 197                                "h2trans2", hz);
 198                 }
 199         }
 200 #endif
 201 }
 202
 203 /*
 204  * Start a sub-transaction, there is no 'subdone' function.  This will
 205  * issue a new modify_tid (mtid) for the current transaction, which is a
 206  * CLC (cluster level change) id and not a per-node id.
 207  *
 208  * This function must be called for each XOP when multiple XOPs are run in
 209  * sequence within a transaction.
 210  *
 211  * Callers typically update the inode with the transaction mtid manually
 212  * to enforce sequencing.
 213  */
 214 hammer2_tid_t
 215 hammer2_trans_sub(hammer2_pfs_t *pmp)
 216 {
 217         hammer2_tid_t mtid;
 218
 219         mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
 220
 221         return (mtid);
 222 }
 223
 224 void
 225 hammer2_trans_setflags(hammer2_pfs_t *pmp, uint32_t flags)
 226 {
 227         atomic_set_int(&pmp->trans.flags, flags);
 228 }
 229
 230 /*
 231  * Typically used to clear trans flags asynchronously.  If TRANS_WAITING
 232  * is in the mask, and was previously set, this function will wake up
 233  * any waiters.
 234  */
 235 void
 236 hammer2_trans_clearflags(hammer2_pfs_t *pmp, uint32_t flags)
 237 {
 238         uint32_t oflags;
 239         uint32_t nflags;
 240
 241         for (;;) {
 242                 oflags = pmp->trans.flags;
 243                 cpu_ccfence();
 244                 nflags = oflags & ~flags;
 245                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 246                         if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING)
 247                                 wakeup(&pmp->trans.sync_wait);
 248                         break;
 249                 }
 250                 cpu_pause();
 251                 /* retry */
 252         }
 253 }
 254
 255 void
 256 hammer2_trans_done(hammer2_pfs_t *pmp, uint32_t flags)
 257 {
 258         uint32_t oflags;
 259         uint32_t nflags;
 260
 261 #if 0
 262         /*
 263          * Modifying ops on the front-end can cause dirty inodes to
 264          * build up in the sideq.  We don't flush these on inactive/reclaim
 265          * due to potential deadlocks, so we have to deal with them from
 266          * inside other nominal modifying front-end transactions.
 267          */
 268         if ((flags & HAMMER2_TRANS_SIDEQ) &&
 269             pmp->sideq_count > hammer2_limit_dirty_inodes / 2 &&
 270             pmp->sideq_count > (pmp->inum_count >> 3) &&
 271             pmp->mp) {
 272                 speedup_syncer(pmp->mp);
 273         }
 274 #endif
 275
 276         /*
 277          * Clean-up the transaction.  Wakeup any waiters when finishing
 278          * a flush transaction or transitioning the non-flush transaction
 279          * count from 2->1 while a flush transaction is pending.
 280          */
 281         for (;;) {
 282                 oflags = pmp->trans.flags;
 283                 cpu_ccfence();
 284                 KKASSERT(oflags & HAMMER2_TRANS_MASK);
 285
 286                 nflags = (oflags - 1) & ~flags;
 287                 if (flags & HAMMER2_TRANS_ISFLUSH) {
 288                         nflags &= ~HAMMER2_TRANS_WAITING;
 289                 }
 290                 if ((oflags & (HAMMER2_TRANS_ISFLUSH|HAMMER2_TRANS_MASK)) ==
 291                     (HAMMER2_TRANS_ISFLUSH|2)) {
 292                         nflags &= ~HAMMER2_TRANS_WAITING;
 293                 }
 294                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 295                         if ((oflags ^ nflags) & HAMMER2_TRANS_WAITING)
 296                                 wakeup(&pmp->trans.sync_wait);
 297                         break;
 298                 }
 299                 cpu_pause();
 300                 /* retry */
 301         }
 302 }
 303
 304 /*
 305  * Obtain new, unique inode number (not serialized by caller).
 306  */
 307 hammer2_tid_t
 308 hammer2_trans_newinum(hammer2_pfs_t *pmp)
 309 {
 310         hammer2_tid_t tid;
 311
 312         tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
 313
 314         return tid;
 315 }
 316
 317 /*
 318  * Assert that a strategy call is ok here.  Currently we allow strategy
 319  * calls in all situations, including during flushes.  Previously:
 320  *      (old) (1) In a normal transaction.
 321  */
 322 void
 323 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
 324 {
 325 #if 0
 326         KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0);
 327 #endif
 328 }
 329
 330 /*
 331  * Flush the chain and all modified sub-chains through the specified
 332  * synchronization point, propagating blockref updates back up.  As
 333  * part of this propagation, mirror_tid and inode/data usage statistics
 334  * propagates back upward.
 335  *
 336  * Returns a HAMMER2 error code, 0 if no error.  Note that I/O errors from
 337  * buffers dirtied during the flush operation can occur later.
 338  *
 339  * modify_tid (clc - cluster level change) is not propagated.
 340  *
 341  * update_tid (clc) is used for validation and is not propagated by this
 342  * function.
 343  *
 344  * This routine can be called from several places but the most important
 345  * is from VFS_SYNC (frontend) via hammer2_xop_inode_flush (backend).
 346  *
 347  * chain is locked on call and will remain locked on return.  The chain's
 348  * UPDATE flag indicates that its parent's block table (which is not yet
 349  * part of the flush) should be updated.
 350  *
 351  * flags:
 352  *      HAMMER2_FLUSH_TOP       Indicates that this is the top of the flush.
 353  *                              Is cleared for the recursion.
 354  *
 355  *      HAMMER2_FLUSH_ALL       Recurse everything
 356  *
 357  *      HAMMER2_FLUSH_INODE_STOP
 358  *                              Stop at PFS inode or normal inode boundary
 359  */
 360 int
 361 hammer2_flush(hammer2_chain_t *chain, int flags)
 362 {
 363         hammer2_flush_info_t info;
 364         hammer2_dev_t *hmp;
 365         int loops;
 366
 367         /*
 368          * Execute the recursive flush and handle deferrals.
 369          *
 370          * Chains can be ridiculously long (thousands deep), so to
 371          * avoid blowing out the kernel stack the recursive flush has a
 372          * depth limit.  Elements at the limit are placed on a list
 373          * for re-execution after the stack has been popped.
 374          */
 375         bzero(&info, sizeof(info));
 376         info.flags = flags & ~HAMMER2_FLUSH_TOP;
 377
 378         /*
 379          * Calculate parent (can be NULL), if not NULL the flush core
 380          * expects the parent to be referenced so it can easily lock/unlock
 381          * it without it getting ripped up.
 382          */
 383         if ((info.parent = chain->parent) != NULL)
 384                 hammer2_chain_ref(info.parent);
 385
 386         /*
 387          * Extra ref needed because flush_core expects it when replacing
 388          * chain.
 389          */
 390         hammer2_chain_ref(chain);
 391         hmp = chain->hmp;
 392         loops = 0;
 393
 394         for (;;) {
 395                 /*
 396                  * [re]flush chain as the deep recursion may have generated
 397                  * additional modifications.
 398                  */
 399                 if (info.parent != chain->parent) {
 400                         if (hammer2_debug & 0x0040) {
 401                                 kprintf("LOST CHILD4 %p->%p "
 402                                         "(actual parent %p)\n",
 403                                         info.parent, chain, chain->parent);
 404                         }
 405                         hammer2_chain_drop(info.parent);
 406                         info.parent = chain->parent;
 407                         hammer2_chain_ref(info.parent);
 408                 }
 409                 if (hammer2_flush_core(&info, chain, flags) == 0)
 410                         break;
 411
 412                 if (++loops % 1000 == 0) {
 413                         kprintf("hammer2_flush: excessive loops on %p\n",
 414                                 chain);
 415                         if (hammer2_debug & 0x100000)
 416                                 Debugger("hell4");
 417                 }
 418         }
 419 #ifdef HAMMER2_SCAN_DEBUG
 420         if (info.scan_count >= 10)
 421         kprintf("hammer2_flush: scan_count %ld (%ld,%ld,%ld,%ld) "
 422                 "bt(%ld,%ld,%ld,%ld,%ld,%ld)\n",
 423                 info.scan_count,
 424                 info.scan_mod_count,
 425                 info.scan_upd_count,
 426                 info.scan_onf_count,
 427                 info.scan_del_count,
 428                 info.scan_btype[1],
 429                 info.scan_btype[2],
 430                 info.scan_btype[3],
 431                 info.scan_btype[4],
 432                 info.scan_btype[5],
 433                 info.scan_btype[6]);
 434 #endif
 435         hammer2_chain_drop(chain);
 436         if (info.parent)
 437                 hammer2_chain_drop(info.parent);
 438         return (info.error);
 439 }
 440
 441 /*
 442  * This is the core of the chain flushing code.  The chain is locked by the
 443  * caller and must also have an extra ref on it by the caller, and remains
 444  * locked and will have an extra ref on return.  info.parent is referenced
 445  * but not locked.
 446  *
 447  * Upon return, the caller can test the UPDATE bit on the chain to determine
 448  * if the parent needs updating.
 449  *
 450  * If non-zero is returned, the chain's parent changed during the flush and
 451  * the caller must retry the operation.
 452  *
 453  * (1) Determine if this node is a candidate for the flush, return if it is
 454  *     not.  fchain and vchain are always candidates for the flush.
 455  *
 456  * (2) If we recurse too deep the chain is entered onto the deferral list and
 457  *     the current flush stack is aborted until after the deferral list is
 458  *     run.
 459  *
 460  * (3) Recursively flush live children (rbtree).  This can create deferrals.
 461  *     A successful flush clears the MODIFIED and UPDATE bits on the children
 462  *     and typically causes the parent to be marked MODIFIED as the children
 463  *     update the parent's block table.  A parent might already be marked
 464  *     MODIFIED due to a deletion (whos blocktable update in the parent is
 465  *     handled by the frontend), or if the parent itself is modified by the
 466  *     frontend for other reasons.
 467  *
 468  * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
 469  *     Deleted-but-open inodes can still be individually flushed via the
 470  *     filesystem syncer.
 471  *
 472  * (5) Delete parents on the way back up if they are normal indirect blocks
 473  *     and have no children.
 474  *
 475  * (6) Note that an unmodified child may still need the block table in its
 476  *     parent updated (e.g. rename/move).  The child will have UPDATE set
 477  *     in this case.
 478  *
 479  *                      WARNING ON BREF MODIFY_TID/MIRROR_TID
 480  *
 481  * blockref.modify_tid is consistent only within a PFS, and will not be
 482  * consistent during synchronization.  mirror_tid is consistent across the
 483  * block device regardless of the PFS.
 484  */
 485 static int
 486 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
 487                    int flags)
 488 {
 489         hammer2_chain_t *parent;
 490         hammer2_dev_t *hmp;
 491         int save_error;
 492         int retry;
 493
 494         retry = 0;
 495
 496         /*
 497          * (1) Optimize downward recursion to locate nodes needing action.
 498          *     Nothing to do if none of these flags are set.
 499          */
 500         if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0)
 501                 return 0;
 502
 503         hmp = chain->hmp;
 504
 505         /*
 506          * NOTE: parent can be NULL, usually due to destroy races.
 507          */
 508         parent = info->parent;
 509         KKASSERT(chain->parent == parent);
 510
 511         /*
 512          * Downward search recursion
 513          *
 514          * We must be careful on cold stops, which often occur on inode
 515          * boundaries due to the way hammer2_vfs_sync() sequences the flush.
 516          * Be sure to issue an appropriate chain_setflush()
 517          */
 518         if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) &&
 519             (flags & HAMMER2_FLUSH_ALL) == 0 &&
 520             (flags & HAMMER2_FLUSH_TOP) == 0 &&
 521             chain->pmp && chain->pmp->mp) {
 522                 /*
 523                  * If FLUSH_ALL is not specified the caller does not want
 524                  * to recurse through PFS roots that have been mounted.
 525                  *
 526                  * (If the PFS has not been mounted there may not be
 527                  *  anything monitoring its chains and its up to us
 528                  *  to flush it).
 529                  *
 530                  * The typical sequence is to flush dirty PFS's starting at
 531                  * their root downward, then flush the device root (vchain).
 532                  * It is this second flush that typically leaves out the
 533                  * ALL flag.
 534                  *
 535                  * However we must still process the PFSROOT chains for block
 536                  * table updates in their parent (which IS part of our flush).
 537                  *
 538                  * NOTE: The volume root, vchain, does not set PFSBOUNDARY.
 539                  *
 540                  * NOTE: We must re-set ONFLUSH in the parent to retain if
 541                  *       this chain (that we are skipping) requires work.
 542                  */
 543                 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
 544                                     HAMMER2_CHAIN_DESTROY |
 545                                     HAMMER2_CHAIN_MODIFIED)) {
 546                         hammer2_chain_setflush(parent);
 547                 }
 548                 goto done;
 549         } else if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
 550                    (flags & HAMMER2_FLUSH_INODE_STOP) &&
 551                    (flags & HAMMER2_FLUSH_ALL) == 0 &&
 552                    (flags & HAMMER2_FLUSH_TOP) == 0 &&
 553                    chain->pmp && chain->pmp->mp) {
 554                 /*
 555                  * When FLUSH_INODE_STOP is specified we are being asked not
 556                  * to include any inode changes for inodes we encounter,
 557                  * with the exception of the inode that the flush began with.
 558                  * So: INODE, INODE_STOP, and TOP==0 basically.
 559                  *
 560                  * Dirty inodes are flushed based on the hammer2_inode
 561                  * in-memory structure, issuing a chain_setflush() here
 562                  * will only cause unnecessary traversals of the topology.
 563                  */
 564                 goto done;
 565 #if 0
 566                 /*
 567                  * If FLUSH_INODE_STOP is specified and both ALL and TOP
 568                  * are clear, we must not flush the chain.  The chain should
 569                  * have already been flushed and any further ONFLUSH/UPDATE
 570                  * setting will be related to the next flush.
 571                  *
 572                  * This features allows us to flush inodes independently of
 573                  * each other and meta-data above the inodes separately.
 574                  */
 575                 if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
 576                                     HAMMER2_CHAIN_DESTROY |
 577                                     HAMMER2_CHAIN_MODIFIED)) {
 578                         if (parent)
 579                                 hammer2_chain_setflush(parent);
 580                 }
 581 #endif
 582         } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
 583                 /*
 584                  * Recursion depth reached.
 585                  */
 586                 panic("hammer2: flush depth limit");
 587         } else if (chain->flags & (HAMMER2_CHAIN_ONFLUSH |
 588                                    HAMMER2_CHAIN_DESTROY)) {
 589                 /*
 590                  * Downward recursion search (actual flush occurs bottom-up).
 591                  * pre-clear ONFLUSH.  It can get set again due to races or
 592                  * flush errors, which we want so the scan finds us again in
 593                  * the next flush.
 594                  *
 595                  * We must also recurse if DESTROY is set so we can finally
 596                  * get rid of the related children, otherwise the node will
 597                  * just get re-flushed on lastdrop.
 598                  *
 599                  * WARNING!  The recursion will unlock/relock info->parent
 600                  *           (which is 'chain'), potentially allowing it
 601                  *           to be ripped up.
 602                  */
 603                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
 604                 save_error = info->error;
 605                 info->error = 0;
 606                 info->parent = chain;
 607
 608                 /*
 609                  * We may have to do this twice to catch any indirect
 610                  * block maintenance that occurs.
 611                  */
 612                 hammer2_spin_ex(&chain->core.spin);
 613                 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
 614                         NULL, hammer2_flush_recurse, info);
 615                 if (chain->flags & HAMMER2_CHAIN_ONFLUSH) {
 616                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
 617                         RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
 618                                 NULL, hammer2_flush_recurse, info);
 619                 }
 620                 hammer2_spin_unex(&chain->core.spin);
 621                 info->parent = parent;
 622
 623                 /*
 624                  * Re-set the flush bits if the flush was incomplete or
 625                  * an error occurred.  If an error occurs it is typically
 626                  * an allocation error.  Errors do not cause deferrals.
 627                  */
 628                 if (info->error)
 629                         hammer2_chain_setflush(chain);
 630                 info->error |= save_error;
 631
 632                 /*
 633                  * If we lost the parent->chain association we have to
 634                  * stop processing this chain because it is no longer
 635                  * in this recursion.  If it moved, it will be handled
 636                  * by the ONFLUSH flag elsewhere.
 637                  */
 638                 if (chain->parent != parent) {
 639                         kprintf("LOST CHILD2 %p->%p (actual parent %p)\n",
 640                                 parent, chain, chain->parent);
 641                         goto done;
 642                 }
 643         }
 644
 645         /*
 646          * Now we are in the bottom-up part of the recursion.
 647          *
 648          * We continue to try to update the chain on lower-level errors, but
 649          * the flush code may decide not to flush the volume root.
 650          *
 651          * XXX should we continue to try to update the chain if an error
 652          *     occurred?
 653          */
 654
 655         /*
 656          * Both parent and chain must be locked in order to flush chain,
 657          * in order to properly update the parent under certain conditions.
 658          *
 659          * In addition, we can't safely unlock/relock the chain once we
 660          * start flushing the chain itself, which we would have to do later
 661          * on in order to lock the parent if we didn't do that now.
 662          */
 663         hammer2_chain_ref_hold(chain);
 664         hammer2_chain_unlock(chain);
 665         if (parent)
 666                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
 667         hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
 668         hammer2_chain_drop_unhold(chain);
 669
 670         /*
 671          * Can't process if we can't access their content.
 672          */
 673         if ((parent && parent->error) || chain->error) {
 674                 kprintf("hammer2: chain error during flush\n");
 675                 info->error |= chain->error;
 676                 if (parent) {
 677                         info->error |= parent->error;
 678                         hammer2_chain_unlock(parent);
 679                 }
 680                 goto done;
 681         }
 682
 683         if (chain->parent != parent) {
 684                 if (hammer2_debug & 0x0040) {
 685                         kprintf("LOST CHILD3 %p->%p (actual parent %p)\n",
 686                                 parent, chain, chain->parent);
 687                 }
 688                 KKASSERT(parent != NULL);
 689                 hammer2_chain_unlock(parent);
 690                 retry = 1;
 691                 goto done;
 692         }
 693
 694         /*
 695          * Propagate the DESTROY flag downwards.  This dummies up the flush
 696          * code and tries to invalidate related buffer cache buffers to
 697          * avoid the disk write.
 698          */
 699         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
 700                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
 701
 702         /*
 703          * Dispose of the modified bit.
 704          *
 705          * If parent is present, the UPDATE bit should already be set.
 706          * UPDATE should already be set.
 707          * bref.mirror_tid should already be set.
 708          */
 709         if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
 710                 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
 711                          chain->parent == NULL);
 712                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
 713                 atomic_add_long(&hammer2_count_modified_chains, -1);
 714
 715                 /*
 716                  * Manage threads waiting for excessive dirty memory to
 717                  * be retired.
 718                  */
 719                 if (chain->pmp)
 720                         hammer2_pfs_memory_wakeup(chain->pmp, -1);
 721
 722 #if 0
 723                 if ((chain->flags & HAMMER2_CHAIN_UPDATE) == 0 &&
 724                     chain != &hmp->vchain &&
 725                     chain != &hmp->fchain) {
 726                         /*
 727                          * Set UPDATE bit indicating that the parent block
 728                          * table requires updating.
 729                          */
 730                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 731                 }
 732 #endif
 733
 734                 /*
 735                  * Issue the flush.  This is indirect via the DIO.
 736                  *
 737                  * NOTE: A DELETED node that reaches this point must be
 738                  *       flushed for synchronization point consistency.
 739                  *
 740                  * NOTE: Even though MODIFIED was already set, the related DIO
 741                  *       might not be dirty due to a system buffer cache
 742                  *       flush and must be set dirty if we are going to make
 743                  *       further modifications to the buffer.  Chains with
 744                  *       embedded data don't need this.
 745                  */
 746                 if (hammer2_debug & 0x1000) {
 747                         kprintf("Flush %p.%d %016jx/%d data=%016jx\n",
 748                                 chain, chain->bref.type,
 749                                 (uintmax_t)chain->bref.key,
 750                                 chain->bref.keybits,
 751                                 (uintmax_t)chain->bref.data_off);
 752                 }
 753
 754                 /*
 755                  * Update chain CRCs for flush.
 756                  *
 757                  * NOTE: Volume headers are NOT flushed here as they require
 758                  *       special processing.
 759                  */
 760                 switch(chain->bref.type) {
 761                 case HAMMER2_BREF_TYPE_FREEMAP:
 762                         /*
 763                          * Update the volume header's freemap_tid to the
 764                          * freemap's flushing mirror_tid.
 765                          *
 766                          * (note: embedded data, do not call setdirty)
 767                          */
 768                         KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
 769                         KKASSERT(chain == &hmp->fchain);
 770                         hmp->voldata.freemap_tid = chain->bref.mirror_tid;
 771                         if (hammer2_debug & 0x8000) {
 772                                 /* debug only, avoid syslogd loop */
 773                                 kprintf("sync freemap mirror_tid %08jx\n",
 774                                         (intmax_t)chain->bref.mirror_tid);
 775                         }
 776
 777                         /*
 778                          * The freemap can be flushed independently of the
 779                          * main topology, but for the case where it is
 780                          * flushed in the same transaction, and flushed
 781                          * before vchain (a case we want to allow for
 782                          * performance reasons), make sure modifications
 783                          * made during the flush under vchain use a new
 784                          * transaction id.
 785                          *
 786                          * Otherwise the mount recovery code will get confused.
 787                          */
 788                         ++hmp->voldata.mirror_tid;
 789                         break;
 790                 case HAMMER2_BREF_TYPE_VOLUME:
 791                         /*
 792                          * The free block table is flushed by
 793                          * hammer2_vfs_sync() before it flushes vchain.
 794                          * We must still hold fchain locked while copying
 795                          * voldata to volsync, however.
 796                          *
 797                          * These do not error per-say since their data does
 798                          * not need to be re-read from media on lock.
 799                          *
 800                          * (note: embedded data, do not call setdirty)
 801                          */
 802                         hammer2_chain_lock(&hmp->fchain,
 803                                            HAMMER2_RESOLVE_ALWAYS);
 804                         hammer2_voldata_lock(hmp);
 805                         if (hammer2_debug & 0x8000) {
 806                                 /* debug only, avoid syslogd loop */
 807                                 kprintf("sync volume  mirror_tid %08jx\n",
 808                                         (intmax_t)chain->bref.mirror_tid);
 809                         }
 810
 811                         /*
 812                          * Update the volume header's mirror_tid to the
 813                          * main topology's flushing mirror_tid.  It is
 814                          * possible that voldata.mirror_tid is already
 815                          * beyond bref.mirror_tid due to the bump we made
 816                          * above in BREF_TYPE_FREEMAP.
 817                          */
 818                         if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
 819                                 hmp->voldata.mirror_tid =
 820                                         chain->bref.mirror_tid;
 821                         }
 822
 823                         /*
 824                          * The volume header is flushed manually by the
 825                          * syncer, not here.  All we do here is adjust the
 826                          * crc's.
 827                          */
 828                         KKASSERT(chain->data != NULL);
 829                         KKASSERT(chain->dio == NULL);
 830
 831                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
 832                                 hammer2_icrc32(
 833                                         (char *)&hmp->voldata +
 834                                          HAMMER2_VOLUME_ICRC1_OFF,
 835                                         HAMMER2_VOLUME_ICRC1_SIZE);
 836                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
 837                                 hammer2_icrc32(
 838                                         (char *)&hmp->voldata +
 839                                          HAMMER2_VOLUME_ICRC0_OFF,
 840                                         HAMMER2_VOLUME_ICRC0_SIZE);
 841                         hmp->voldata.icrc_volheader =
 842                                 hammer2_icrc32(
 843                                         (char *)&hmp->voldata +
 844                                          HAMMER2_VOLUME_ICRCVH_OFF,
 845                                         HAMMER2_VOLUME_ICRCVH_SIZE);
 846
 847                         if (hammer2_debug & 0x8000) {
 848                                 /* debug only, avoid syslogd loop */
 849                                 kprintf("syncvolhdr %016jx %016jx\n",
 850                                         hmp->voldata.mirror_tid,
 851                                         hmp->vchain.bref.mirror_tid);
 852                         }
 853                         hmp->volsync = hmp->voldata;
 854                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
 855                         hammer2_voldata_unlock(hmp);
 856                         hammer2_chain_unlock(&hmp->fchain);
 857                         break;
 858                 case HAMMER2_BREF_TYPE_DATA:
 859                         /*
 860                          * Data elements have already been flushed via the
 861                          * logical file buffer cache.  Their hash was set in
 862                          * the bref by the vop_write code.  Do not re-dirty.
 863                          *
 864                          * Make sure any device buffer(s) have been flushed
 865                          * out here (there aren't usually any to flush) XXX.
 866                          */
 867                         break;
 868                 case HAMMER2_BREF_TYPE_INDIRECT:
 869                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
 870                 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
 871                         /*
 872                          * Buffer I/O will be cleaned up when the volume is
 873                          * flushed (but the kernel is free to flush it before
 874                          * then, as well).
 875                          */
 876                         hammer2_chain_setcheck(chain, chain->data);
 877                         break;
 878                 case HAMMER2_BREF_TYPE_DIRENT:
 879                         /*
 880                          * A directory entry can use the check area to store
 881                          * the filename for filenames <= 64 bytes, don't blow
 882                          * it up!
 883                          */
 884                         if (chain->bytes)
 885                                 hammer2_chain_setcheck(chain, chain->data);
 886                         break;
 887                 case HAMMER2_BREF_TYPE_INODE:
 888                         /*
 889                          * NOTE: We must call io_setdirty() to make any late
 890                          *       changes to the inode data, the system might
 891                          *       have already flushed the buffer.
 892                          */
 893                         if (chain->data->ipdata.meta.op_flags &
 894                             HAMMER2_OPFLAG_PFSROOT) {
 895                                 /*
 896                                  * non-NULL pmp if mounted as a PFS.  We must
 897                                  * sync fields cached in the pmp? XXX
 898                                  */
 899                                 hammer2_inode_data_t *ipdata;
 900
 901                                 hammer2_io_setdirty(chain->dio);
 902                                 ipdata = &chain->data->ipdata;
 903                                 if (chain->pmp) {
 904                                         ipdata->meta.pfs_inum =
 905                                                 chain->pmp->inode_tid;
 906                                 }
 907                         } else {
 908                                 /* can't be mounted as a PFS */
 909                         }
 910
 911                         hammer2_chain_setcheck(chain, chain->data);
 912                         break;
 913                 default:
 914                         panic("hammer2_flush_core: unsupported "
 915                               "embedded bref %d",
 916                               chain->bref.type);
 917                         /* NOT REACHED */
 918                 }
 919
 920                 /*
 921                  * If the chain was destroyed try to avoid unnecessary I/O
 922                  * that might not have yet occurred.  Remove the data range
 923                  * from dedup candidacy and attempt to invalidation that
 924                  * potentially dirty portion of the I/O buffer.
 925                  */
 926                 if (chain->flags & HAMMER2_CHAIN_DESTROY) {
 927                         hammer2_io_dedup_delete(hmp,
 928                                                 chain->bref.type,
 929                                                 chain->bref.data_off,
 930                                                 chain->bytes);
 931 #if 0
 932                         hammer2_io_t *dio;
 933                         if (chain->dio) {
 934                                 hammer2_io_inval(chain->dio,
 935                                                  chain->bref.data_off,
 936                                                  chain->bytes);
 937                         } else if ((dio = hammer2_io_getquick(hmp,
 938                                                   chain->bref.data_off,
 939                                                   chain->bytes,
 940                                                   1)) != NULL) {
 941                                 hammer2_io_inval(dio,
 942                                                  chain->bref.data_off,
 943                                                  chain->bytes);
 944                                 hammer2_io_putblk(&dio);
 945                         }
 946 #endif
 947                 }
 948         }
 949
 950         /*
 951          * If UPDATE is set the parent block table may need to be updated.
 952          * This can fail if the hammer2_chain_modify() fails.
 953          *
 954          * NOTE: UPDATE may be set on vchain or fchain in which case
 955          *       parent could be NULL, or on an inode that has not yet
 956          *       been inserted into the radix tree.  It's easiest to allow
 957          *       the case and test for NULL.  parent can also wind up being
 958          *       NULL due to a deletion so we need to handle the case anyway.
 959          *
 960          * NOTE: UPDATE can be set when chains are renamed into or out of
 961          *       an indirect block, without the chain itself being flagged
 962          *       MODIFIED.
 963          *
 964          * If no parent exists we can just clear the UPDATE bit.  If the
 965          * chain gets reattached later on the bit will simply get set
 966          * again.
 967          */
 968         if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL)
 969                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 970
 971         /*
 972          * When flushing an inode outside of a FLUSH_FSSYNC we must NOT
 973          * update the parent block table to point at the flushed inode.
 974          * The block table should only ever be updated by the filesystem
 975          * sync code.  If we do, inode<->inode dependencies (such as
 976          * directory entries vs inode nlink count) can wind up not being
 977          * flushed together and result in a broken topology if a crash/reboot
 978          * occurs at the wrong time.
 979          */
 980         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE &&
 981             (flags & HAMMER2_FLUSH_INODE_STOP) &&
 982             (flags & HAMMER2_FLUSH_FSSYNC) == 0 &&
 983             (flags & HAMMER2_FLUSH_ALL) == 0 &&
 984             chain->pmp && chain->pmp->mp) {
 985 #ifdef HAMMER2_DEBUG_SYNC
 986                 kprintf("inum %ld do not update parent, non-fssync\n",
 987                         (long)chain->bref.key);
 988 #endif
 989                 goto skipupdate;
 990         }
 991 #ifdef HAMMER2_DEBUG_SYNC
 992         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE)
 993                 kprintf("inum %ld update parent\n", (long)chain->bref.key);
 994 #endif
 995
 996         /*
 997          * The chain may need its blockrefs updated in the parent, normal
 998          * path.
 999          */
1000         if (chain->flags & HAMMER2_CHAIN_UPDATE) {
1001                 hammer2_blockref_t *base;
1002                 int count;
1003
1004                 /*
1005                  * Clear UPDATE flag, mark parent modified, update its
1006                  * modify_tid if necessary, and adjust the parent blockmap.
1007                  */
1008                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1009
1010                 /*
1011                  * (optional code)
1012                  *
1013                  * Avoid actually modifying and updating the parent if it
1014                  * was flagged for destruction.  This can greatly reduce
1015                  * disk I/O in large tree removals because the
1016                  * hammer2_io_setinval() call in the upward recursion
1017                  * (see MODIFIED code above) can only handle a few cases.
1018                  */
1019                 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
1020                         if (parent->bref.modify_tid < chain->bref.modify_tid) {
1021                                 parent->bref.modify_tid =
1022                                         chain->bref.modify_tid;
1023                         }
1024                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BLKMAPPED |
1025                                                         HAMMER2_CHAIN_BLKMAPUPD);
1026                         goto skipupdate;
1027                 }
1028
1029                 /*
1030                  * The flusher is responsible for deleting empty indirect
1031                  * blocks at this point.  If we don't do this, no major harm
1032                  * will be done but the empty indirect blocks will stay in
1033                  * the topology and make it a messy and inefficient.
1034                  *
1035                  * The flusher is also responsible for collapsing the
1036                  * content of an indirect block into its parent whenever
1037                  * possible (with some hysteresis).  Not doing this will also
1038                  * not harm the topology, but would make it messy and
1039                  * inefficient.
1040                  */
1041                 if (chain->bref.type == HAMMER2_BREF_TYPE_INDIRECT) {
1042                         if (hammer2_chain_indirect_maintenance(parent, chain))
1043                                 goto skipupdate;
1044                 }
1045
1046                 /*
1047                  * We are updating the parent's blockmap, the parent must
1048                  * be set modified.  If this fails we re-set the UPDATE flag
1049                  * in the child.
1050                  *
1051                  * NOTE! A modification error can be ENOSPC.  We still want
1052                  *       to flush modified chains recursively, not break out,
1053                  *       so we just skip the update in this situation and
1054                  *       continue.  That is, we still need to try to clean
1055                  *       out dirty chains and buffers.
1056                  *
1057                  *       This may not help bulkfree though. XXX
1058                  */
1059                 save_error = hammer2_chain_modify(parent, 0, 0, 0);
1060                 if (save_error) {
1061                         info->error |= save_error;
1062                         kprintf("hammer2_flush: %016jx.%02x error=%08x\n",
1063                                 parent->bref.data_off, parent->bref.type,
1064                                 save_error);
1065                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
1066                         goto skipupdate;
1067                 }
1068                 if (parent->bref.modify_tid < chain->bref.modify_tid)
1069                         parent->bref.modify_tid = chain->bref.modify_tid;
1070
1071                 /*
1072                  * Calculate blockmap pointer
1073                  */
1074                 switch(parent->bref.type) {
1075                 case HAMMER2_BREF_TYPE_INODE:
1076                         /*
1077                          * Access the inode's block array.  However, there is
1078                          * no block array if the inode is flagged DIRECTDATA.
1079                          */
1080                         if (parent->data &&
1081                             (parent->data->ipdata.meta.op_flags &
1082                              HAMMER2_OPFLAG_DIRECTDATA) == 0) {
1083                                 base = &parent->data->
1084                                         ipdata.u.blockset.blockref[0];
1085                         } else {
1086                                 base = NULL;
1087                         }
1088                         count = HAMMER2_SET_COUNT;
1089                         break;
1090                 case HAMMER2_BREF_TYPE_INDIRECT:
1091                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
1092                         if (parent->data)
1093                                 base = &parent->data->npdata[0];
1094                         else
1095                                 base = NULL;
1096                         count = parent->bytes / sizeof(hammer2_blockref_t);
1097                         break;
1098                 case HAMMER2_BREF_TYPE_VOLUME:
1099                         base = &chain->hmp->voldata.sroot_blockset.blockref[0];
1100                         count = HAMMER2_SET_COUNT;
1101                         break;
1102                 case HAMMER2_BREF_TYPE_FREEMAP:
1103                         base = &parent->data->npdata[0];
1104                         count = HAMMER2_SET_COUNT;
1105                         break;
1106                 default:
1107                         base = NULL;
1108                         count = 0;
1109                         panic("hammer2_flush_core: "
1110                               "unrecognized blockref type: %d",
1111                               parent->bref.type);
1112                         break;
1113                 }
1114
1115                 /*
1116                  * Blocktable updates
1117                  *
1118                  * We synchronize pending statistics at this time.  Delta
1119                  * adjustments designated for the current and upper level
1120                  * are synchronized.
1121                  */
1122                 if (base && (chain->flags & HAMMER2_CHAIN_BLKMAPUPD)) {
1123                         if (chain->flags & HAMMER2_CHAIN_BLKMAPPED) {
1124                                 hammer2_spin_ex(&parent->core.spin);
1125                                 hammer2_base_delete(parent, base, count, chain,
1126                                                     NULL);
1127                                 hammer2_spin_unex(&parent->core.spin);
1128                                 /* base_delete clears both bits */
1129                         } else {
1130                                 atomic_clear_int(&chain->flags,
1131                                                  HAMMER2_CHAIN_BLKMAPUPD);
1132                         }
1133                 }
1134                 if (base && (chain->flags & HAMMER2_CHAIN_BLKMAPPED) == 0) {
1135                         hammer2_spin_ex(&parent->core.spin);
1136                         hammer2_base_insert(parent, base, count,
1137                                             chain, &chain->bref);
1138                         hammer2_spin_unex(&parent->core.spin);
1139                         /* base_insert sets BLKMAPPED */
1140                 }
1141         }
1142 skipupdate:
1143         if (parent)
1144                 hammer2_chain_unlock(parent);
1145
1146         /*
1147          * Final cleanup after flush
1148          */
1149 done:
1150         KKASSERT(chain->refs > 0);
1151
1152         return retry;
1153 }
1154
1155 /*
1156  * Flush recursion helper, called from flush_core, calls flush_core.
1157  *
1158  * Flushes the children of the caller's chain (info->parent), restricted
1159  * by sync_tid.
1160  *
1161  * This function may set info->error as a side effect.
1162  *
1163  * WARNING! If we do not call hammer2_flush_core() we must update
1164  *          bref.mirror_tid ourselves to indicate that the flush has
1165  *          processed the child.
1166  *
1167  * WARNING! parent->core spinlock is held on entry and return.
1168  */
1169 static int
1170 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
1171 {
1172         hammer2_flush_info_t *info = data;
1173         hammer2_chain_t *parent = info->parent;
1174
1175 #ifdef HAMMER2_SCAN_DEBUG
1176         ++info->scan_count;
1177         if (child->flags & HAMMER2_CHAIN_MODIFIED)
1178                 ++info->scan_mod_count;
1179         if (child->flags & HAMMER2_CHAIN_UPDATE)
1180                 ++info->scan_upd_count;
1181         if (child->flags & HAMMER2_CHAIN_ONFLUSH)
1182                 ++info->scan_onf_count;
1183 #endif
1184
1185         /*
1186          * (child can never be fchain or vchain so a special check isn't
1187          *  needed).
1188          *
1189          * We must ref the child before unlocking the spinlock.
1190          *
1191          * The caller has added a ref to the parent so we can temporarily
1192          * unlock it in order to lock the child.  However, if it no longer
1193          * winds up being the child of the parent we must skip this child.
1194          *
1195          * NOTE! chain locking errors are fatal.  They are never out-of-space
1196          *       errors.
1197          */
1198         hammer2_chain_ref(child);
1199         hammer2_spin_unex(&parent->core.spin);
1200
1201         hammer2_chain_ref_hold(parent);
1202         hammer2_chain_unlock(parent);
1203         hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
1204         if (child->parent != parent) {
1205                 kprintf("LOST CHILD1 %p->%p (actual parent %p)\n",
1206                         parent, child, child->parent);
1207                 goto done;
1208         }
1209         if (child->error) {
1210                 kprintf("CHILD ERROR DURING FLUSH LOCK %p->%p\n",
1211                         parent, child);
1212                 info->error |= child->error;
1213                 goto done;
1214         }
1215
1216         /*
1217          * Must propagate the DESTROY flag downwards, otherwise the
1218          * parent could end up never being removed because it will
1219          * be requeued to the flusher if it survives this run due to
1220          * the flag.
1221          */
1222         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
1223                 atomic_set_int(&child->flags, HAMMER2_CHAIN_DESTROY);
1224 #ifdef HAMMER2_SCAN_DEBUG
1225         if (child->flags & HAMMER2_CHAIN_DESTROY)
1226                 ++info->scan_del_count;
1227 #endif
1228         /*
1229          * Special handling of the root inode.  Because the root inode
1230          * contains an index of all the inodes in the PFS in addition to
1231          * its normal directory entries, any flush that is not part of a
1232          * filesystem sync must only flush the directory entries, and not
1233          * anything else.
1234          *
1235          * The child might be an indirect block, but H2 guarantees that
1236          * the key-range will fully partition the inode index from the
1237          * directory entries so the case just works naturally.
1238          */
1239         if ((parent->bref.flags & HAMMER2_BREF_FLAG_PFSROOT) &&
1240             (child->flags & HAMMER2_CHAIN_DESTROY) == 0 &&
1241             parent->bref.type == HAMMER2_BREF_TYPE_INODE &&
1242             (info->flags & HAMMER2_FLUSH_FSSYNC) == 0) {
1243                 if ((child->bref.key & HAMMER2_DIRHASH_VISIBLE) == 0) {
1244                         if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1245                                 hammer2_chain_setflush(parent);
1246                         }
1247                         goto done;
1248                 }
1249         }
1250
1251         /*
1252          * Recurse and collect deferral data.  We're in the media flush,
1253          * this can cross PFS boundaries.
1254          */
1255         if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1256 #ifdef HAMMER2_SCAN_DEBUG
1257                 if (child->bref.type < 7)
1258                         ++info->scan_btype[child->bref.type];
1259 #endif
1260                 ++info->depth;
1261                 hammer2_flush_core(info, child, info->flags);
1262                 --info->depth;
1263         }
1264
1265 done:
1266         /*
1267          * Relock to continue the loop.
1268          */
1269         hammer2_chain_unlock(child);
1270         hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1271         hammer2_chain_drop_unhold(parent);
1272         if (parent->error) {
1273                 kprintf("PARENT ERROR DURING FLUSH LOCK %p->%p\n",
1274                         parent, child);
1275                 info->error |= parent->error;
1276         }
1277         hammer2_chain_drop(child);
1278         KKASSERT(info->parent == parent);
1279         hammer2_spin_ex(&parent->core.spin);
1280
1281         return (0);
1282 }
1283
1284 /*
1285  * flush helper (backend threaded)
1286  *
1287  * Flushes chain topology for the specified inode.
1288  *
1289  * HAMMER2_XOP_INODE_STOP       The flush recursion stops at inode boundaries.
1290  *                              Inodes belonging to the same flush are flushed
1291  *                              separately.
1292  *
1293  * chain->parent can be NULL, usually due to destroy races or detached inodes.
1294  *
1295  * Primarily called from vfs_sync().
1296  */
1297 void
1298 hammer2_xop_inode_flush(hammer2_xop_t *arg, void *scratch __unused, int clindex)
1299 {
1300         hammer2_xop_flush_t *xop = &arg->xop_flush;
1301         hammer2_chain_t *chain;
1302         hammer2_inode_t *ip;
1303         hammer2_dev_t *hmp;
1304         hammer2_pfs_t *pmp;
1305         hammer2_devvp_t *e;
1306         struct vnode *devvp;
1307         int flush_error = 0;
1308         int fsync_error = 0;
1309         int total_error = 0;
1310         int j;
1311         int xflags;
1312         int ispfsroot = 0;
1313
1314         xflags = HAMMER2_FLUSH_TOP;
1315         if (xop->head.flags & HAMMER2_XOP_INODE_STOP)
1316                 xflags |= HAMMER2_FLUSH_INODE_STOP;
1317         if (xop->head.flags & HAMMER2_XOP_FSSYNC)
1318                 xflags |= HAMMER2_FLUSH_FSSYNC;
1319
1320         /*
1321          * Flush core chains
1322          */
1323         ip = xop->head.ip1;
1324         pmp = ip->pmp;
1325         chain = hammer2_inode_chain(ip, clindex, HAMMER2_RESOLVE_ALWAYS);
1326         if (chain) {
1327                 hmp = chain->hmp;
1328                 if (chain->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1329                         /*
1330                          * Due to flush partitioning the chain topology
1331                          * above the inode's chain may no longer be flagged.
1332                          * When asked to flush an inode, remark the topology
1333                          * leading to that inode.
1334                          */
1335                         if (chain->parent)
1336                                 hammer2_chain_setflush(chain->parent);
1337                         hammer2_flush(chain, xflags);
1338
1339                         /* XXX cluster */
1340                         if (ip == pmp->iroot && pmp != hmp->spmp) {
1341                                 hammer2_spin_ex(&pmp->inum_spin);
1342                                 pmp->pfs_iroot_blocksets[clindex] =
1343                                         chain->data->ipdata.u.blockset;
1344                                 hammer2_spin_unex(&pmp->inum_spin);
1345                         }
1346
1347 #if 0
1348                         /*
1349                          * Propogate upwards but only cross an inode boundary
1350                          * for inodes associated with the current filesystem
1351                          * sync.
1352                          */
1353                         if ((xop->head.flags & HAMMER2_XOP_PARENTONFLUSH) ||
1354                             chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
1355                                 parent = chain->parent;
1356                                 if (parent)
1357                                         hammer2_chain_setflush(parent);
1358                         }
1359 #endif
1360                 }
1361                 if (chain->flags & HAMMER2_CHAIN_PFSBOUNDARY)
1362                         ispfsroot = 1;
1363                 hammer2_chain_unlock(chain);
1364                 hammer2_chain_drop(chain);
1365                 chain = NULL;
1366         } else {
1367                 hmp = NULL;
1368         }
1369
1370         /*
1371          * Only flush the volume header if asked to, plus the inode must also
1372          * be the PFS root.
1373          */
1374         if ((xop->head.flags & HAMMER2_XOP_VOLHDR) == 0)
1375                 goto skip;
1376         if (ispfsroot == 0)
1377                 goto skip;
1378
1379         /*
1380          * Flush volume roots.  Avoid replication, we only want to
1381          * flush each hammer2_dev (hmp) once.
1382          */
1383         for (j = clindex - 1; j >= 0; --j) {
1384                 if ((chain = ip->cluster.array[j].chain) != NULL) {
1385                         if (chain->hmp == hmp) {
1386                                 chain = NULL;   /* safety */
1387                                 goto skip;
1388                         }
1389                 }
1390         }
1391         chain = NULL;   /* safety */
1392
1393         /*
1394          * spmp transaction.  The super-root is never directly mounted so
1395          * there shouldn't be any vnodes, let alone any dirty vnodes
1396          * associated with it, so we shouldn't have to mess around with any
1397          * vnode flushes here.
1398          */
1399         hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1400
1401         /*
1402          * We must flush the superroot down to the PFS iroot.  Remember
1403          * that hammer2_chain_setflush() stops at inode boundaries, so
1404          * the pmp->iroot has been flushed and flagged down to the superroot,
1405          * but the volume root (vchain) probably has not yet been flagged.
1406          */
1407         if (hmp->spmp->iroot) {
1408                 chain = hmp->spmp->iroot->cluster.array[0].chain;
1409                 if (chain) {
1410                         hammer2_chain_ref(chain);
1411                         hammer2_chain_lock(chain, HAMMER2_RESOLVE_ALWAYS);
1412                         flush_error |=
1413                                 hammer2_flush(chain,
1414                                               HAMMER2_FLUSH_TOP |
1415                                               HAMMER2_FLUSH_INODE_STOP |
1416                                               HAMMER2_FLUSH_FSSYNC);
1417                         hammer2_chain_unlock(chain);
1418                         hammer2_chain_drop(chain);
1419                 }
1420         }
1421
1422         /*
1423          * Media mounts have two 'roots', vchain for the topology
1424          * and fchain for the free block table.  Flush both.
1425          *
1426          * Note that the topology and free block table are handled
1427          * independently, so the free block table can wind up being
1428          * ahead of the topology.  We depend on the bulk free scan
1429          * code to deal with any loose ends.
1430          *
1431          * vchain and fchain do not error on-lock since their data does
1432          * not have to be re-read from media.
1433          */
1434         hammer2_chain_ref(&hmp->vchain);
1435         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1436         hammer2_chain_ref(&hmp->fchain);
1437         hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1438         if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1439                 /*
1440                  * This will also modify vchain as a side effect,
1441                  * mark vchain as modified now.
1442                  */
1443                 hammer2_voldata_modify(hmp);
1444                 chain = &hmp->fchain;
1445                 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1446                 KKASSERT(chain == &hmp->fchain);
1447         }
1448         hammer2_chain_unlock(&hmp->fchain);
1449         hammer2_chain_unlock(&hmp->vchain);
1450         hammer2_chain_drop(&hmp->fchain);
1451         /* vchain dropped down below */
1452
1453         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1454         if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1455                 chain = &hmp->vchain;
1456                 flush_error |= hammer2_flush(chain, HAMMER2_FLUSH_TOP);
1457                 KKASSERT(chain == &hmp->vchain);
1458         }
1459         hammer2_chain_unlock(&hmp->vchain);
1460         hammer2_chain_drop(&hmp->vchain);
1461
1462         /*
1463          * We can't safely flush the volume header until we have
1464          * flushed any device buffers which have built up.
1465          *
1466          * XXX this isn't being incremental
1467          */
1468         TAILQ_FOREACH(e, &hmp->devvpl, entry) {
1469                 devvp = e->devvp;
1470                 KKASSERT(devvp);
1471                 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY);
1472                 fsync_error = VOP_FSYNC(devvp, MNT_WAIT, 0);
1473                 vn_unlock(devvp);
1474                 if (fsync_error || flush_error) {
1475                         kprintf("hammer2: sync error fsync=%d h2flush=0x%04x dev=%s\n",
1476                                 fsync_error, flush_error, e->path);
1477                 }
1478         }
1479
1480         /*
1481          * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1482          * volume header needs synchronization via hmp->volsync.
1483          *
1484          * XXX synchronize the flag & data with only this flush XXX
1485          */
1486         if (fsync_error == 0 && flush_error == 0 &&
1487             (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1488                 struct buf *bp;
1489                 int vol_error = 0;
1490
1491                 /*
1492                  * Synchronize the disk before flushing the volume
1493                  * header.
1494                  */
1495                 bp = getpbuf(NULL);
1496                 bp->b_bio1.bio_offset = 0;
1497                 bp->b_bufsize = 0;
1498                 bp->b_bcount = 0;
1499                 bp->b_cmd = BUF_CMD_FLUSH;
1500                 bp->b_bio1.bio_done = biodone_sync;
1501                 bp->b_bio1.bio_flags |= BIO_SYNC;
1502                 vn_strategy(hmp->devvp, &bp->b_bio1);
1503                 fsync_error = biowait(&bp->b_bio1, "h2vol");
1504                 relpbuf(bp, NULL);
1505
1506                 /*
1507                  * Then we can safely flush the version of the
1508                  * volume header synchronized by the flush code.
1509                  */
1510                 j = hmp->volhdrno + 1;
1511                 if (j < 0)
1512                         j = 0;
1513                 if (j >= HAMMER2_NUM_VOLHDRS)
1514                         j = 0;
1515                 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1516                     hmp->volsync.volu_size) {
1517                         j = 0;
1518                 }
1519                 if (hammer2_debug & 0x8000) {
1520                         /* debug only, avoid syslogd loop */
1521                         kprintf("sync volhdr %d %jd\n",
1522                                 j, (intmax_t)hmp->volsync.volu_size);
1523                 }
1524                 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1525                             HAMMER2_PBUFSIZE, GETBLK_KVABIO, 0);
1526                 atomic_clear_int(&hmp->vchain.flags,
1527                                  HAMMER2_CHAIN_VOLUMESYNC);
1528                 bkvasync(bp);
1529                 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1530                 vol_error = bwrite(bp);
1531                 hmp->volhdrno = j;
1532                 if (vol_error)
1533                         fsync_error = vol_error;
1534         }
1535         if (flush_error)
1536                 total_error = flush_error;
1537         if (fsync_error)
1538                 total_error = hammer2_errno_to_error(fsync_error);
1539
1540         /* spmp trans */
1541         hammer2_trans_done(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1542 skip:
1543         hammer2_xop_feed(&xop->head, NULL, clindex, total_error);
1544 }