sys/vfs/hammer2/hammer2_flush.c

   1 /*
   2  * Copyright (c) 2011-2015 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@dragonflybsd.org>
   6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in
  16  *    the documentation and/or other materials provided with the
  17  *    distribution.
  18  * 3. Neither the name of The DragonFly Project nor the names of its
  19  *    contributors may be used to endorse or promote products derived
  20  *    from this software without specific, prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  26  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  28  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  29  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  30  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  31  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  32  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  */
  35 /*
  36  *                      TRANSACTION AND FLUSH HANDLING
  37  *
  38  * Deceptively simple but actually fairly difficult to implement properly is
  39  * how I would describe it.
  40  *
  41  * Flushing generally occurs bottom-up but requires a top-down scan to
  42  * locate chains with MODIFIED and/or UPDATE bits set.  The ONFLUSH flag
  43  * tells how to recurse downward to find these chains.
  44  */
  45
  46 #include <sys/cdefs.h>
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/types.h>
  50 #include <sys/lock.h>
  51 #include <sys/uuid.h>
  52
  53 #include "hammer2.h"
  54
  55 #define FLUSH_DEBUG 0
  56
  57 #define HAMMER2_FLUSH_DEPTH_LIMIT       10      /* stack recursion limit */
  58
  59
  60 /*
  61  * Recursively flush the specified chain.  The chain is locked and
  62  * referenced by the caller and will remain so on return.  The chain
  63  * will remain referenced throughout but can temporarily lose its
  64  * lock during the recursion to avoid unnecessarily stalling user
  65  * processes.
  66  */
  67 struct hammer2_flush_info {
  68         hammer2_chain_t *parent;
  69         int             depth;
  70         int             diddeferral;
  71         int             cache_index;
  72         hammer2_tid_t   mtid;
  73         struct h2_flush_list flushq;
  74         hammer2_chain_t *debug;
  75 };
  76
  77 typedef struct hammer2_flush_info hammer2_flush_info_t;
  78
  79 static void hammer2_flush_core(hammer2_flush_info_t *info,
  80                                 hammer2_chain_t *chain, int deleting);
  81 static int hammer2_flush_recurse(hammer2_chain_t *child, void *data);
  82
  83 /*
  84  * Any per-pfs transaction initialization goes here.
  85  */
  86 void
  87 hammer2_trans_manage_init(hammer2_pfs_t *pmp)
  88 {
  89 }
  90
  91 /*
  92  * Transaction support for any modifying operation.  Transactions are used
  93  * in the pmp layer by the frontend and in the spmp layer by the backend.
  94  *
  95  * 0                    - Normal transaction, interlocked against flush
  96  *                        transaction.
  97  *
  98  * TRANS_ISFLUSH        - Flush transaction, interlocked against normal
  99  *                        transaction.
 100  *
 101  * TRANS_BUFCACHE       - Buffer cache transaction, no interlock.
 102  *
 103  * Initializing a new transaction allocates a transaction ID.  Typically
 104  * passed a pmp (hmp passed as NULL), indicating a cluster transaction.  Can
 105  * be passed a NULL pmp and non-NULL hmp to indicate a transaction on a single
 106  * media target.  The latter mode is used by the recovery code.
 107  *
 108  * TWO TRANSACTION IDs can run concurrently, where one is a flush and the
 109  * other is a set of any number of concurrent filesystem operations.  We
 110  * can either have <running_fs_ops> + <waiting_flush> + <blocked_fs_ops>
 111  * or we can have <running_flush> + <concurrent_fs_ops>.
 112  *
 113  * During a flush, new fs_ops are only blocked until the fs_ops prior to
 114  * the flush complete.  The new fs_ops can then run concurrent with the flush.
 115  *
 116  * Buffer-cache transactions operate as fs_ops but never block.  A
 117  * buffer-cache flush will run either before or after the current pending
 118  * flush depending on its state.
 119  */
 120 void
 121 hammer2_trans_init(hammer2_pfs_t *pmp, uint32_t flags)
 122 {
 123         uint32_t oflags;
 124         uint32_t nflags;
 125         int dowait;
 126
 127         for (;;) {
 128                 oflags = pmp->trans.flags;
 129                 cpu_ccfence();
 130                 dowait = 0;
 131
 132                 if (flags & HAMMER2_TRANS_ISFLUSH) {
 133                         /*
 134                          * Requesting flush transaction.  Wait for all
 135                          * currently running transactions to finish.
 136                          */
 137                         if (oflags & HAMMER2_TRANS_MASK) {
 138                                 nflags = oflags | HAMMER2_TRANS_FPENDING |
 139                                                   HAMMER2_TRANS_WAITING;
 140                                 dowait = 1;
 141                         } else {
 142                                 nflags = (oflags | flags) + 1;
 143                         }
 144                 } else if (flags & HAMMER2_TRANS_BUFCACHE) {
 145                         /*
 146                          * Requesting strategy transaction.  Generally
 147                          * allowed in all situations unless a flush
 148                          * is running without the preflush flag.
 149                          */
 150                         if ((oflags & (HAMMER2_TRANS_ISFLUSH |
 151                                        HAMMER2_TRANS_PREFLUSH)) ==
 152                             HAMMER2_TRANS_ISFLUSH) {
 153                                 nflags = oflags | HAMMER2_TRANS_WAITING;
 154                                 dowait = 1;
 155                         } else {
 156                                 nflags = (oflags | flags) + 1;
 157                         }
 158                 } else {
 159                         /*
 160                          * Requesting normal transaction.  Wait for any
 161                          * flush to finish before allowing.
 162                          */
 163                         if (oflags & HAMMER2_TRANS_ISFLUSH) {
 164                                 nflags = oflags | HAMMER2_TRANS_WAITING;
 165                                 dowait = 1;
 166                         } else {
 167                                 nflags = (oflags | flags) + 1;
 168                         }
 169                 }
 170                 if (dowait)
 171                         tsleep_interlock(&pmp->trans.sync_wait, 0);
 172                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 173                         if (dowait == 0)
 174                                 break;
 175                         tsleep(&pmp->trans.sync_wait, PINTERLOCKED,
 176                                "h2trans", hz);
 177                 } else {
 178                         cpu_pause();
 179                 }
 180                 /* retry */
 181         }
 182 }
 183
 184 /*
 185  * Start a sub-transaction, there is no 'subdone' function.  This will
 186  * issue a new modify_tid (mtid) for the current transaction and must
 187  * be called for each XOP when multiple XOPs are run in sequence.
 188  */
 189 hammer2_tid_t
 190 hammer2_trans_sub(hammer2_pfs_t *pmp)
 191 {
 192         hammer2_tid_t mtid;
 193
 194         mtid = atomic_fetchadd_64(&pmp->modify_tid, 1);
 195
 196         return (mtid);
 197 }
 198
 199 /*
 200  * Clears the PREFLUSH stage, called during a flush transaction after all
 201  * logical buffer I/O has completed.
 202  */
 203 void
 204 hammer2_trans_clear_preflush(hammer2_pfs_t *pmp)
 205 {
 206         atomic_clear_int(&pmp->trans.flags, HAMMER2_TRANS_PREFLUSH);
 207 }
 208
 209 void
 210 hammer2_trans_done(hammer2_pfs_t *pmp)
 211 {
 212         uint32_t oflags;
 213         uint32_t nflags;
 214
 215         for (;;) {
 216                 oflags = pmp->trans.flags;
 217                 cpu_ccfence();
 218                 KKASSERT(oflags & HAMMER2_TRANS_MASK);
 219                 if ((oflags & HAMMER2_TRANS_MASK) == 1) {
 220                         /*
 221                          * This was the last transaction
 222                          */
 223                         nflags = (oflags - 1) & ~(HAMMER2_TRANS_ISFLUSH |
 224                                                   HAMMER2_TRANS_BUFCACHE |
 225                                                   HAMMER2_TRANS_PREFLUSH |
 226                                                   HAMMER2_TRANS_FPENDING |
 227                                                   HAMMER2_TRANS_WAITING);
 228                 } else {
 229                         /*
 230                          * Still transactions pending
 231                          */
 232                         nflags = oflags - 1;
 233                 }
 234                 if (atomic_cmpset_int(&pmp->trans.flags, oflags, nflags)) {
 235                         if ((nflags & HAMMER2_TRANS_MASK) == 0 &&
 236                             (oflags & HAMMER2_TRANS_WAITING)) {
 237                                 wakeup(&pmp->trans.sync_wait);
 238                         }
 239                         break;
 240                 } else {
 241                         cpu_pause();
 242                 }
 243                 /* retry */
 244         }
 245 }
 246
 247 /*
 248  * Obtain new, unique inode number (not serialized by caller).
 249  */
 250 hammer2_tid_t
 251 hammer2_trans_newinum(hammer2_pfs_t *pmp)
 252 {
 253         hammer2_tid_t tid;
 254
 255         tid = atomic_fetchadd_64(&pmp->inode_tid, 1);
 256
 257         return tid;
 258 }
 259
 260 /*
 261  * Assert that a strategy call is ok here.  Strategy calls are legal
 262  *
 263  * (1) In a normal transaction.
 264  * (2) In a flush transaction only if PREFLUSH is also set.
 265  */
 266 void
 267 hammer2_trans_assert_strategy(hammer2_pfs_t *pmp)
 268 {
 269         KKASSERT((pmp->trans.flags & HAMMER2_TRANS_ISFLUSH) == 0 ||
 270                  (pmp->trans.flags & HAMMER2_TRANS_PREFLUSH));
 271 }
 272
 273
 274 /*
 275  * Chains undergoing destruction are removed from the in-memory topology.
 276  * To avoid getting lost these chains are placed on the delayed flush
 277  * queue which will properly dispose of them.
 278  *
 279  * We do this instead of issuing an immediate flush in order to give
 280  * recursive deletions (rm -rf, etc) a chance to remove more of the
 281  * hierarchy, potentially allowing an enormous amount of write I/O to
 282  * be avoided.
 283  */
 284 void
 285 hammer2_delayed_flush(hammer2_chain_t *chain)
 286 {
 287         if ((chain->flags & HAMMER2_CHAIN_DELAYED) == 0) {
 288                 hammer2_spin_ex(&chain->hmp->list_spin);
 289                 if ((chain->flags & (HAMMER2_CHAIN_DELAYED |
 290                                      HAMMER2_CHAIN_DEFERRED)) == 0) {
 291                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_DELAYED |
 292                                                       HAMMER2_CHAIN_DEFERRED);
 293                         TAILQ_INSERT_TAIL(&chain->hmp->flushq,
 294                                           chain, flush_node);
 295                         hammer2_chain_ref(chain);
 296                 }
 297                 hammer2_spin_unex(&chain->hmp->list_spin);
 298         }
 299 }
 300
 301 /*
 302  * Flush the chain and all modified sub-chains through the specified
 303  * synchronization point, propagating parent chain modifications, modify_tid,
 304  * and mirror_tid updates back up as needed.
 305  *
 306  * Caller must have already vetted synchronization points to ensure they
 307  * are properly flushed.  Only snapshots and cluster flushes can create
 308  * these sorts of synchronization points.
 309  *
 310  * This routine can be called from several places but the most important
 311  * is from VFS_SYNC.
 312  *
 313  * chain is locked on call and will remain locked on return.  The chain's
 314  * UPDATE flag indicates that its parent's block table (which is not yet
 315  * part of the flush) should be updated.  The chain may be replaced by
 316  * the call if it was modified.
 317  */
 318 void
 319 hammer2_flush(hammer2_chain_t *chain, hammer2_tid_t mtid, int istop)
 320 {
 321         hammer2_chain_t *scan;
 322         hammer2_flush_info_t info;
 323         hammer2_dev_t *hmp;
 324         int loops;
 325
 326         /*
 327          * Execute the recursive flush and handle deferrals.
 328          *
 329          * Chains can be ridiculously long (thousands deep), so to
 330          * avoid blowing out the kernel stack the recursive flush has a
 331          * depth limit.  Elements at the limit are placed on a list
 332          * for re-execution after the stack has been popped.
 333          */
 334         bzero(&info, sizeof(info));
 335         TAILQ_INIT(&info.flushq);
 336         info.cache_index = -1;
 337         info.mtid = mtid;
 338
 339         /*
 340          * Calculate parent (can be NULL), if not NULL the flush core
 341          * expects the parent to be referenced so it can easily lock/unlock
 342          * it without it getting ripped up.
 343          */
 344         if ((info.parent = chain->parent) != NULL)
 345                 hammer2_chain_ref(info.parent);
 346
 347         /*
 348          * Extra ref needed because flush_core expects it when replacing
 349          * chain.
 350          */
 351         hammer2_chain_ref(chain);
 352         hmp = chain->hmp;
 353         loops = 0;
 354
 355         for (;;) {
 356                 /*
 357                  * Move hmp->flushq to info.flushq if non-empty so it can
 358                  * be processed.
 359                  */
 360                 if (TAILQ_FIRST(&hmp->flushq) != NULL) {
 361                         hammer2_spin_ex(&chain->hmp->list_spin);
 362                         TAILQ_CONCAT(&info.flushq, &hmp->flushq, flush_node);
 363                         hammer2_spin_unex(&chain->hmp->list_spin);
 364                 }
 365
 366                 /*
 367                  * Unwind deep recursions which had been deferred.  This
 368                  * can leave the FLUSH_* bits set for these chains, which
 369                  * will be handled when we [re]flush chain after the unwind.
 370                  */
 371                 while ((scan = TAILQ_FIRST(&info.flushq)) != NULL) {
 372                         KKASSERT(scan->flags & HAMMER2_CHAIN_DEFERRED);
 373                         TAILQ_REMOVE(&info.flushq, scan, flush_node);
 374                         atomic_clear_int(&scan->flags, HAMMER2_CHAIN_DEFERRED |
 375                                                        HAMMER2_CHAIN_DELAYED);
 376
 377                         /*
 378                          * Now that we've popped back up we can do a secondary
 379                          * recursion on the deferred elements.
 380                          *
 381                          * NOTE: hammer2_flush() may replace scan.
 382                          */
 383                         if (hammer2_debug & 0x0040)
 384                                 kprintf("deferred flush %p\n", scan);
 385                         hammer2_chain_lock(scan, HAMMER2_RESOLVE_MAYBE);
 386                         hammer2_flush(scan, mtid, 0);
 387                         hammer2_chain_unlock(scan);
 388                         hammer2_chain_drop(scan);       /* ref from deferral */
 389                 }
 390
 391                 /*
 392                  * [re]flush chain.
 393                  */
 394                 info.diddeferral = 0;
 395                 hammer2_flush_core(&info, chain, istop);
 396
 397                 /*
 398                  * Only loop if deep recursions have been deferred.
 399                  */
 400                 if (TAILQ_EMPTY(&info.flushq))
 401                         break;
 402
 403                 if (++loops % 1000 == 0) {
 404                         kprintf("hammer2_flush: excessive loops on %p\n",
 405                                 chain);
 406                         if (hammer2_debug & 0x100000)
 407                                 Debugger("hell4");
 408                 }
 409         }
 410         hammer2_chain_drop(chain);
 411         if (info.parent)
 412                 hammer2_chain_drop(info.parent);
 413 }
 414
 415 /*
 416  * This is the core of the chain flushing code.  The chain is locked by the
 417  * caller and must also have an extra ref on it by the caller, and remains
 418  * locked and will have an extra ref on return.  Upon return, the caller can
 419  * test the UPDATE bit on the child to determine if the parent needs updating.
 420  *
 421  * (1) Determine if this node is a candidate for the flush, return if it is
 422  *     not.  fchain and vchain are always candidates for the flush.
 423  *
 424  * (2) If we recurse too deep the chain is entered onto the deferral list and
 425  *     the current flush stack is aborted until after the deferral list is
 426  *     run.
 427  *
 428  * (3) Recursively flush live children (rbtree).  This can create deferrals.
 429  *     A successful flush clears the MODIFIED and UPDATE bits on the children
 430  *     and typically causes the parent to be marked MODIFIED as the children
 431  *     update the parent's block table.  A parent might already be marked
 432  *     MODIFIED due to a deletion (whos blocktable update in the parent is
 433  *     handled by the frontend), or if the parent itself is modified by the
 434  *     frontend for other reasons.
 435  *
 436  * (4) Permanently disconnected sub-trees are cleaned up by the front-end.
 437  *     Deleted-but-open inodes can still be individually flushed via the
 438  *     filesystem syncer.
 439  *
 440  * (5) Note that an unmodified child may still need the block table in its
 441  *     parent updated (e.g. rename/move).  The child will have UPDATE set
 442  *     in this case.
 443  *
 444  *                      WARNING ON BREF MODIFY_TID/MIRROR_TID
 445  *
 446  * blockref.modify_tid is consistent only within a PFS, and will not be
 447  * consistent during synchronization.  mirror_tid is consistent across the
 448  * block device regardless of the PFS.
 449  */
 450 static void
 451 hammer2_flush_core(hammer2_flush_info_t *info, hammer2_chain_t *chain,
 452                    int istop)
 453 {
 454         hammer2_chain_t *parent;
 455         hammer2_dev_t *hmp;
 456         int diddeferral;
 457
 458         /*
 459          * (1) Optimize downward recursion to locate nodes needing action.
 460          *     Nothing to do if none of these flags are set.
 461          */
 462         if ((chain->flags & HAMMER2_CHAIN_FLUSH_MASK) == 0) {
 463                 if (hammer2_debug & 0x200) {
 464                         if (info->debug == NULL)
 465                                 info->debug = chain;
 466                 } else {
 467                         return;
 468                 }
 469         }
 470
 471         hmp = chain->hmp;
 472         diddeferral = info->diddeferral;
 473         parent = info->parent;          /* can be NULL */
 474
 475         /*
 476          * Downward search recursion
 477          */
 478         if (chain->flags & (HAMMER2_CHAIN_DEFERRED | HAMMER2_CHAIN_DELAYED)) {
 479                 /*
 480                  * Already deferred.
 481                  */
 482                 ++info->diddeferral;
 483         } else if (info->depth == HAMMER2_FLUSH_DEPTH_LIMIT) {
 484                 /*
 485                  * Recursion depth reached.
 486                  */
 487                 KKASSERT((chain->flags & HAMMER2_CHAIN_DELAYED) == 0);
 488                 hammer2_chain_ref(chain);
 489                 TAILQ_INSERT_TAIL(&info->flushq, chain, flush_node);
 490                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEFERRED);
 491                 ++info->diddeferral;
 492         } else if ((chain->flags & HAMMER2_CHAIN_PFSBOUNDARY) && istop == 0) {
 493                 /*
 494                  * We do not recurse through PFSROOTs.  PFSROOT flushes are
 495                  * handled by the related pmp's (whether mounted or not,
 496                  * including during recovery).
 497                  *
 498                  * But we must still process the PFSROOT chains for block
 499                  * table updates in their parent (which IS part of our flush).
 500                  *
 501                  * Note that the volume root, vchain, does not set this flag.
 502                  */
 503                 ;
 504         } else if (chain->flags & HAMMER2_CHAIN_ONFLUSH) {
 505                 /*
 506                  * Downward recursion search (actual flush occurs bottom-up).
 507                  * pre-clear ONFLUSH.  It can get set again due to races,
 508                  * which we want so the scan finds us again in the next flush.
 509                  * These races can also include
 510                  *
 511                  * Flush recursions stop at PFSROOT boundaries.  Each PFS
 512                  * must be individually flushed and then the root must
 513                  * be flushed.
 514                  */
 515                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_ONFLUSH);
 516                 info->parent = chain;
 517                 hammer2_spin_ex(&chain->core.spin);
 518                 RB_SCAN(hammer2_chain_tree, &chain->core.rbtree,
 519                         NULL, hammer2_flush_recurse, info);
 520                 hammer2_spin_unex(&chain->core.spin);
 521                 info->parent = parent;
 522                 if (info->diddeferral)
 523                         hammer2_chain_setflush(chain);
 524         }
 525
 526         /*
 527          * Now we are in the bottom-up part of the recursion.
 528          *
 529          * Do not update chain if lower layers were deferred.
 530          */
 531         if (info->diddeferral)
 532                 goto done;
 533
 534         /*
 535          * Propagate the DESTROY flag downwards.  This dummies up the flush
 536          * code and tries to invalidate related buffer cache buffers to
 537          * avoid the disk write.
 538          */
 539         if (parent && (parent->flags & HAMMER2_CHAIN_DESTROY))
 540                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_DESTROY);
 541
 542         /*
 543          * Chain was already modified or has become modified, flush it out.
 544          */
 545 again:
 546         if ((hammer2_debug & 0x200) &&
 547             info->debug &&
 548             (chain->flags & (HAMMER2_CHAIN_MODIFIED | HAMMER2_CHAIN_UPDATE))) {
 549                 hammer2_chain_t *scan = chain;
 550
 551                 kprintf("DISCONNECTED FLUSH %p->%p\n", info->debug, chain);
 552                 while (scan) {
 553                         kprintf("    chain %p [%08x] bref=%016jx:%02x\n",
 554                                 scan, scan->flags,
 555                                 scan->bref.key, scan->bref.type);
 556                         if (scan == info->debug)
 557                                 break;
 558                         scan = scan->parent;
 559                 }
 560         }
 561
 562         if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
 563                 /*
 564                  * Dispose of the modified bit.
 565                  *
 566                  * UPDATE should already be set.
 567                  * bref.mirror_tid should already be set.
 568                  */
 569                 KKASSERT((chain->flags & HAMMER2_CHAIN_UPDATE) ||
 570                          chain == &hmp->vchain);
 571                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
 572
 573                 /*
 574                  * Manage threads waiting for excessive dirty memory to
 575                  * be retired.
 576                  */
 577                 if (chain->pmp)
 578                         hammer2_pfs_memory_wakeup(chain->pmp);
 579
 580                 if ((chain->flags & HAMMER2_CHAIN_UPDATE) ||
 581                     chain == &hmp->vchain ||
 582                     chain == &hmp->fchain) {
 583                         /*
 584                          * Drop the ref from the MODIFIED bit we cleared,
 585                          * net -1 ref.
 586                          */
 587                         hammer2_chain_drop(chain);
 588                 } else {
 589                         /*
 590                          * Drop the ref from the MODIFIED bit we cleared and
 591                          * set a ref for the UPDATE bit we are setting.  Net
 592                          * 0 refs.
 593                          */
 594                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 595                 }
 596
 597                 /*
 598                  * Issue the flush.  This is indirect via the DIO.
 599                  *
 600                  * NOTE: A DELETED node that reaches this point must be
 601                  *       flushed for synchronization point consistency.
 602                  *
 603                  * NOTE: Even though MODIFIED was already set, the related DIO
 604                  *       might not be dirty due to a system buffer cache
 605                  *       flush and must be set dirty if we are going to make
 606                  *       further modifications to the buffer.  Chains with
 607                  *       embedded data don't need this.
 608                  */
 609                 if (hammer2_debug & 0x1000) {
 610                         kprintf("Flush %p.%d %016jx/%d data=%016jx",
 611                                 chain, chain->bref.type,
 612                                 (uintmax_t)chain->bref.key,
 613                                 chain->bref.keybits,
 614                                 (uintmax_t)chain->bref.data_off);
 615                 }
 616                 if (hammer2_debug & 0x2000) {
 617                         Debugger("Flush hell");
 618                 }
 619
 620                 /*
 621                  * Update chain CRCs for flush.
 622                  *
 623                  * NOTE: Volume headers are NOT flushed here as they require
 624                  *       special processing.
 625                  */
 626                 switch(chain->bref.type) {
 627                 case HAMMER2_BREF_TYPE_FREEMAP:
 628                         /*
 629                          * Update the volume header's freemap_tid to the
 630                          * freemap's flushing mirror_tid.
 631                          *
 632                          * (note: embedded data, do not call setdirty)
 633                          */
 634                         KKASSERT(hmp->vchain.flags & HAMMER2_CHAIN_MODIFIED);
 635                         KKASSERT(chain == &hmp->fchain);
 636                         hmp->voldata.freemap_tid = chain->bref.mirror_tid;
 637                         if (hammer2_debug & 0x8000) {
 638                                 /* debug only, avoid syslogd loop */
 639                                 kprintf("sync freemap mirror_tid %08jx\n",
 640                                         (intmax_t)chain->bref.mirror_tid);
 641                         }
 642
 643                         /*
 644                          * The freemap can be flushed independently of the
 645                          * main topology, but for the case where it is
 646                          * flushed in the same transaction, and flushed
 647                          * before vchain (a case we want to allow for
 648                          * performance reasons), make sure modifications
 649                          * made during the flush under vchain use a new
 650                          * transaction id.
 651                          *
 652                          * Otherwise the mount recovery code will get confused.
 653                          */
 654                         ++hmp->voldata.mirror_tid;
 655                         break;
 656                 case HAMMER2_BREF_TYPE_VOLUME:
 657                         /*
 658                          * The free block table is flushed by
 659                          * hammer2_vfs_sync() before it flushes vchain.
 660                          * We must still hold fchain locked while copying
 661                          * voldata to volsync, however.
 662                          *
 663                          * (note: embedded data, do not call setdirty)
 664                          */
 665                         hammer2_chain_lock(&hmp->fchain,
 666                                            HAMMER2_RESOLVE_ALWAYS);
 667                         hammer2_voldata_lock(hmp);
 668                         if (hammer2_debug & 0x8000) {
 669                                 /* debug only, avoid syslogd loop */
 670                                 kprintf("sync volume  mirror_tid %08jx\n",
 671                                         (intmax_t)chain->bref.mirror_tid);
 672                         }
 673
 674                         /*
 675                          * Update the volume header's mirror_tid to the
 676                          * main topology's flushing mirror_tid.  It is
 677                          * possible that voldata.mirror_tid is already
 678                          * beyond bref.mirror_tid due to the bump we made
 679                          * above in BREF_TYPE_FREEMAP.
 680                          */
 681                         if (hmp->voldata.mirror_tid < chain->bref.mirror_tid) {
 682                                 hmp->voldata.mirror_tid =
 683                                         chain->bref.mirror_tid;
 684                         }
 685
 686                         /*
 687                          * The volume header is flushed manually by the
 688                          * syncer, not here.  All we do here is adjust the
 689                          * crc's.
 690                          */
 691                         KKASSERT(chain->data != NULL);
 692                         KKASSERT(chain->dio == NULL);
 693
 694                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT1]=
 695                                 hammer2_icrc32(
 696                                         (char *)&hmp->voldata +
 697                                          HAMMER2_VOLUME_ICRC1_OFF,
 698                                         HAMMER2_VOLUME_ICRC1_SIZE);
 699                         hmp->voldata.icrc_sects[HAMMER2_VOL_ICRC_SECT0]=
 700                                 hammer2_icrc32(
 701                                         (char *)&hmp->voldata +
 702                                          HAMMER2_VOLUME_ICRC0_OFF,
 703                                         HAMMER2_VOLUME_ICRC0_SIZE);
 704                         hmp->voldata.icrc_volheader =
 705                                 hammer2_icrc32(
 706                                         (char *)&hmp->voldata +
 707                                          HAMMER2_VOLUME_ICRCVH_OFF,
 708                                         HAMMER2_VOLUME_ICRCVH_SIZE);
 709
 710                         if (hammer2_debug & 0x8000) {
 711                                 /* debug only, avoid syslogd loop */
 712                                 kprintf("syncvolhdr %016jx %016jx\n",
 713                                         hmp->voldata.mirror_tid,
 714                                         hmp->vchain.bref.mirror_tid);
 715                         }
 716                         hmp->volsync = hmp->voldata;
 717                         atomic_set_int(&chain->flags, HAMMER2_CHAIN_VOLUMESYNC);
 718                         hammer2_voldata_unlock(hmp);
 719                         hammer2_chain_unlock(&hmp->fchain);
 720                         break;
 721                 case HAMMER2_BREF_TYPE_DATA:
 722                         /*
 723                          * Data elements have already been flushed via the
 724                          * logical file buffer cache.  Their hash was set in
 725                          * the bref by the vop_write code.  Do not re-dirty.
 726                          *
 727                          * Make sure any device buffer(s) have been flushed
 728                          * out here (there aren't usually any to flush) XXX.
 729                          */
 730                         break;
 731                 case HAMMER2_BREF_TYPE_INDIRECT:
 732                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
 733                 case HAMMER2_BREF_TYPE_FREEMAP_LEAF:
 734                         /*
 735                          * Buffer I/O will be cleaned up when the volume is
 736                          * flushed (but the kernel is free to flush it before
 737                          * then, as well).
 738                          */
 739                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
 740                         hammer2_chain_setcheck(chain, chain->data);
 741                         break;
 742                 case HAMMER2_BREF_TYPE_INODE:
 743                         /*
 744                          * NOTE: We must call io_setdirty() to make any late
 745                          *       changes to the inode data, the system might
 746                          *       have already flushed the buffer.
 747                          */
 748                         if (chain->data->ipdata.meta.op_flags &
 749                             HAMMER2_OPFLAG_PFSROOT) {
 750                                 /*
 751                                  * non-NULL pmp if mounted as a PFS.  We must
 752                                  * sync fields cached in the pmp? XXX
 753                                  */
 754                                 hammer2_inode_data_t *ipdata;
 755
 756                                 hammer2_io_setdirty(chain->dio);
 757                                 ipdata = &chain->data->ipdata;
 758                                 if (chain->pmp) {
 759                                         ipdata->meta.pfs_inum =
 760                                                 chain->pmp->inode_tid;
 761                                 }
 762                         } else {
 763                                 /* can't be mounted as a PFS */
 764                         }
 765
 766                         KKASSERT((chain->flags & HAMMER2_CHAIN_EMBEDDED) == 0);
 767                         hammer2_chain_setcheck(chain, chain->data);
 768                         break;
 769                 default:
 770                         KKASSERT(chain->flags & HAMMER2_CHAIN_EMBEDDED);
 771                         panic("hammer2_flush_core: unsupported "
 772                               "embedded bref %d",
 773                               chain->bref.type);
 774                         /* NOT REACHED */
 775                 }
 776
 777                 /*
 778                  * If the chain was destroyed try to avoid unnecessary I/O.
 779                  * (this only really works if the DIO system buffer is the
 780                  * same size as chain->bytes).
 781                  */
 782                 if ((chain->flags & HAMMER2_CHAIN_DESTROY) && chain->dio) {
 783                         hammer2_io_setinval(chain->dio, chain->bytes);
 784                 }
 785         }
 786
 787         /*
 788          * If UPDATE is set the parent block table may need to be updated.
 789          *
 790          * NOTE: UPDATE may be set on vchain or fchain in which case
 791          *       parent could be NULL.  It's easiest to allow the case
 792          *       and test for NULL.  parent can also wind up being NULL
 793          *       due to a deletion so we need to handle the case anyway.
 794          *
 795          * If no parent exists we can just clear the UPDATE bit.  If the
 796          * chain gets reattached later on the bit will simply get set
 797          * again.
 798          */
 799         if ((chain->flags & HAMMER2_CHAIN_UPDATE) && parent == NULL) {
 800                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 801                 hammer2_chain_drop(chain);
 802         }
 803
 804         /*
 805          * The chain may need its blockrefs updated in the parent.  This
 806          * requires some fancy footwork.
 807          */
 808         if (chain->flags & HAMMER2_CHAIN_UPDATE) {
 809                 hammer2_blockref_t *base;
 810                 int count;
 811
 812                 /*
 813                  * Both parent and chain must be locked.  This requires
 814                  * temporarily unlocking the chain.  We have to deal with
 815                  * the case where the chain might be reparented or modified
 816                  * while it was unlocked.
 817                  */
 818                 hammer2_chain_unlock(chain);
 819                 hammer2_chain_lock(parent, HAMMER2_RESOLVE_ALWAYS);
 820                 hammer2_chain_lock(chain, HAMMER2_RESOLVE_MAYBE);
 821                 if (chain->parent != parent) {
 822                         kprintf("PARENT MISMATCH ch=%p p=%p/%p\n",
 823                                 chain, chain->parent, parent);
 824                         hammer2_chain_unlock(parent);
 825                         goto done;
 826                 }
 827
 828                 /*
 829                  * Check race condition.  If someone got in and modified
 830                  * it again while it was unlocked, we have to loop up.
 831                  */
 832                 if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
 833                         hammer2_chain_unlock(parent);
 834                         kprintf("hammer2_flush: chain %p flush-mod race\n",
 835                                 chain);
 836                         goto again;
 837                 }
 838
 839                 /*
 840                  * Clear UPDATE flag, mark parent modified, update its
 841                  * modify_tid if necessary, and adjust the parent blockmap.
 842                  */
 843                 if (chain->flags & HAMMER2_CHAIN_UPDATE) {
 844                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_UPDATE);
 845                         hammer2_chain_drop(chain);
 846                 }
 847
 848                 /*
 849                  * (optional code)
 850                  *
 851                  * Avoid actually modifying and updating the parent if it
 852                  * was flagged for destruction.  This can greatly reduce
 853                  * disk I/O in large tree removals because the
 854                  * hammer2_io_setinval() call in the upward recursion
 855                  * (see MODIFIED code above) can only handle a few cases.
 856                  */
 857                 if (parent->flags & HAMMER2_CHAIN_DESTROY) {
 858                         if (parent->bref.modify_tid < chain->bref.modify_tid) {
 859                                 parent->bref.modify_tid =
 860                                         chain->bref.modify_tid;
 861                         }
 862                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_BMAPPED |
 863                                                         HAMMER2_CHAIN_BMAPUPD);
 864                         hammer2_chain_unlock(parent);
 865                         goto skipupdate;
 866                 }
 867
 868                 /*
 869                  * We are updating the parent's blockmap, the parent must
 870                  * be set modified.
 871                  */
 872                 hammer2_chain_modify(parent, info->mtid, 0);
 873                 if (parent->bref.modify_tid < chain->bref.modify_tid)
 874                         parent->bref.modify_tid = chain->bref.modify_tid;
 875
 876                 /*
 877                  * Calculate blockmap pointer
 878                  */
 879                 switch(parent->bref.type) {
 880                 case HAMMER2_BREF_TYPE_INODE:
 881                         /*
 882                          * Access the inode's block array.  However, there is
 883                          * no block array if the inode is flagged DIRECTDATA.
 884                          */
 885                         if (parent->data &&
 886                             (parent->data->ipdata.meta.op_flags &
 887                              HAMMER2_OPFLAG_DIRECTDATA) == 0) {
 888                                 base = &parent->data->
 889                                         ipdata.u.blockset.blockref[0];
 890                         } else {
 891                                 base = NULL;
 892                         }
 893                         count = HAMMER2_SET_COUNT;
 894                         break;
 895                 case HAMMER2_BREF_TYPE_INDIRECT:
 896                 case HAMMER2_BREF_TYPE_FREEMAP_NODE:
 897                         if (parent->data)
 898                                 base = &parent->data->npdata[0];
 899                         else
 900                                 base = NULL;
 901                         count = parent->bytes / sizeof(hammer2_blockref_t);
 902                         break;
 903                 case HAMMER2_BREF_TYPE_VOLUME:
 904                         base = &chain->hmp->voldata.sroot_blockset.blockref[0];
 905                         count = HAMMER2_SET_COUNT;
 906                         break;
 907                 case HAMMER2_BREF_TYPE_FREEMAP:
 908                         base = &parent->data->npdata[0];
 909                         count = HAMMER2_SET_COUNT;
 910                         break;
 911                 default:
 912                         base = NULL;
 913                         count = 0;
 914                         panic("hammer2_flush_core: "
 915                               "unrecognized blockref type: %d",
 916                               parent->bref.type);
 917                 }
 918
 919                 /*
 920                  * Blocktable updates
 921                  *
 922                  * We synchronize pending statistics at this time.  Delta
 923                  * adjustments designated for the current and upper level
 924                  * are synchronized.
 925                  */
 926                 if (base && (chain->flags & HAMMER2_CHAIN_BMAPUPD)) {
 927                         if (chain->flags & HAMMER2_CHAIN_BMAPPED) {
 928                                 hammer2_spin_ex(&parent->core.spin);
 929                                 hammer2_base_delete(parent, base, count,
 930                                                     &info->cache_index, chain);
 931                                 hammer2_spin_unex(&parent->core.spin);
 932                                 /* base_delete clears both bits */
 933                         } else {
 934                                 atomic_clear_int(&chain->flags,
 935                                                  HAMMER2_CHAIN_BMAPUPD);
 936                         }
 937                 }
 938                 if (base && (chain->flags & HAMMER2_CHAIN_BMAPPED) == 0) {
 939                         hammer2_spin_ex(&parent->core.spin);
 940                         hammer2_base_insert(parent, base, count,
 941                                             &info->cache_index, chain);
 942                         hammer2_spin_unex(&parent->core.spin);
 943                         /* base_insert sets BMAPPED */
 944                 }
 945                 hammer2_chain_unlock(parent);
 946         }
 947 skipupdate:
 948         ;
 949
 950         /*
 951          * Final cleanup after flush
 952          */
 953 done:
 954         KKASSERT(chain->refs > 0);
 955         if (hammer2_debug & 0x200) {
 956                 if (info->debug == chain)
 957                         info->debug = NULL;
 958         }
 959 }
 960
 961 /*
 962  * Flush recursion helper, called from flush_core, calls flush_core.
 963  *
 964  * Flushes the children of the caller's chain (info->parent), restricted
 965  * by sync_tid.  Set info->domodify if the child's blockref must propagate
 966  * back up to the parent.
 967  *
 968  * Ripouts can move child from rbtree to dbtree or dbq but the caller's
 969  * flush scan order prevents any chains from being lost.  A child can be
 970  * executes more than once.
 971  *
 972  * WARNING! If we do not call hammer2_flush_core() we must update
 973  *          bref.mirror_tid ourselves to indicate that the flush has
 974  *          processed the child.
 975  *
 976  * WARNING! parent->core spinlock is held on entry and return.
 977  */
 978 static int
 979 hammer2_flush_recurse(hammer2_chain_t *child, void *data)
 980 {
 981         hammer2_flush_info_t *info = data;
 982         hammer2_chain_t *parent = info->parent;
 983
 984         /*
 985          * (child can never be fchain or vchain so a special check isn't
 986          *  needed).
 987          *
 988          * We must ref the child before unlocking the spinlock.
 989          *
 990          * The caller has added a ref to the parent so we can temporarily
 991          * unlock it in order to lock the child.
 992          */
 993         hammer2_chain_ref(child);
 994         hammer2_spin_unex(&parent->core.spin);
 995
 996         hammer2_chain_unlock(parent);
 997         hammer2_chain_lock(child, HAMMER2_RESOLVE_MAYBE);
 998
 999         /*
1000          * Recurse and collect deferral data.  We're in the media flush,
1001          * this can cross PFS boundaries.
1002          */
1003         if (child->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1004                 ++info->depth;
1005                 hammer2_flush_core(info, child, 0);
1006                 --info->depth;
1007         } else if (hammer2_debug & 0x200) {
1008                 if (info->debug == NULL)
1009                         info->debug = child;
1010                 ++info->depth;
1011                 hammer2_flush_core(info, child, 0);
1012                 --info->depth;
1013                 if (info->debug == child)
1014                         info->debug = NULL;
1015         }
1016
1017         /*
1018          * Relock to continue the loop
1019          */
1020         hammer2_chain_unlock(child);
1021         hammer2_chain_lock(parent, HAMMER2_RESOLVE_MAYBE);
1022         hammer2_chain_drop(child);
1023         KKASSERT(info->parent == parent);
1024         hammer2_spin_ex(&parent->core.spin);
1025
1026         return (0);
1027 }
1028
1029 /*
1030  * flush helper (backend threaded)
1031  *
1032  * Flushes core chains, issues disk sync, flushes volume roots.
1033  *
1034  * Primarily called from vfs_sync().
1035  */
1036 void
1037 hammer2_inode_xop_flush(hammer2_xop_t *arg, int clindex)
1038 {
1039         hammer2_xop_flush_t *xop = &arg->xop_flush;
1040         hammer2_chain_t *chain;
1041         hammer2_chain_t *parent;
1042         hammer2_dev_t *hmp;
1043         int error = 0;
1044         int total_error = 0;
1045         int j;
1046
1047         /*
1048          * Flush core chains
1049          */
1050         chain = hammer2_inode_chain(xop->head.ip1, clindex,
1051                                     HAMMER2_RESOLVE_ALWAYS);
1052         if (chain) {
1053                 hmp = chain->hmp;
1054                 if (chain->flags & HAMMER2_CHAIN_FLUSH_MASK) {
1055                         hammer2_flush(chain, xop->head.mtid, 1);
1056                         parent = chain->parent;
1057                         KKASSERT(chain->pmp != parent->pmp);
1058                         hammer2_chain_setflush(parent);
1059                 }
1060                 hammer2_chain_unlock(chain);
1061                 hammer2_chain_drop(chain);
1062                 chain = NULL;
1063         } else {
1064                 hmp = NULL;
1065         }
1066
1067         /*
1068          * Flush volume roots.  Avoid replication, we only want to
1069          * flush each hammer2_dev (hmp) once.
1070          */
1071         for (j = clindex - 1; j >= 0; --j) {
1072                 if ((chain = xop->head.ip1->cluster.array[j].chain) != NULL) {
1073                         if (chain->hmp == hmp) {
1074                                 chain = NULL;   /* safety */
1075                                 goto skip;
1076                         }
1077                 }
1078         }
1079         chain = NULL;   /* safety */
1080
1081         /*
1082          * spmp transaction.  The super-root is never directly mounted so
1083          * there shouldn't be any vnodes, let alone any dirty vnodes
1084          * associated with it.
1085          */
1086         hammer2_trans_init(hmp->spmp, HAMMER2_TRANS_ISFLUSH);
1087
1088         /*
1089          * Media mounts have two 'roots', vchain for the topology
1090          * and fchain for the free block table.  Flush both.
1091          *
1092          * Note that the topology and free block table are handled
1093          * independently, so the free block table can wind up being
1094          * ahead of the topology.  We depend on the bulk free scan
1095          * code to deal with any loose ends.
1096          */
1097         hammer2_chain_ref(&hmp->vchain);
1098         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1099         hammer2_chain_ref(&hmp->fchain);
1100         hammer2_chain_lock(&hmp->fchain, HAMMER2_RESOLVE_ALWAYS);
1101         if (hmp->fchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1102                 /*
1103                  * This will also modify vchain as a side effect,
1104                  * mark vchain as modified now.
1105                  */
1106                 hammer2_voldata_modify(hmp);
1107                 chain = &hmp->fchain;
1108                 hammer2_flush(chain, xop->head.mtid, 1);
1109                 KKASSERT(chain == &hmp->fchain);
1110         }
1111         hammer2_chain_unlock(&hmp->fchain);
1112         hammer2_chain_unlock(&hmp->vchain);
1113         hammer2_chain_drop(&hmp->fchain);
1114         /* vchain dropped down below */
1115
1116         hammer2_chain_lock(&hmp->vchain, HAMMER2_RESOLVE_ALWAYS);
1117         if (hmp->vchain.flags & HAMMER2_CHAIN_FLUSH_MASK) {
1118                 chain = &hmp->vchain;
1119                 hammer2_flush(chain, xop->head.mtid, 1);
1120                 KKASSERT(chain == &hmp->vchain);
1121         }
1122         hammer2_chain_unlock(&hmp->vchain);
1123         hammer2_chain_drop(&hmp->vchain);
1124
1125         error = 0;
1126
1127         /*
1128          * We can't safely flush the volume header until we have
1129          * flushed any device buffers which have built up.
1130          *
1131          * XXX this isn't being incremental
1132          */
1133         vn_lock(hmp->devvp, LK_EXCLUSIVE | LK_RETRY);
1134         error = VOP_FSYNC(hmp->devvp, MNT_WAIT, 0);
1135         vn_unlock(hmp->devvp);
1136
1137         /*
1138          * The flush code sets CHAIN_VOLUMESYNC to indicate that the
1139          * volume header needs synchronization via hmp->volsync.
1140          *
1141          * XXX synchronize the flag & data with only this flush XXX
1142          */
1143         if (error == 0 &&
1144             (hmp->vchain.flags & HAMMER2_CHAIN_VOLUMESYNC)) {
1145                 struct buf *bp;
1146
1147                 /*
1148                  * Synchronize the disk before flushing the volume
1149                  * header.
1150                  */
1151                 bp = getpbuf(NULL);
1152                 bp->b_bio1.bio_offset = 0;
1153                 bp->b_bufsize = 0;
1154                 bp->b_bcount = 0;
1155                 bp->b_cmd = BUF_CMD_FLUSH;
1156                 bp->b_bio1.bio_done = biodone_sync;
1157                 bp->b_bio1.bio_flags |= BIO_SYNC;
1158                 vn_strategy(hmp->devvp, &bp->b_bio1);
1159                 biowait(&bp->b_bio1, "h2vol");
1160                 relpbuf(bp, NULL);
1161
1162                 /*
1163                  * Then we can safely flush the version of the
1164                  * volume header synchronized by the flush code.
1165                  */
1166                 j = hmp->volhdrno + 1;
1167                 if (j >= HAMMER2_NUM_VOLHDRS)
1168                         j = 0;
1169                 if (j * HAMMER2_ZONE_BYTES64 + HAMMER2_SEGSIZE >
1170                     hmp->volsync.volu_size) {
1171                         j = 0;
1172                 }
1173                 if (hammer2_debug & 0x8000) {
1174                         /* debug only, avoid syslogd loop */
1175                         kprintf("sync volhdr %d %jd\n",
1176                                 j, (intmax_t)hmp->volsync.volu_size);
1177                 }
1178                 bp = getblk(hmp->devvp, j * HAMMER2_ZONE_BYTES64,
1179                             HAMMER2_PBUFSIZE, 0, 0);
1180                 atomic_clear_int(&hmp->vchain.flags,
1181                                  HAMMER2_CHAIN_VOLUMESYNC);
1182                 bcopy(&hmp->volsync, bp->b_data, HAMMER2_PBUFSIZE);
1183                 bawrite(bp);
1184                 hmp->volhdrno = j;
1185         }
1186         if (error)
1187                 total_error = error;
1188
1189         hammer2_trans_done(hmp->spmp);  /* spmp trans */
1190 skip:
1191         error = hammer2_xop_feed(&xop->head, NULL, clindex, total_error);
1192 }